From 7e058b29e0909f8f445dc33556096eb6c5991b06 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 7 Jun 2019 00:09:14 -0500 Subject: [PATCH] * Removed hard coded data dir in daemon launch * Documented subprocess pow better * Calculate block size better for pow + Added monero donation address --- README.md | 8 +++- onionr/onionrcommands/daemonlaunch.py | 4 +- onionr/onionrproofs.py | 11 +---- onionr/subprocesspow.py | 65 +++++++++++++++------------ 4 files changed, 46 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index e5f18105..5de6fc78 100755 --- a/README.md +++ b/README.md @@ -69,12 +69,14 @@ More docs coming soon. The following applies to Ubuntu Bionic. Other distros may have different package or command names. +`$ sudo apt install python3-pip python3-dev tor` + * Have python3.6+, python3-pip, Tor (daemon, not browser) installed (python3-dev recommended) * Clone the git repo: `$ git clone https://gitlab.com/beardog/onionr` * cd into install direction: `$ cd onionr/` * Install the Python dependencies ([virtualenv strongly recommended](https://virtualenv.pypa.io/en/stable/userguide/)): `$ pip3 install --require-hashes -r requirements.txt` -(--require-hashes is intended to prevent exploitation via compromise of Pypi/CA certificates) +(--require-hashes is intended to prevent exploitation via compromise of PyPi/CA certificates) ## Help out @@ -95,7 +97,9 @@ Contribute money: Donating at least $5 gets you cool Onionr stickers. Get in touch if you want them. -Bitcoin: [1onion55FXzm6h8KQw3zFw2igpHcV7LPq](bitcoin:1onion55FXzm6h8KQw3zFw2igpHcV7LPq) (Contact us for privacy coins like Monero) +Bitcoin: [1onion55FXzm6h8KQw3zFw2igpHcV7LPq](bitcoin:1onion55FXzm6h8KQw3zFw2igpHcV7LPq) (Contact us for a unique address or for other coins) + +Monero: 4B5BA24d1P3R5aWEpkGY5TP7buJJcn2aSGBVRQCHhpiahxeB4aWsu15XwmuTjC6VF62NApZeJGTS248RMVECP8aW73Uj2ax USD (Card/Paypal): [Ko-Fi](https://www.ko-fi.com/beardogkf) diff --git a/onionr/onionrcommands/daemonlaunch.py b/onionr/onionrcommands/daemonlaunch.py index 9dab9f57..98708258 100644 --- a/onionr/onionrcommands/daemonlaunch.py +++ b/onionr/onionrcommands/daemonlaunch.py @@ -29,9 +29,9 @@ def daemon(o_inst): ''' # remove runcheck if it exists - if os.path.isfile('data/.runcheck'): + if os.path.isfile('%s/.runcheck' % (o_inst.onionrCore.dataDir,)): logger.debug('Runcheck file found on daemon start, deleting in advance.') - os.remove('data/.runcheck') + os.remove('%s/.runcheck' % (o_inst.onionrCore.dataDir,)) Thread(target=api.API, args=(o_inst, o_inst.debug, onionr.API_VERSION)).start() Thread(target=api.PublicAPI, args=[o_inst.getClientApi()]).start() diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py index bd0600cf..287ba30f 100755 --- a/onionr/onionrproofs.py +++ b/onionr/onionrproofs.py @@ -53,14 +53,9 @@ def getDifficultyForNewBlock(data, ourBlock=True): dataSize = 0 if isinstance(data, onionrblockapi.Block): dataSize = len(data.getRaw().encode('utf-8')) - elif isinstance(data, str): - dataSize = len(data.encode('utf-8')) - elif isinstance(data, bytes): - dataSize = len(data) - elif isinstance(data, int): - dataSize = data else: - raise ValueError('not Block, str, or int') + dataSize = len(onionrutils.OnionrUtils.strToBytes(data)) + if ourBlock: minDifficulty = config.get('general.minimum_send_pow', 4) else: @@ -131,8 +126,6 @@ class DataPOW: for i in range(max(1, threadCount)): t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,myCore)) t.start() - - return def pow(self, reporting = False, myCore = None): startTime = math.floor(time.time()) diff --git a/onionr/subprocesspow.py b/onionr/subprocesspow.py index fee2f2dd..6df08ca2 100755 --- a/onionr/subprocesspow.py +++ b/onionr/subprocesspow.py @@ -1,34 +1,39 @@ #!/usr/bin/env python3 -import subprocess, sys, os -import multiprocessing, threading, time, json, math, binascii +import subprocess, os +import multiprocessing, threading, time, json, math from multiprocessing import Pipe, Process import core, onionrblockapi, config, onionrutils, logger, onionrproofs class SubprocessPOW: - def __init__(self, data, metadata, core_inst=None, subprocCount=None): + def __init__(self, data, metadata, core_inst=None, subproc_count=None): + ''' + Onionr proof of work using multiple processes + Accepts block data, block metadata + and optionally an onionr core library instance. + if subproc_count is not set, os.cpu_count() is used to determine the number of processes + + Do to Python GIL multiprocessing or use of external libraries is necessary to accelerate CPU bound tasks + ''' + # Option to accept existing core instance to save memory if core_inst is None: core_inst = core.Core() - if subprocCount is None: - subprocCount = os.cpu_count() - self.subprocCount = subprocCount + # No known benefit to using more processes than there are cores. + # Note: os.cpu_count perhaps not always accurate + if subproc_count is None: + subproc_count = os.cpu_count() + self.subproc_count = subproc_count self.result = '' self.shutdown = False self.core_inst = core_inst self.data = data self.metadata = metadata - dataLen = len(data) + len(json.dumps(metadata)) + # dump dict to measure bytes of json metadata. Cannot reuse later because the pow token must be added + json_metadata = json.dumps(metadata).encode() - #if forceDifficulty > 0: - # self.difficulty = forceDifficulty - #else: - # Calculate difficulty. Dumb for now, may use good algorithm in the future. - self.difficulty = onionrproofs.getDifficultyForNewBlock(dataLen) - - try: - self.data = self.data.encode() - except AttributeError: - pass + self.data = onionrutils.OnionrUtils.strToBytes(data) + # Calculate difficulty. Dumb for now, may use good algorithm in the future. + self.difficulty = onionrproofs.getDifficultyForNewBlock(bytes(json_metadata + b'\n' + self.data)) logger.info('Computing POW (difficulty: %s)...' % self.difficulty) @@ -38,9 +43,10 @@ class SubprocessPOW: self.payload = None def start(self): - startTime = self.core_inst._utils.getEpoch() - for x in range(self.subprocCount): + # Create a new thread for each subprocess + for x in range(self.subproc_count): threading.Thread(target=self._spawn_proc).start() + # Monitor the processes for a payload, shut them down when its found while True: if self.payload is None: time.sleep(0.1) @@ -49,6 +55,7 @@ class SubprocessPOW: return self.payload def _spawn_proc(self): + # Create a child proof of work process, wait for data and send shutdown signal when its found parent_conn, child_conn = Pipe() p = Process(target=self.do_pow, args=(child_conn,)) p.start() @@ -67,24 +74,24 @@ class SubprocessPOW: self.payload = payload def do_pow(self, pipe): - nonce = int(binascii.hexlify(os.urandom(2)), 16) + nonce = -10000000 # Start nonce at negative 10 million so that the chosen nonce is likely to be small in length nonceStart = nonce data = self.data metadata = self.metadata puzzle = self.puzzle difficulty = self.difficulty - mcore = core.Core() + mcore = core.Core() # I think we make a new core here because of multiprocess bugs while True: - metadata['pow'] = nonce - payload = json.dumps(metadata).encode() + b'\n' + data - token = mcore._crypto.sha3Hash(payload) - try: - # on some versions, token is bytes - token = token.decode() - except AttributeError: - pass + # Break if shutdown received if pipe.poll() and pipe.recv() == 'shutdown': break + # Load nonce into block metadata + metadata['pow'] = nonce + # Serialize metadata, combine with block data + payload = json.dumps(metadata).encode() + b'\n' + data + # Check sha3_256 hash of block, compare to puzzle. Send payload if puzzle finished + token = mcore._crypto.sha3Hash(payload) + token = onionrutils.OnionrUtils.bytesToStr(token) # ensure token is string if puzzle == token[0:difficulty]: pipe.send(payload) break