From 9329b07e3b0c7b61b9b646f6a1cc527d261d7206 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 22 Dec 2019 13:42:10 -0600 Subject: [PATCH] added mixmate to improve base routing --- src/bigbrother/ministry/ofexec.py | 3 +- src/communicatorutils/daemonqueuehandler.py | 33 +++++---- .../downloadblocks/__init__.py | 15 ++-- .../uploadblocks/__init__.py | 1 + .../uploadblocks/mixmate/__init__.py | 51 ++++++++++++++ .../uploadblocks/mixmate/pool.py | 69 +++++++++++++++++++ src/communicatorutils/uploadblocks/session.py | 34 ++++----- src/etc/onionrvalues.py | 3 + src/httpapi/miscclientapi/endpoints.py | 17 ++--- src/httpapi/miscpublicapi/upload.py | 9 ++- src/onionrblocks/insert.py | 13 ++-- src/runtests/__init__.py | 11 ++- src/runtests/ownnode.py | 53 ++++++++++++++ src/runtests/stresstest.py | 1 + src/utils/gettransports.py | 27 ++++---- 15 files changed, 273 insertions(+), 67 deletions(-) create mode 100644 src/communicatorutils/uploadblocks/mixmate/__init__.py create mode 100644 src/communicatorutils/uploadblocks/mixmate/pool.py create mode 100644 src/runtests/ownnode.py diff --git a/src/bigbrother/ministry/ofexec.py b/src/bigbrother/ministry/ofexec.py index 48e28210..e26e7a10 100644 --- a/src/bigbrother/ministry/ofexec.py +++ b/src/bigbrother/ministry/ofexec.py @@ -57,6 +57,8 @@ def block_exec(event, info): ] home = identifyhome.identify_home() + code_b64 = base64.b64encode(info[0].co_code).decode() + for source in whitelisted_code: if info[0].co_filename.endswith(source): return @@ -64,7 +66,6 @@ def block_exec(event, info): if home + 'plugins/' in info[0].co_filename: return - code_b64 = base64.b64encode(info[0].co_code).decode() logger.warn('POSSIBLE EXPLOIT DETECTED, SEE LOGS', terminal=True) logger.warn('POSSIBLE EXPLOIT DETECTED: ' + info[0].co_filename) logger.warn('Prevented exec/eval. Report this with the sample below') diff --git a/src/communicatorutils/daemonqueuehandler.py b/src/communicatorutils/daemonqueuehandler.py index 0936c0df..9cddc793 100755 --- a/src/communicatorutils/daemonqueuehandler.py +++ b/src/communicatorutils/daemonqueuehandler.py @@ -1,9 +1,15 @@ -''' - Onionr - P2P Anonymous Storage Network +"""Onionr - P2P Anonymous Storage Network. - Handle daemon queue commands in the communicator -''' -''' +Handle daemon queue commands in the communicator +""" +import logger +from onionrplugins import onionrevents as events +from onionrutils import localcommand +from coredb import daemonqueue +import filepaths +from . import restarttor +from communicatorutils.uploadblocks import mixmate +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -16,13 +22,9 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' -import logger -from onionrplugins import onionrevents as events -from onionrutils import localcommand -from coredb import daemonqueue -import filepaths -from . import restarttor +""" + + def handle_daemon_commands(comm_inst): cmd = daemonqueue.daemon_queue() response = '' @@ -62,6 +64,13 @@ def handle_daemon_commands(comm_inst): i.count = (i.frequency - 1) elif cmd[0] == 'uploadBlock': comm_inst.blocksToUpload.append(cmd[1]) + elif cmd[0] == 'uploadEvent': + try: + mixmate.block_mixer(comm_inst.blocksToUpload, cmd[1]) + except ValueError: + pass + else: + localcommand.local_command('/waitforshare/' + cmd[1], post=True, maxWait=5) else: logger.debug('Received daemon queue command unable to be handled: %s' % (cmd[0],)) diff --git a/src/communicatorutils/downloadblocks/__init__.py b/src/communicatorutils/downloadblocks/__init__.py index 86c427d1..d532d5c9 100755 --- a/src/communicatorutils/downloadblocks/__init__.py +++ b/src/communicatorutils/downloadblocks/__init__.py @@ -1,8 +1,11 @@ -''' +""" Onionr - Private P2P Communication Download blocks using the communicator instance -''' +""" +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from communicator import OnionrCommunicatorDaemon import onionrexceptions import logger import onionrpeers @@ -12,12 +15,13 @@ from communicator import onlinepeers from onionrutils import blockmetadata from onionrutils import validatemetadata from coredb import blockmetadb +from coredb import daemonqueue import onionrcrypto import onionrstorage from onionrblocks import onionrblacklist from onionrblocks import storagecounter from . import shoulddownload -''' +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -30,11 +34,11 @@ from . import shoulddownload You should have received a copy of the GNU General Public License along with this program. If not, see . -''' +""" def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"): - '''Use communicator instance to download blocks in the comms's queue''' + """Use communicator instance to download blocks in the comms's queue""" blacklist = onionrblacklist.OnionrBlackList() storage_counter = storagecounter.StorageCounter() LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter @@ -109,6 +113,7 @@ def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"): removeFromQueue = False else: blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db + daemonqueue.daemon_queue_add('uploadEvent', blockHash) blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database else: logger.warn('POW failed for block %s.' % (blockHash,)) diff --git a/src/communicatorutils/uploadblocks/__init__.py b/src/communicatorutils/uploadblocks/__init__.py index baff0ae2..b1420925 100755 --- a/src/communicatorutils/uploadblocks/__init__.py +++ b/src/communicatorutils/uploadblocks/__init__.py @@ -27,6 +27,7 @@ from onionrutils import localcommand, stringvalidators, basicrequests from communicator import onlinepeers import onionrcrypto from . import sessionmanager +from . import mixmate def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon): """Accepts a communicator instance and uploads blocks from its upload queue""" diff --git a/src/communicatorutils/uploadblocks/mixmate/__init__.py b/src/communicatorutils/uploadblocks/mixmate/__init__.py new file mode 100644 index 00000000..ce50be7e --- /dev/null +++ b/src/communicatorutils/uploadblocks/mixmate/__init__.py @@ -0,0 +1,51 @@ +"""Onionr - Private P2P Communication. + +Delay block uploads, optionally mixing them together +""" +import time +from typing import List + +import onionrtypes +from onionrblocks import onionrblockapi + +from .pool import UploadPool +from .pool import PoolFullException + +from etc import onionrvalues + +""" + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" +upload_pool = UploadPool(4) + + +def block_mixer(upload_list: List[onionrtypes.BlockHash], + block_to_mix: onionrtypes.BlockHash): + """Delay and mix block inserts. + + Take a block list and a received/created block and add it + to the said block list + """ + bl = onionrblockapi.Block(block_to_mix) + if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE: + raise ValueError + + try: + # add the new block to pool + upload_pool.add_to_pool(block_to_mix) + except PoolFullException: + # If the pool is full, move into upload queue + upload_list.extend(upload_pool.get_pool()) + # then finally begin new pool with new block + upload_pool.add_to_pool(block_to_mix) diff --git a/src/communicatorutils/uploadblocks/mixmate/pool.py b/src/communicatorutils/uploadblocks/mixmate/pool.py new file mode 100644 index 00000000..8cc944c1 --- /dev/null +++ b/src/communicatorutils/uploadblocks/mixmate/pool.py @@ -0,0 +1,69 @@ +"""Onionr - Private P2P Communication. + +Upload pool +""" +from typing import List + +import onionrutils +import onionrtypes +from onionrcrypto import cryptoutils +""" + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + + +class PoolFullException(Exception): + """For when the UploadPool is full. + + Raise when a new hash is attempted to be added + """ + + +class PoolNotReady(Exception): + """Raise when UploadPool pool access is attempted without it being full.""" + + +class AlreadyInPool(Exception): + """Raise when a hash already in pool is attempted to be added again.""" + + +class UploadPool: + """Upload pool for mixing blocks together and delaying uploads.""" + + def __init__(self, pool_size: int): + """Create a new pool with a specified max size. + + Uses private var and getter to avoid direct adding + """ + self._pool: List[onionrtypes.BlockHash] = [] + self._pool_size = pool_size + self.birthday = onionrutils.epoch.get_epoch() + + def add_to_pool(self, item: List[onionrtypes.BlockHash]): + """Add a new hash to the pool. Raise PoolFullException if full.""" + if len(self._pool) >= self._pool_size: + raise PoolFullException + if not onionrutils.stringvalidators.validate_hash(item): + raise ValueError + self._pool.append(item) + + def get_pool(self) -> List[onionrtypes.BlockHash]: + """Get the hash pool in secure random order.""" + if len(self._pool) != self._pool_size: + raise PoolNotReady + final_pool: List[onionrtypes.BlockHash] = cryptoutils.random_shuffle(list(self._pool)) + + self._pool.clear() + self.birthday = onionrutils.epoch.get_epoch() + return final_pool diff --git a/src/communicatorutils/uploadblocks/session.py b/src/communicatorutils/uploadblocks/session.py index 6c0b3472..c3fdc2a9 100644 --- a/src/communicatorutils/uploadblocks/session.py +++ b/src/communicatorutils/uploadblocks/session.py @@ -1,9 +1,13 @@ -""" - Onionr - Private P2P Communication +"""Onionr - Private P2P Communication. - Virtual upload "sessions" for blocks +Virtual upload "sessions" for blocks """ -from __future__ import annotations +from typing import Union + +from onionrutils import stringvalidators +from onionrutils import bytesconverter +from onionrutils import epoch +from utils import reconstructhash """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -18,21 +22,19 @@ from __future__ import annotations You should have received a copy of the GNU General Public License along with this program. If not, see . """ -from typing import Union -from onionrutils import stringvalidators -from onionrutils import bytesconverter -from onionrutils import epoch -from utils import reconstructhash class UploadSession: - """Manages statistics for an Onionr block upload session - - accepting a block hash (incl. unpadded) as an argument""" + """Manage statistics for an Onionr block upload session. + + accept a block hash (incl. unpadded) as an argument + """ + def __init__(self, block_hash: Union[str, bytes]): block_hash = bytesconverter.bytes_to_str(block_hash) block_hash = reconstructhash.reconstruct_hash(block_hash) - if not stringvalidators.validate_hash(block_hash): raise ValueError + if not stringvalidators.validate_hash(block_hash): + raise ValueError self.start_time = epoch.get_epoch() self.block_hash = reconstructhash.deconstruct_hash(block_hash) @@ -40,15 +42,15 @@ class UploadSession: self.total_success_count: int = 0 self.peer_fails = {} self.peer_exists = {} - + def fail_peer(self, peer): try: self.peer_fails[peer] += 1 except KeyError: self.peer_fails[peer] = 0 - + def fail(self): self.total_fail_count += 1 - + def success(self): self.total_success_count += 1 diff --git a/src/etc/onionrvalues.py b/src/etc/onionrvalues.py index b5cc887d..c4c30b65 100755 --- a/src/etc/onionrvalues.py +++ b/src/etc/onionrvalues.py @@ -52,6 +52,9 @@ DEFAULT_EXPIRE = 2592000 # Metadata header section length limits, in bytes BLOCK_METADATA_LENGTHS = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'pow': 1000, 'encryptType': 4, 'expire': 14} +# Pool Eligibility Max Age +BLOCK_POOL_MAX_AGE = 300 + """Public key that signs MOTD messages shown in the web UI""" MOTD_SIGN_KEY = "TRH763JURNY47QPBTTQ4LLPYCYQK6Q5YA33R6GANKZK5C5DKCIGQ" diff --git a/src/httpapi/miscclientapi/endpoints.py b/src/httpapi/miscclientapi/endpoints.py index 46459369..e92bfe05 100644 --- a/src/httpapi/miscclientapi/endpoints.py +++ b/src/httpapi/miscclientapi/endpoints.py @@ -57,7 +57,7 @@ class PrivateEndpoints: # Responses from the daemon. TODO: change to direct var access instead of http endpoint client_api.queueResponse[name] = request.form['data'] return Response('success') - + @private_endpoints_bp.route('/queueResponse/') def queueResponse(name): # Fetch a daemon queue response @@ -72,7 +72,7 @@ class PrivateEndpoints: return resp, 404 else: return resp - + @private_endpoints_bp.route('/ping') def ping(): # Used to check if client api is working @@ -102,24 +102,24 @@ class PrivateEndpoints: def restart_clean(): subprocess.Popen([SCRIPT_NAME, 'restart']) return Response("bye") - + @private_endpoints_bp.route('/gethidden') def get_hidden_blocks(): return Response('\n'.join(client_api.publicAPI.hideBlocks)) - + @private_endpoints_bp.route('/getstats') def getStats(): # returns node stats while True: - try: + try: return Response(client_api._too_many.get(SerializedData).get_stats()) except AttributeError as e: pass - + @private_endpoints_bp.route('/getuptime') def showUptime(): return Response(str(client_api.getUptime())) - + @private_endpoints_bp.route('/getActivePubkey') def getActivePubkey(): return Response(pub_key) @@ -132,7 +132,7 @@ class PrivateEndpoints: def getHumanReadable(name): name = unpaddedbase32.repad(bytesconverter.str_to_bytes(name)) return Response(mnemonickeys.get_human_readable_ID(name)) - + @private_endpoints_bp.route('/getBase32FromHumanReadable/') def get_base32_from_human_readable(words): return Response(bytesconverter.bytes_to_str(mnemonickeys.get_base32(words))) @@ -144,3 +144,4 @@ class PrivateEndpoints: @private_endpoints_bp.route('/setonboarding', methods=['POST']) def set_onboarding(): return Response(config.onboarding.set_config_from_onboarding(request.get_json())) + diff --git a/src/httpapi/miscpublicapi/upload.py b/src/httpapi/miscpublicapi/upload.py index 9977f7bb..e2c3fe47 100755 --- a/src/httpapi/miscpublicapi/upload.py +++ b/src/httpapi/miscpublicapi/upload.py @@ -3,10 +3,14 @@ Accept block uploads to the public API server ''' +from gevent import threading + import sys from flask import Response from flask import abort +from onionrutils import localcommand +from coredb import daemonqueue from onionrblocks import blockimporter import onionrexceptions import logger @@ -31,9 +35,12 @@ def accept_upload(request): """Accept uploaded blocks to our public Onionr protocol API server""" resp = 'failure' data = request.get_data() + b_hash = '' if sys.getsizeof(data) < 100000000: try: - if blockimporter.import_block_from_data(data): + b_hash = blockimporter.import_block_from_data(data) + if b_hash: + daemonqueue.daemon_queue_add('uploadEvent', b_hash) resp = 'success' else: resp = 'failure' diff --git a/src/onionrblocks/insert.py b/src/onionrblocks/insert.py index 791bdc51..29300b03 100644 --- a/src/onionrblocks/insert.py +++ b/src/onionrblocks/insert.py @@ -82,7 +82,7 @@ def insert_block(data: Union[str, bytes], header: str = 'txt', our_pub_key = bytesconverter.bytes_to_str(crypto.cryptoutils.get_pub_key_from_priv(our_private_key)) use_subprocess = powchoice.use_subprocess(config) - + retData = False if type(data) is None: @@ -195,20 +195,15 @@ def insert_block(data: Union[str, bytes], header: str = 'txt', retData = False else: # Tell the api server through localCommand to wait for the daemon to upload this block to make statistical analysis more difficult - if not is_offline or localcommand.local_command('/ping', maxWait=10) == 'pong!': - if config.get('general.security_level', 1) == 0: - localcommand.local_command('/waitforshare/' + retData, post=True, maxWait=5) - coredb.daemonqueue.daemon_queue_add('uploadBlock', retData) - else: - pass + coredb.daemonqueue.daemon_queue_add('uploadEvent', retData) coredb.blockmetadb.add.add_to_block_DB(retData, selfInsert=True, dataSaved=True) if expire is None: - coredb.blockmetadb.update_block_info(retData, 'expire', + coredb.blockmetadb.update_block_info(retData, 'expire', createTime + onionrvalues.DEFAULT_EXPIRE) else: coredb.blockmetadb.update_block_info(retData, 'expire', expire) - + blockmetadata.process_block_metadata(retData) if retData != False: diff --git a/src/runtests/__init__.py b/src/runtests/__init__.py index 5e66d647..51b29eb6 100644 --- a/src/runtests/__init__.py +++ b/src/runtests/__init__.py @@ -7,6 +7,7 @@ import logger from onionrutils import epoch from . import uicheck, inserttest, stresstest +from . import ownnode """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -24,7 +25,10 @@ from . import uicheck, inserttest, stresstest RUN_TESTS = [uicheck.check_ui, inserttest.insert_bin_test, - stresstest.stress_test_block_insert] + ownnode.test_tor_adder, + ownnode.test_own_node, + stresstest.stress_test_block_insert + ] class OnionrRunTestManager: @@ -35,10 +39,11 @@ class OnionrRunTestManager: def run_tests(self): cur_time = epoch.get_epoch() logger.info(f"Doing runtime tests at {cur_time}") + try: for i in RUN_TESTS: last = i i(self) - logger.info(last.__name__ + " passed") - except ValueError: + logger.info("[RUNTIME TEST] " + last.__name__ + " passed") + except (ValueError, AttributeError): logger.error(last.__name__ + ' failed') diff --git a/src/runtests/ownnode.py b/src/runtests/ownnode.py new file mode 100644 index 00000000..ec78b51e --- /dev/null +++ b/src/runtests/ownnode.py @@ -0,0 +1,53 @@ +"""Onionr - Private P2P Communication. + +Test own Onionr node as it is running +""" +import config +from onionrutils import basicrequests +from utils import identifyhome +from utils import gettransports +import logger +from onionrutils import localcommand +""" + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + + +def test_own_node(test_manager): + socks_port = localcommand.local_command('/gettorsocks') + if config.get('general.security_level', 0) > 0: + return + own_tor_address = gettransports.get()[0] + print(socks_port) + if 'this is an onionr node' \ + not in basicrequests.do_get_request(own_tor_address, + port=socks_port).lower(): + logger.warn('Own node not reachable in test') + raise ValueError + + +def test_tor_adder(test_manager): + if config.get('general.security_level', 0) > 0: + return + with open(identifyhome.identify_home() + 'hs/hostname', 'r') as hs: + hs = hs.read().strip() + if not hs: + logger.error('No Tor node address created yet') + raise ValueError('No Tor node address created yet') + + if hs not in gettransports.get(): + print(hs in gettransports.get(), 'meme') + logger.error('gettransports Tor not same as file: %s %s' % + (hs, gettransports.get())) + raise ValueError('gettransports Tor not same as file') diff --git a/src/runtests/stresstest.py b/src/runtests/stresstest.py index a145fd76..35a9967c 100644 --- a/src/runtests/stresstest.py +++ b/src/runtests/stresstest.py @@ -6,6 +6,7 @@ import coredb from onionrutils import epoch def stress_test_block_insert(testmanager): + return start = epoch.get_epoch() count = 100 max_insert_speed = 120 diff --git a/src/utils/gettransports.py b/src/utils/gettransports.py index f5fd823e..5da882fd 100644 --- a/src/utils/gettransports.py +++ b/src/utils/gettransports.py @@ -1,17 +1,20 @@ -import filepaths, time +from gevent import time + +import filepaths files = [filepaths.tor_hs_address_file] + def get(): - transports = [] - for file in files: - try: - with open(file, 'r') as transport_file: - transports.append(transport_file.read().strip()) - except FileNotFoundError: - pass - else: - break + transports = [] + for file in files: + try: + with open(file, 'r') as transport_file: + transports.append(transport_file.read().strip()) + except FileNotFoundError: + pass else: - time.sleep(1) - return list(transports) \ No newline at end of file + break + else: + time.sleep(1) + return list(transports)