From 57791c34a5f0cddf72c8539e9406a2f2011467bc Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 8 Sep 2019 19:21:36 -0500 Subject: [PATCH] fixed broken random IP binding and Improved Onionr code safety --- .gitignore | 3 ++- README.md | 2 +- onionr/__init__.py | 1 + onionr/config.py | 2 +- onionr/coredb/blockmetadb/expiredblocks.py | 5 ++-- onionr/coredb/blockmetadb/updateblockinfo.py | 6 +++-- onionr/coredb/daemonqueue/__init__.py | 9 +++---- onionr/coredb/keydb/transportinfo.py | 2 +- onionr/coredb/keydb/userinfo.py | 3 +-- onionr/httpapi/apiutils/getblockdata.py | 3 ++- onionr/httpapi/apiutils/setbindip.py | 25 ++++++++++++++++--- onionr/httpapi/miscclientapi/endpoints.py | 2 +- onionr/httpapi/onionrsitesapi/__init__.py | 3 ++- onionr/onionrblockapi.py | 6 ++--- onionr/onionrblocks/insert.py | 2 +- onionr/onionrcommands/exportblocks.py | 4 +-- onionr/onionrcommands/pubkeymanager.py | 4 +-- onionr/onionrproofs.py | 3 +-- onionr/onionrservices/__init__.py | 2 +- onionr/onionrservices/bootstrapservice.py | 4 +-- onionr/onionrstorage/__init__.py | 6 ++--- onionr/onionrutils/blockmetadata/process.py | 5 ++-- onionr/onionrutils/stringvalidators.py | 4 +-- onionr/onionrutils/validatemetadata.py | 8 +++--- .../default-plugins/encrypt/main.py | 4 +-- 25 files changed, 70 insertions(+), 48 deletions(-) diff --git a/.gitignore b/.gitignore index 672aadb2..cc6624db 100755 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ core .vscode/* venv/* onionr/fs* +onionr/tmp/* *.dll *.exe @@ -35,4 +36,4 @@ onionr/data/*.log onionr-*.pkg.tar.gz pkg/ src/ -spawnnodes.py \ No newline at end of file +spawnnodes.py diff --git a/README.md b/README.md index d0e9116e..014df08f 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Everyone is welcome to contribute. Help is wanted for the following: * Creation of a shared lib for use from other languages and faster proof-of-work * Android and IOS development * Windows and Mac support (already partially supported, testers needed) - * General bug fixes and development of new features + * Bug fixes and development of new features * Testing * Translations/localizations * UI/UX design diff --git a/onionr/__init__.py b/onionr/__init__.py index c7439429..610e656f 100755 --- a/onionr/__init__.py +++ b/onionr/__init__.py @@ -20,6 +20,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' + # Set the user's locale for encoding reasons import locale locale.setlocale(locale.LC_ALL, '') diff --git a/onionr/config.py b/onionr/config.py index 89298e40..63086a2c 100755 --- a/onionr/config.py +++ b/onionr/config.py @@ -118,7 +118,7 @@ def reload(): try: with open(get_config_file(), 'r', encoding="utf8") as configfile: set_config(json.loads(configfile.read())) - except: + except (FileNotFoundError, json.JSONDecodeError) as e: pass #logger.debug('Failed to parse configuration file.') diff --git a/onionr/coredb/blockmetadb/expiredblocks.py b/onionr/coredb/blockmetadb/expiredblocks.py index 859abc9d..ac66be66 100644 --- a/onionr/coredb/blockmetadb/expiredblocks.py +++ b/onionr/coredb/blockmetadb/expiredblocks.py @@ -26,10 +26,11 @@ def get_expired_blocks(): c = conn.cursor() date = int(epoch.get_epoch()) - execute = 'SELECT hash FROM hashes WHERE expire <= %s ORDER BY dateReceived;' % (date,) + compiled = (date,) + execute = 'SELECT hash FROM hashes WHERE expire <= ? ORDER BY dateReceived;' rows = list() - for row in c.execute(execute): + for row in c.execute(execute, compiled): for i in row: rows.append(i) conn.close() diff --git a/onionr/coredb/blockmetadb/updateblockinfo.py b/onionr/coredb/blockmetadb/updateblockinfo.py index e32c37c8..d79ebece 100644 --- a/onionr/coredb/blockmetadb/updateblockinfo.py +++ b/onionr/coredb/blockmetadb/updateblockinfo.py @@ -34,12 +34,14 @@ def update_block_info(hash, key, data): dateClaimed - timestamp claimed inside the block, only as trustworthy as the block author is expire - expire date for a block ''' - if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'): - return False + if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', + 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'): + raise ValueError('Key must be in the allowed list') conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30) c = conn.cursor() args = (data, hash) + # Unfortunately, not really possible c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args) conn.commit() conn.close() diff --git a/onionr/coredb/daemonqueue/__init__.py b/onionr/coredb/daemonqueue/__init__.py index c0480c01..1ad2e6a2 100644 --- a/onionr/coredb/daemonqueue/__init__.py +++ b/onionr/coredb/daemonqueue/__init__.py @@ -86,10 +86,7 @@ def clear_daemon_queue(): conn = sqlite3.connect(dbfiles.daemon_queue_db, timeout=30) c = conn.cursor() - try: - c.execute('DELETE FROM commands;') - conn.commit() - except: - pass + c.execute('DELETE FROM commands;') + conn.commit() - conn.close() \ No newline at end of file + conn.close() diff --git a/onionr/coredb/keydb/transportinfo.py b/onionr/coredb/keydb/transportinfo.py index cd6739dc..442d0c13 100644 --- a/onionr/coredb/keydb/transportinfo.py +++ b/onionr/coredb/keydb/transportinfo.py @@ -66,7 +66,7 @@ def set_address_info(address, key, data): command = (data, address) if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'failure', 'powValue', 'lastConnect', 'lastConnectAttempt', 'trust', 'introduced'): - raise Exception("Got invalid database key when setting address info") + raise ValueError("Got invalid database key when setting address info, must be in whitelist") else: c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command) conn.commit() diff --git a/onionr/coredb/keydb/userinfo.py b/onionr/coredb/keydb/userinfo.py index 23ee8b7c..c395f493 100644 --- a/onionr/coredb/keydb/userinfo.py +++ b/onionr/coredb/keydb/userinfo.py @@ -61,9 +61,8 @@ def set_peer_info(peer, key, data): command = (data, peer) - # TODO: validate key on whitelist if key not in ('id', 'name', 'pubkey', 'forwardKey', 'dateSeen', 'trust'): - raise Exception("Got invalid database key when setting peer info") + raise ValueError("Got invalid database key when setting peer info") c.execute('UPDATE peers SET ' + key + ' = ? WHERE id=?', command) conn.commit() diff --git a/onionr/httpapi/apiutils/getblockdata.py b/onionr/httpapi/apiutils/getblockdata.py index 1a8f7df0..7de04a29 100644 --- a/onionr/httpapi/apiutils/getblockdata.py +++ b/onionr/httpapi/apiutils/getblockdata.py @@ -1,12 +1,13 @@ import json import onionrblockapi from onionrutils import bytesconverter, stringvalidators +import onionrexceptions class GetBlockData: def __init__(self, client_api_inst=None): return def get_block_data(self, bHash, decrypt=False, raw=False, headerOnly=False): - assert stringvalidators.validate_hash(bHash) + if not stringvalidators.validate_hash(bHash): raise onionrexceptions.InvalidHexHash("block hash not valid hash format") bl = onionrblockapi.Block(bHash) if decrypt: bl.decrypt() diff --git a/onionr/httpapi/apiutils/setbindip.py b/onionr/httpapi/apiutils/setbindip.py index c9c716f5..0a1d4e18 100644 --- a/onionr/httpapi/apiutils/setbindip.py +++ b/onionr/httpapi/apiutils/setbindip.py @@ -1,11 +1,30 @@ -import random, socket +import gevent +from gevent import socket, sleep +import secrets, random import config, logger +import os + +# Hacky monkey patch so we can bind random localhosts without gevent trying to switch with an empty hub +socket.getfqdn = lambda n: n + +def _get_acceptable_random_number()->int: + """Return a cryptographically random number in the inclusive range (1, 255)""" + number = 0 + while number == 0: + number = secrets.randbelow(0xFF) + return number + def set_bind_IP(filePath=''): '''Set a random localhost IP to a specified file (intended for private or public API localhost IPs)''' - if config.get('general.random_bind_ip', True): - hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))] + hostOctets = [] + # Build the random localhost address + for i in range(3): + hostOctets.append(str(_get_acceptable_random_number())) + hostOctets = ['127'] + hostOctets + # Convert the localhost address to a normal string address data = '.'.join(hostOctets) + # Try to bind IP. Some platforms like Mac block non normal 127.x.x.x s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: diff --git a/onionr/httpapi/miscclientapi/endpoints.py b/onionr/httpapi/miscclientapi/endpoints.py index 553123dc..a87310a4 100644 --- a/onionr/httpapi/miscclientapi/endpoints.py +++ b/onionr/httpapi/miscclientapi/endpoints.py @@ -76,7 +76,7 @@ class PrivateEndpoints: @private_endpoints_bp.route('/waitforshare/', methods=['post']) def waitforshare(name): '''Used to prevent the **public** api from sharing blocks we just created''' - assert name.isalnum() + if not name.isalnum(): raise ValueError('block hash needs to be alpha numeric') if name in client_api.publicAPI.hideBlocks: client_api.publicAPI.hideBlocks.remove(name) return Response("removed") diff --git a/onionr/httpapi/onionrsitesapi/__init__.py b/onionr/httpapi/onionrsitesapi/__init__.py index f491cc1c..7c3b40c8 100644 --- a/onionr/httpapi/onionrsitesapi/__init__.py +++ b/onionr/httpapi/onionrsitesapi/__init__.py @@ -18,6 +18,7 @@ along with this program. If not, see . ''' import base64 +import binascii from flask import Blueprint, Response, request, abort import onionrblockapi, onionrexceptions from onionrutils import stringvalidators @@ -37,7 +38,7 @@ def site(name): pass try: resp = base64.b64decode(resp) - except: + except binascii.Error: pass if resp == 'Not Found' or not resp: abort(404) diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index 1cfd3b31..4e989099 100755 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -90,8 +90,8 @@ class Block: # Check for replay attacks try: if epoch.get_epoch() - blockmetadb.get_block_date(self.hash) > 60: - assert cryptoutils.replay_validator(self.bmetadata['rply']) - except (AssertionError, KeyError, TypeError) as e: + if not cryptoutils.replay_validator(self.bmetadata['rply']): raise onionrexceptions.ReplayAttack + except (AssertionError, KeyError, TypeError, onionrexceptions.ReplayAttack) as e: if not self.bypassReplayCheck: # Zero out variables to prevent reading of replays self.bmetadata = {} @@ -101,7 +101,7 @@ class Block: self.signature = '' raise onionrexceptions.ReplayAttack('Signature is too old. possible replay attack') try: - assert self.bmetadata['forwardEnc'] is True + if not self.bmetadata['forwardEnc']: raise KeyError except (AssertionError, KeyError) as e: pass else: diff --git a/onionr/onionrblocks/insert.py b/onionr/onionrblocks/insert.py index e2404735..bc9ab158 100644 --- a/onionr/onionrblocks/insert.py +++ b/onionr/onionrblocks/insert.py @@ -120,7 +120,7 @@ def insert_block(data: Union[str, bytes], header: str ='txt', # ensure expire is integer and of sane length if type(expire) is not type(None): - assert len(str(int(expire))) < 20 + if not len(str(int(expire))) < 20: raise ValueError('expire must be valid int less than 20 digits in length') metadata['expire'] = expire # send block data (and metadata) to POW module to get tokenized block data diff --git a/onionr/onionrcommands/exportblocks.py b/onionr/onionrcommands/exportblocks.py index 77e88439..c1e3b624 100755 --- a/onionr/onionrcommands/exportblocks.py +++ b/onionr/onionrcommands/exportblocks.py @@ -32,8 +32,8 @@ def doExport(bHash): def export_block(): exportDir = filepaths.export_location try: - assert stringvalidators.validate_hash(sys.argv[2]) - except (IndexError, AssertionError): + if not stringvalidators.validate_hash(sys.argv[2]): raise ValueError + except (IndexError, ValueError): logger.error('No valid block hash specified.', terminal=True) sys.exit(1) else: diff --git a/onionr/onionrcommands/pubkeymanager.py b/onionr/onionrcommands/pubkeymanager.py index 24452368..6e5e2b57 100755 --- a/onionr/onionrcommands/pubkeymanager.py +++ b/onionr/onionrcommands/pubkeymanager.py @@ -32,8 +32,8 @@ def add_ID(): key_manager = keymanager.KeyManager() try: sys.argv[2] - assert sys.argv[2] == 'true' - except (IndexError, AssertionError) as e: + if not sys.argv[2].lower() == 'true': raise ValueError + except (IndexError, ValueError) as e: newID = key_manager.addKey()[0] else: logger.warn('Deterministic keys require random and long passphrases.', terminal=True) diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py index f476981c..f02a15b2 100755 --- a/onionr/onionrproofs.py +++ b/onionr/onionrproofs.py @@ -60,12 +60,11 @@ def getDifficultyForNewBlock(data, ourBlock=True): return retData -def getHashDifficulty(h): +def getHashDifficulty(h: str): ''' Return the amount of leading zeroes in a hex hash string (h) ''' difficulty = 0 - assert type(h) is str for character in h: if character == '0': difficulty += 1 diff --git a/onionr/onionrservices/__init__.py b/onionr/onionrservices/__init__.py index 2aefbc07..a17cd1bf 100755 --- a/onionr/onionrservices/__init__.py +++ b/onionr/onionrservices/__init__.py @@ -37,7 +37,7 @@ class OnionrServices: When a client wants to connect, contact their bootstrap address and tell them our ephemeral address for our service by creating a new ConnectionServer instance ''' - assert stringvalidators.validate_transport(address) + if not stringvalidators.validate_transport(address): raise ValueError('address must be valid') BOOTSTRAP_TRIES = 10 # How many times to attempt contacting the bootstrap server TRY_WAIT = 3 # Seconds to wait before trying bootstrap again # HTTP is fine because .onion/i2p is encrypted/authenticated diff --git a/onionr/onionrservices/bootstrapservice.py b/onionr/onionrservices/bootstrapservice.py index 7e1322e9..a493290b 100755 --- a/onionr/onionrservices/bootstrapservice.py +++ b/onionr/onionrservices/bootstrapservice.py @@ -54,8 +54,8 @@ def bootstrap_client_service(peer, comm_inst=None, bootstrap_timeout=300): http_server = WSGIServer(('127.0.0.1', bootstrap_port), bootstrap_app, log=None) try: - assert comm_inst is not None - except (AttributeError, AssertionError) as e: + if comm_inst is None: raise ValueError + except (AttributeError, ValueError) as e: pass else: comm_inst.service_greenlets.append(http_server) diff --git a/onionr/onionrstorage/__init__.py b/onionr/onionrstorage/__init__.py index d6436800..7b722630 100755 --- a/onionr/onionrstorage/__init__.py +++ b/onionr/onionrstorage/__init__.py @@ -58,10 +58,10 @@ def deleteBlock(blockHash): return True def store(data, blockHash=''): - assert stringvalidators.validate_hash(blockHash) + if not stringvalidators.validate_hash(blockHash): raise ValueError ourHash = hashers.sha3_hash(data) if blockHash != '': - assert ourHash == blockHash + if not ourHash == blockHash: raise ValueError('Hash specified does not meet internal hash check') else: blockHash = ourHash @@ -72,7 +72,7 @@ def store(data, blockHash=''): blockFile.write(data) def getData(bHash): - assert stringvalidators.validate_hash(bHash) + if not stringvalidators.validate_hash(bHash): raise ValueError bHash = bytesconverter.bytes_to_str(bHash) diff --git a/onionr/onionrutils/blockmetadata/process.py b/onionr/onionrutils/blockmetadata/process.py index be595194..5164c794 100644 --- a/onionr/onionrutils/blockmetadata/process.py +++ b/onionr/onionrutils/blockmetadata/process.py @@ -58,8 +58,9 @@ def process_block_metadata(blockHash: str): # Set block expire time if specified try: expireTime = int(myBlock.getHeader('expire')) - assert len(str(expireTime)) < 20 # test that expire time is an integer of sane length (for epoch) - except (AssertionError, ValueError, TypeError) as e: + # test that expire time is an integer of sane length (for epoch) + if not len(str(expireTime)) < 20: raise ValueError('timestamp invalid') + except (ValueError, TypeError) as e: expireTime = onionrvalues.DEFAULT_EXPIRE + curTime finally: expireTime = min(expireTime, curTime + onionrvalues.DEFAULT_EXPIRE) diff --git a/onionr/onionrutils/stringvalidators.py b/onionr/onionrutils/stringvalidators.py index 951c25ed..922916e0 100644 --- a/onionr/onionrutils/stringvalidators.py +++ b/onionr/onionrutils/stringvalidators.py @@ -86,14 +86,14 @@ def validate_transport(id): if peerType == 'i2p': try: id.split('.b32.i2p')[2] - except: + except IndexError: pass else: retVal = False elif peerType == 'onion': try: id.split('.onion')[2] - except: + except IndexError: pass else: retVal = False diff --git a/onionr/onionrutils/validatemetadata.py b/onionr/onionrutils/validatemetadata.py index da45e716..ddb74455 100644 --- a/onionr/onionrutils/validatemetadata.py +++ b/onionr/onionrutils/validatemetadata.py @@ -66,14 +66,14 @@ def validate_metadata(metadata, block_data) -> bool: break elif i == 'expire': try: - assert int(metadata[i]) > epoch.get_epoch() - except AssertionError: + if not int(metadata[i]) > epoch.get_epoch(): raise ValueError + except ValueError: logger.warn('Block is expired: %s less than %s' % (metadata[i], epoch.get_epoch())) break elif i == 'encryptType': try: - assert metadata[i] in ('asym', 'sym', '') - except AssertionError: + if not metadata[i] in ('asym', 'sym', ''): raise ValueError + except ValueError: logger.warn('Invalid encryption mode') break elif i == 'sig': diff --git a/onionr/static-data/default-plugins/encrypt/main.py b/onionr/static-data/default-plugins/encrypt/main.py index 05ce88ea..cbd21a05 100755 --- a/onionr/static-data/default-plugins/encrypt/main.py +++ b/onionr/static-data/default-plugins/encrypt/main.py @@ -100,8 +100,8 @@ class PlainEncryption: logger.info('Decrypted Message: \n\n%s' % data['data'], terminal=True) try: logger.info("Signing public key: %s" % (data['signer'],), terminal=True) - assert signing.ed_verify(data['data'], data['signer'], data['sig']) != False - except (AssertionError, KeyError) as e: + if not signing.ed_verify(data['data'], data['signer'], data['sig']): raise ValueError + except (ValueError, KeyError) as e: logger.warn("WARNING: THIS MESSAGE HAS A MISSING OR INVALID SIGNATURE", terminal=True) else: logger.info("Message has good signature.", terminal=True)