From d3783400991f1e6199ce3ad2c547317005c8219b Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 23 Jun 2019 12:41:07 -0500 Subject: [PATCH] * further splitting onionrutils into a module --- onionr/communicator.py | 7 +- onionr/communicatorutils/connectnewpeers.py | 3 +- .../communicatorutils/daemonqueuehandler.py | 5 +- onionr/communicatorutils/lookupadders.py | 3 +- onionr/communicatorutils/netcheck.py | 6 +- onionr/communicatorutils/servicecreator.py | 4 +- onionr/communicatorutils/uploadblocks.py | 3 +- onionr/core.py | 7 +- onionr/coredb/daemonqueue/__init__.py | 4 +- onionr/coredb/keydb/addkeys.py | 8 +- onionr/coredb/keydb/removekeys.py | 4 +- onionr/httpapi/miscpublicapi/announce.py | 4 +- onionr/onionrcommands/__init__.py | 5 +- onionr/onionrcommands/daemonlaunch.py | 9 +- onionr/onionrcommands/onionrstatistics.py | 3 +- onionr/onionrcommands/openwebinterface.py | 3 +- onionr/onionrcommands/resettor.py | 4 +- onionr/onionrpluginapi.py | 3 +- onionr/onionrservices/__init__.py | 3 +- onionr/onionrservices/bootstrapservice.py | 3 +- onionr/onionrutils/__init__.py | 156 +----------------- onionr/onionrutils/checkcommunicator.py | 19 +++ onionr/onionrutils/getclientapiserver.py | 10 ++ onionr/onionrutils/importnewblocks.py | 28 ++++ onionr/onionrutils/localcommand.py | 5 +- onionr/onionrutils/stringvalidators.py | 97 +++++++++++ onionr/onionrutils/validatemetadata.py | 5 +- .../static-data/default-plugins/cliui/main.py | 3 +- .../default-plugins/esoteric/peerserver.py | 5 +- .../default-plugins/pluginmanager/main.py | 3 +- onionr/tests/test_stringvalidations.py | 11 +- 31 files changed, 234 insertions(+), 199 deletions(-) create mode 100644 onionr/onionrutils/checkcommunicator.py create mode 100644 onionr/onionrutils/getclientapiserver.py create mode 100644 onionr/onionrutils/importnewblocks.py create mode 100644 onionr/onionrutils/stringvalidators.py diff --git a/onionr/communicator.py b/onionr/communicator.py index d59b76da..57917b9a 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -27,6 +27,7 @@ from communicatorutils import downloadblocks, lookupblocks, lookupadders from communicatorutils import servicecreator, connectnewpeers, uploadblocks from communicatorutils import daemonqueuehandler, announcenode, deniableinserts from communicatorutils import cooldownpeer, housekeeping, netcheck +from onionrutils import localcommand from etc import humanreadabletime import onionrservices, onionr, onionrproofs @@ -184,7 +185,7 @@ class OnionrCommunicatorDaemon: else: for server in self.service_greenlets: server.stop() - self._core._utils.localCommand('shutdown') # shutdown the api + localcommand.local_command(self._core, 'shutdown') # shutdown the api time.sleep(0.5) def lookupAdders(self): @@ -364,9 +365,9 @@ class OnionrCommunicatorDaemon: def detectAPICrash(self): '''exit if the api server crashes/stops''' - if self._core._utils.localCommand('ping', silent=False) not in ('pong', 'pong!'): + if localcommand.local_command(self._core, 'ping', silent=False) not in ('pong', 'pong!'): for i in range(300): - if self._core._utils.localCommand('ping') in ('pong', 'pong!') or self.shutdown: + if localcommand.local_command(self._core, 'ping') in ('pong', 'pong!') or self.shutdown: break # break for loop time.sleep(1) else: diff --git a/onionr/communicatorutils/connectnewpeers.py b/onionr/communicatorutils/connectnewpeers.py index 96e4ecac..25929c24 100755 --- a/onionr/communicatorutils/connectnewpeers.py +++ b/onionr/communicatorutils/connectnewpeers.py @@ -20,6 +20,7 @@ import time, sys import onionrexceptions, logger, onionrpeers from utils import networkmerger +from onionrutils import stringvalidators # secrets module was added into standard lib in 3.6+ if sys.version_info[0] == 3 and sys.version_info[1] < 6: from dependencies import secrets @@ -30,7 +31,7 @@ def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False): retData = False tried = comm_inst.offlinePeers if peer != '': - if comm_inst._core._utils.validateID(peer): + if stringvalidators.validate_transport(peer): peerList = [peer] else: raise onionrexceptions.InvalidAddress('Will not attempt connection test to invalid address') diff --git a/onionr/communicatorutils/daemonqueuehandler.py b/onionr/communicatorutils/daemonqueuehandler.py index ddbb0783..c11cc929 100755 --- a/onionr/communicatorutils/daemonqueuehandler.py +++ b/onionr/communicatorutils/daemonqueuehandler.py @@ -19,6 +19,7 @@ ''' import logger import onionrevents as events +from onionrutils import localcommand def handle_daemon_commands(comm_inst): cmd = comm_inst._core.daemonQueue() response = '' @@ -39,7 +40,7 @@ def handle_daemon_commands(comm_inst): if response == '': response = 'none' elif cmd[0] == 'localCommand': - response = comm_inst._core._utils.localCommand(cmd[1]) + response = localcommand.local_command(comm_inst._core, cmd[1]) elif cmd[0] == 'pex': for i in comm_inst.timers: if i.timerFunction.__name__ == 'lookupAdders': @@ -49,7 +50,7 @@ def handle_daemon_commands(comm_inst): if cmd[0] not in ('', None): if response != '': - comm_inst._core._utils.localCommand('queueResponseAdd/' + cmd[4], post=True, postData={'data': response}) + localcommand.local_command(comm_inst._core, 'queueResponseAdd/' + cmd[4], post=True, postData={'data': response}) response = '' comm_inst.decrementThreadCount('daemonCommands') \ No newline at end of file diff --git a/onionr/communicatorutils/lookupadders.py b/onionr/communicatorutils/lookupadders.py index 4f861682..fc4527dc 100755 --- a/onionr/communicatorutils/lookupadders.py +++ b/onionr/communicatorutils/lookupadders.py @@ -18,6 +18,7 @@ along with this program. If not, see . ''' import logger +from onionrutils import stringvalidators def lookup_new_peer_transports_with_communicator(comm_inst): logger.info('Looking up new addresses...') @@ -39,7 +40,7 @@ def lookup_new_peer_transports_with_communicator(comm_inst): invalid = [] for x in newPeers: x = x.strip() - if not comm_inst._core._utils.validateID(x) or x in comm_inst.newPeers or x == comm_inst._core.hsAddress: + if not stringvalidators.validate_transport(x) or x in comm_inst.newPeers or x == comm_inst._core.hsAddress: # avoid adding if its our address invalid.append(x) for x in invalid: diff --git a/onionr/communicatorutils/netcheck.py b/onionr/communicatorutils/netcheck.py index c5e95906..688feeea 100755 --- a/onionr/communicatorutils/netcheck.py +++ b/onionr/communicatorutils/netcheck.py @@ -20,17 +20,19 @@ ''' import logger from utils import netutils +from onionrutils import localcommand def net_check(comm_inst): '''Check if we are connected to the internet or not when we can't connect to any peers''' rec = False # for detecting if we have received incoming connections recently + c = comm_inst._core if len(comm_inst.onlinePeers) == 0: try: - if (comm_inst._core._utils.getEpoch() - int(comm_inst._core._utils.localCommand('/lastconnect'))) <= 60: + if (c._utils.getEpoch() - int(localcommand.local_command(c, '/lastconnect'))) <= 60: comm_inst.isOnline = True rec = True except ValueError: pass - if not rec and not netutils.checkNetwork(comm_inst._core._utils, torPort=comm_inst.proxyPort): + if not rec and not netutils.checkNetwork(c._utils, torPort=comm_inst.proxyPort): if not comm_inst.shutdown: logger.warn('Network check failed, are you connected to the Internet, and is Tor working?') comm_inst.isOnline = False diff --git a/onionr/communicatorutils/servicecreator.py b/onionr/communicatorutils/servicecreator.py index c1f2c7e2..94fc51c8 100755 --- a/onionr/communicatorutils/servicecreator.py +++ b/onionr/communicatorutils/servicecreator.py @@ -18,6 +18,8 @@ along with this program. If not, see . ''' import communicator, onionrblockapi +from onionrutils import stringvalidators + def service_creator(daemon): assert isinstance(daemon, communicator.OnionrCommunicatorDaemon) core = daemon._core @@ -30,7 +32,7 @@ def service_creator(daemon): if not b in daemon.active_services: bl = onionrblockapi.Block(b, core=core, decrypt=True) bs = utils.bytesToStr(bl.bcontent) + '.onion' - if utils.validatePubKey(bl.signer) and utils.validateID(bs): + if utils.validatePubKey(bl.signer) and stringvalidators.validate_transport(bs): signer = utils.bytesToStr(bl.signer) daemon.active_services.append(b) daemon.active_services.append(signer) diff --git a/onionr/communicatorutils/uploadblocks.py b/onionr/communicatorutils/uploadblocks.py index 69d0353d..497e07d5 100755 --- a/onionr/communicatorutils/uploadblocks.py +++ b/onionr/communicatorutils/uploadblocks.py @@ -20,6 +20,7 @@ import logger from communicatorutils import proxypicker import onionrblockapi as block +from onionrutils import localcommand def upload_blocks_from_communicator(comm_inst): # when inserting a block, we try to upload it to a few peers to add some deniability @@ -42,7 +43,7 @@ def upload_blocks_from_communicator(comm_inst): proxyType = proxypicker.pick_proxy(peer) logger.info("Uploading block to " + peer) if not comm_inst._core._utils.doPostRequest(url, data=data, proxyType=proxyType) == False: - comm_inst._core._utils.localCommand('waitforshare/' + bl, post=True) + localcommand.local_command(comm_inst._core, 'waitforshare/' + bl, post=True) finishedUploads.append(bl) for x in finishedUploads: try: diff --git a/onionr/core.py b/onionr/core.py index ebc35e51..53bf0b4c 100755 --- a/onionr/core.py +++ b/onionr/core.py @@ -28,6 +28,7 @@ from onionrusers import onionrusers from onionrstorage import removeblock, setdata import dbcreator, onionrstorage, serializeddata, subprocesspow from etc import onionrvalues, powchoice +from onionrutils import localcommand class Core: def __init__(self, torPort=0): @@ -433,8 +434,8 @@ class Core: retData = False else: # Tell the api server through localCommand to wait for the daemon to upload this block to make statistical analysis more difficult - if self._utils.localCommand('/ping', maxWait=10) == 'pong!': - self._utils.localCommand('/waitforshare/' + retData, post=True, maxWait=5) + if localcommand.local_command(self, '/ping', maxWait=10) == 'pong!': + localcommand.local_command(self, '/waitforshare/' + retData, post=True, maxWait=5) self.daemonQueueAdd('uploadBlock', retData) self.addToBlockDB(retData, selfInsert=True, dataSaved=True) self._utils.processBlockMetadata(retData) @@ -450,7 +451,7 @@ class Core: ''' Introduces our node into the network by telling X many nodes our HS address ''' - if self._utils.localCommand('/ping', maxWait=10) == 'pong!': + if localcommand.local_command(self, '/ping', maxWait=10) == 'pong!': self.daemonQueueAdd('announceNode') logger.info('Introduction command will be processed.', terminal=True) else: diff --git a/onionr/coredb/daemonqueue/__init__.py b/onionr/coredb/daemonqueue/__init__.py index 8bd21cfd..a2f7a483 100644 --- a/onionr/coredb/daemonqueue/__init__.py +++ b/onionr/coredb/daemonqueue/__init__.py @@ -1,5 +1,7 @@ import sqlite3, os import onionrevents as events +from onionrutils import localcommand + def daemon_queue(core_inst): ''' Gives commands to the communication proccess/daemon by reading an sqlite3 database @@ -55,7 +57,7 @@ def daemon_queue_get_response(core_inst, responseID=''): Get a response sent by communicator to the API, by requesting to the API ''' assert len(responseID) > 0 - resp = core_inst._utils.localCommand('queueResponse/' + responseID) + resp = localcommand.local_command(core_inst, 'queueResponse/' + responseID) return resp def clear_daemon_queue(core_inst): diff --git a/onionr/coredb/keydb/addkeys.py b/onionr/coredb/keydb/addkeys.py index ac219b1a..716581cd 100644 --- a/onionr/coredb/keydb/addkeys.py +++ b/onionr/coredb/keydb/addkeys.py @@ -1,5 +1,7 @@ import sqlite3 -import onionrevents as events, config +import onionrevents as events +from onionrutils import stringvalidators + def add_peer(core_inst, peerID, name=''): ''' Adds a public key to the key database (misleading function name) @@ -40,8 +42,8 @@ def add_address(core_inst, address): if type(address) is None or len(address) == 0: return False - if core_inst._utils.validateID(address): - if address == config.get('i2p.ownAddr', None) or address == core_inst.hsAddress: + if stringvalidators.validate_transport(address): + if address == core_inst.config.get('i2p.ownAddr', None) or address == core_inst.hsAddress: return False conn = sqlite3.connect(core_inst.addressDB, timeout=30) c = conn.cursor() diff --git a/onionr/coredb/keydb/removekeys.py b/onionr/coredb/keydb/removekeys.py index 10f44a1b..ce4ba5fc 100644 --- a/onionr/coredb/keydb/removekeys.py +++ b/onionr/coredb/keydb/removekeys.py @@ -1,11 +1,13 @@ import sqlite3 import onionrevents as events +from onionrutils import stringvalidators + def remove_address(core_inst, address): ''' Remove an address from the address database ''' - if core_inst._utils.validateID(address): + if stringvalidators.validate_transport(address): conn = sqlite3.connect(core_inst.addressDB, timeout=30) c = conn.cursor() t = (address,) diff --git a/onionr/httpapi/miscpublicapi/announce.py b/onionr/httpapi/miscpublicapi/announce.py index 8a25b635..f17c8d83 100755 --- a/onionr/httpapi/miscpublicapi/announce.py +++ b/onionr/httpapi/miscpublicapi/announce.py @@ -21,6 +21,8 @@ import base64 from flask import Response import logger from etc import onionrvalues +from onionrutils import stringvalidators + def handle_announce(clientAPI, request): ''' accept announcement posts, validating POW @@ -52,7 +54,7 @@ def handle_announce(clientAPI, request): pass if powHash.startswith('0' * onionrvalues.OnionrValues().announce_pow): newNode = clientAPI._core._utils.bytesToStr(newNode) - if clientAPI._core._utils.validateID(newNode) and not newNode in clientAPI._core.onionrInst.communicatorInst.newPeers: + if stringvalidators.validate_transport(newNode) and not newNode in clientAPI._core.onionrInst.communicatorInst.newPeers: clientAPI._core.onionrInst.communicatorInst.newPeers.append(newNode) resp = 'Success' else: diff --git a/onionr/onionrcommands/__init__.py b/onionr/onionrcommands/__init__.py index 3a50d59d..59c9d6bc 100755 --- a/onionr/onionrcommands/__init__.py +++ b/onionr/onionrcommands/__init__.py @@ -22,6 +22,7 @@ import webbrowser, sys import logger from . import pubkeymanager, onionrstatistics, daemonlaunch, filecommands, plugincommands, keyadders from . import banblocks, exportblocks, openwebinterface, resettor +from onionrutils import importnewblocks def show_help(o_inst, command): @@ -110,8 +111,8 @@ def get_commands(onionr_inst): 'listconn': onionr_inst.listConn, 'list-conn': onionr_inst.listConn, - 'import-blocks': onionr_inst.onionrUtils.importNewBlocks, - 'importblocks': onionr_inst.onionrUtils.importNewBlocks, + 'import-blocks': importnewblocks.import_new_blocks, + 'importblocks': importnewblocks.import_new_blocks, 'introduce': onionr_inst.onionrCore.introduceNode, 'pex': onionr_inst.doPEX, diff --git a/onionr/onionrcommands/daemonlaunch.py b/onionr/onionrcommands/daemonlaunch.py index 1aaf7230..28202c10 100755 --- a/onionr/onionrcommands/daemonlaunch.py +++ b/onionr/onionrcommands/daemonlaunch.py @@ -23,9 +23,10 @@ from threading import Thread import onionr, api, logger, communicator import onionrevents as events from netcontroller import NetController +from onionrutils import localcommand def _proper_shutdown(o_inst): - o_inst.onionrUtils.localCommand('shutdown') + localcommand.local_command(o_inst.onionrCore, 'shutdown') sys.exit(1) def daemon(o_inst): @@ -63,7 +64,7 @@ def daemon(o_inst): net = NetController(o_inst.onionrCore.config.get('client.public.port', 59497), apiServerIP=apiHost) logger.info('Tor is starting...', terminal=True) if not net.startTor(): - o_inst.onionrUtils.localCommand('shutdown') + localcommand.local_command(o_inst.onionrCore, 'shutdown') sys.exit(1) if len(net.myID) > 0 and o_inst.onionrCore.config.get('general.security_level', 1) == 0: logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID)) @@ -103,10 +104,10 @@ def daemon(o_inst): signal.signal(signal.SIGINT, _ignore_sigint) o_inst.onionrCore.daemonQueueAdd('shutdown') - o_inst.onionrUtils.localCommand('shutdown') + localcommand.local_command(o_inst.onionrCore, 'shutdown') net.killTor() - time.sleep(8) # Time to allow threads to finish, if not any "daemon" threads will be slaughtered http://docs.python.org/library/threading.html#threading.Thread.daemon + time.sleep(5) # Time to allow threads to finish, if not any "daemon" threads will be slaughtered http://docs.python.org/library/threading.html#threading.Thread.daemon o_inst.deleteRunFiles() return diff --git a/onionr/onionrcommands/onionrstatistics.py b/onionr/onionrcommands/onionrstatistics.py index a28fb7db..2f50c6e7 100755 --- a/onionr/onionrcommands/onionrstatistics.py +++ b/onionr/onionrcommands/onionrstatistics.py @@ -21,6 +21,7 @@ import os, uuid, time import logger, onionrutils from onionrblockapi import Block import onionr +from onionrutils import checkcommunicator def show_stats(o_inst): try: @@ -29,7 +30,7 @@ def show_stats(o_inst): signedBlocks = len(Block.getBlocks(signed = True)) messages = { # info about local client - 'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if o_inst.onionrUtils.isCommunicatorRunning(timeout = 9) else logger.colors.fg.red + 'Offline'), + 'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if checkcommunicator.is_communicator_running(o_inst.onionrCore, timeout = 9) else logger.colors.fg.red + 'Offline'), # file and folder size stats 'div1' : True, # this creates a solid line across the screen, a div diff --git a/onionr/onionrcommands/openwebinterface.py b/onionr/onionrcommands/openwebinterface.py index 8ce71744..42c80b57 100755 --- a/onionr/onionrcommands/openwebinterface.py +++ b/onionr/onionrcommands/openwebinterface.py @@ -19,9 +19,10 @@ ''' import webbrowser import logger +from onionrutils import getclientapiserver def open_home(o_inst): try: - url = o_inst.onionrUtils.getClientAPIServer() + url = getclientapiserver.get_client_API_server(o_inst.onionrCore) except FileNotFoundError: logger.error('Onionr seems to not be running (could not get api host)', terminal=True) else: diff --git a/onionr/onionrcommands/resettor.py b/onionr/onionrcommands/resettor.py index cd7c51d2..e327bccc 100755 --- a/onionr/onionrcommands/resettor.py +++ b/onionr/onionrcommands/resettor.py @@ -19,11 +19,13 @@ ''' import os, shutil import logger, core +from onionrutils import localcommand + def reset_tor(): c = core.Core() tor_dir = c.dataDir + 'tordata' if os.path.exists(tor_dir): - if c._utils.localCommand('/ping') == 'pong!': + if localcommand.local_command(c, '/ping') == 'pong!': logger.warn('Cannot delete Tor data while Onionr is running', terminal=True) else: shutil.rmtree(tor_dir) \ No newline at end of file diff --git a/onionr/onionrpluginapi.py b/onionr/onionrpluginapi.py index 78c8a008..24ba4f12 100755 --- a/onionr/onionrpluginapi.py +++ b/onionr/onionrpluginapi.py @@ -19,6 +19,7 @@ ''' import onionrplugins, core as onionrcore, logger +from onionrutils import localcommand class DaemonAPI: def __init__(self, pluginapi): @@ -40,7 +41,7 @@ class DaemonAPI: return def local_command(self, command): - return self.pluginapi.get_utils().localCommand(self, command) + return localcommand.local_command(self.pluginapi.get_core(), command) def queue_pop(self): return self.get_core().daemonQueue() diff --git a/onionr/onionrservices/__init__.py b/onionr/onionrservices/__init__.py index 2792f7e3..1c23e5fb 100755 --- a/onionr/onionrservices/__init__.py +++ b/onionr/onionrservices/__init__.py @@ -21,6 +21,7 @@ import time import stem import core from . import connectionserver, bootstrapservice +from onionrutils import stringvalidators class OnionrServices: ''' @@ -39,7 +40,7 @@ class OnionrServices: When a client wants to connect, contact their bootstrap address and tell them our ephemeral address for our service by creating a new ConnectionServer instance ''' - assert self._core._utils.validateID(address) + assert stringvalidators.validate_transport(address) BOOTSTRAP_TRIES = 10 # How many times to attempt contacting the bootstrap server TRY_WAIT = 3 # Seconds to wait before trying bootstrap again # HTTP is fine because .onion/i2p is encrypted/authenticated diff --git a/onionr/onionrservices/bootstrapservice.py b/onionr/onionrservices/bootstrapservice.py index 077b4657..e06300e6 100755 --- a/onionr/onionrservices/bootstrapservice.py +++ b/onionr/onionrservices/bootstrapservice.py @@ -24,6 +24,7 @@ from flask import Flask, Response import core from netcontroller import getOpenPort from . import httpheaders +from onionrutils import stringvalidators def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300): ''' @@ -61,7 +62,7 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300): @bootstrap_app.route('/bs/
', methods=['POST']) def get_bootstrap(address): - if core_inst._utils.validateID(address + '.onion'): + if stringvalidators.validate_transport(address + '.onion'): # Set the bootstrap address then close the server bootstrap_address = address + '.onion' core_inst.keyStore.put(bs_id, bootstrap_address) diff --git a/onionr/onionrutils/__init__.py b/onionr/onionrutils/__init__.py index a7efbae7..23907a00 100755 --- a/onionr/onionrutils/__init__.py +++ b/onionr/onionrutils/__init__.py @@ -29,6 +29,7 @@ import storagecounter from etc import pgpwords, onionrvalues from onionrusers import onionrusers from . import localcommand, blockmetadata, validatemetadata, basicrequests +from . import stringvalidators config.reload() class OnionrUtils: @@ -50,23 +51,6 @@ class OnionrUtils: ''' epoch = self.getEpoch() return epoch - (epoch % roundS) - - def getClientAPIServer(self): - retData = '' - try: - with open(self._core.privateApiHostFile, 'r') as host: - hostname = host.read() - except FileNotFoundError: - raise FileNotFoundError - else: - retData += '%s:%s' % (hostname, config.get('client.client.port')) - return retData - - def localCommand(self, command, data='', silent = True, post=False, postData = {}, maxWait=20): - ''' - Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers. - ''' - return localcommand.local_command(self, command, data, silent, post, postData, maxWait) def getHumanReadableID(self, pub=''): '''gets a human readable ID from a public key''' @@ -132,19 +116,7 @@ class OnionrUtils: ''' Validate if a string is a valid hash hex digest (does not compare, just checks length and charset) ''' - retVal = True - if data == False or data == True: - return False - data = data.strip() - if len(data) != length: - retVal = False - else: - try: - int(data, 16) - except ValueError: - retVal = False - - return retVal + return stringvalidators.validate_hash(self, data, length) def validateMetadata(self, metadata, blockData): '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string''' @@ -154,129 +126,7 @@ class OnionrUtils: ''' Validate if a string is a valid base32 encoded Ed25519 key ''' - if type(key) is type(None): - return False - # Accept keys that have no = padding - key = unpaddedbase32.repad(self.strToBytes(key)) - - retVal = False - try: - nacl.signing.SigningKey(seed=key, encoder=nacl.encoding.Base32Encoder) - except nacl.exceptions.ValueError: - pass - except base64.binascii.Error as err: - pass - else: - retVal = True - return retVal - - @staticmethod - def validateID(id): - ''' - Validate if an address is a valid tor or i2p hidden service - ''' - try: - idLength = len(id) - retVal = True - idNoDomain = '' - peerType = '' - # i2p b32 addresses are 60 characters long (including .b32.i2p) - if idLength == 60: - peerType = 'i2p' - if not id.endswith('.b32.i2p'): - retVal = False - else: - idNoDomain = id.split('.b32.i2p')[0] - # Onion v2's are 22 (including .onion), v3's are 62 with .onion - elif idLength == 22 or idLength == 62: - peerType = 'onion' - if not id.endswith('.onion'): - retVal = False - else: - idNoDomain = id.split('.onion')[0] - else: - retVal = False - if retVal: - if peerType == 'i2p': - try: - id.split('.b32.i2p')[2] - except: - pass - else: - retVal = False - elif peerType == 'onion': - try: - id.split('.onion')[2] - except: - pass - else: - retVal = False - if not idNoDomain.isalnum(): - retVal = False - - # Validate address is valid base32 (when capitalized and minus extension); v2/v3 onions and .b32.i2p use base32 - for x in idNoDomain.upper(): - if x not in string.ascii_uppercase and x not in '234567': - retVal = False - - return retVal - except: - return False - - @staticmethod - def isIntegerString(data): - '''Check if a string is a valid base10 integer (also returns true if already an int)''' - try: - int(data) - except (ValueError, TypeError) as e: - return False - else: - return True - - def isCommunicatorRunning(self, timeout = 5, interval = 0.1): - try: - runcheck_file = self._core.dataDir + '.runcheck' - - if not os.path.isfile(runcheck_file): - open(runcheck_file, 'w+').close() - - # self._core.daemonQueueAdd('runCheck') # deprecated - starttime = time.time() - - while True: - time.sleep(interval) - - if not os.path.isfile(runcheck_file): - return True - elif time.time() - starttime >= timeout: - return False - except: - return False - - def importNewBlocks(self, scanDir=''): - ''' - This function is intended to scan for new blocks ON THE DISK and import them - ''' - blockList = self._core.getBlockList() - exist = False - if scanDir == '': - scanDir = self._core.blockDataLocation - if not scanDir.endswith('/'): - scanDir += '/' - for block in glob.glob(scanDir + "*.dat"): - if block.replace(scanDir, '').replace('.dat', '') not in blockList: - exist = True - logger.info('Found new block on dist %s' % block) - with open(block, 'rb') as newBlock: - block = block.replace(scanDir, '').replace('.dat', '') - if self._core._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''): - self._core.addToBlockDB(block.replace('.dat', ''), dataSaved=True) - logger.info('Imported block %s.' % block) - self._core._utils.processBlockMetadata(block) - else: - logger.warn('Failed to verify hash for %s' % block) - if not exist: - logger.info('No blocks found to import') + return stringvalidators.validate_pub_key(self, key) def getEpoch(self): '''returns epoch''' diff --git a/onionr/onionrutils/checkcommunicator.py b/onionr/onionrutils/checkcommunicator.py new file mode 100644 index 00000000..07f8dc1a --- /dev/null +++ b/onionr/onionrutils/checkcommunicator.py @@ -0,0 +1,19 @@ +import time, os +def is_communicator_running(core_inst, timeout = 5, interval = 0.1): + try: + runcheck_file = core_inst.dataDir + '.runcheck' + + if not os.path.isfile(runcheck_file): + open(runcheck_file, 'w+').close() + + starttime = time.time() + + while True: + time.sleep(interval) + + if not os.path.isfile(runcheck_file): + return True + elif time.time() - starttime >= timeout: + return False + except: + return False \ No newline at end of file diff --git a/onionr/onionrutils/getclientapiserver.py b/onionr/onionrutils/getclientapiserver.py new file mode 100644 index 00000000..23020f66 --- /dev/null +++ b/onionr/onionrutils/getclientapiserver.py @@ -0,0 +1,10 @@ +def get_client_API_server(core_inst): + retData = '' + try: + with open(core_inst.privateApiHostFile, 'r') as host: + hostname = host.read() + except FileNotFoundError: + raise FileNotFoundError + else: + retData += '%s:%s' % (hostname, core_inst.config.get('client.client.port')) + return retData \ No newline at end of file diff --git a/onionr/onionrutils/importnewblocks.py b/onionr/onionrutils/importnewblocks.py new file mode 100644 index 00000000..a2c1e518 --- /dev/null +++ b/onionr/onionrutils/importnewblocks.py @@ -0,0 +1,28 @@ +import glob +import logger, core +def import_new_blocks(core_inst=None, scanDir=''): + ''' + This function is intended to scan for new blocks ON THE DISK and import them + ''' + if core_inst is None: + core_inst = core.Core() + blockList = core_inst.getBlockList() + exist = False + if scanDir == '': + scanDir = core_inst.blockDataLocation + if not scanDir.endswith('/'): + scanDir += '/' + for block in glob.glob(scanDir + "*.dat"): + if block.replace(scanDir, '').replace('.dat', '') not in blockList: + exist = True + logger.info('Found new block on dist %s' % block) + with open(block, 'rb') as newBlock: + block = block.replace(scanDir, '').replace('.dat', '') + if core_inst._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''): + core_inst.addToBlockDB(block.replace('.dat', ''), dataSaved=True) + logger.info('Imported block %s.' % block) + core_inst._utils.processBlockMetadata(block) + else: + logger.warn('Failed to verify hash for %s' % block) + if not exist: + logger.info('No blocks found to import') \ No newline at end of file diff --git a/onionr/onionrutils/localcommand.py b/onionr/onionrutils/localcommand.py index fff050c8..90b0d7ac 100644 --- a/onionr/onionrutils/localcommand.py +++ b/onionr/onionrutils/localcommand.py @@ -1,6 +1,7 @@ import urllib, requests, time import logger -def local_command(utils_inst, command, data='', silent = True, post=False, postData = {}, maxWait=20): +from onionrutils import getclientapiserver +def local_command(core_inst, command, data='', silent = True, post=False, postData = {}, maxWait=20): ''' Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers. ''' @@ -9,7 +10,7 @@ def local_command(utils_inst, command, data='', silent = True, post=False, postD waited = 0 while hostname == '': try: - hostname = utils_inst.getClientAPIServer() + hostname = getclientapiserver.get_client_API_server(core_inst) except FileNotFoundError: time.sleep(1) waited += 1 diff --git a/onionr/onionrutils/stringvalidators.py b/onionr/onionrutils/stringvalidators.py new file mode 100644 index 00000000..49f2c33a --- /dev/null +++ b/onionr/onionrutils/stringvalidators.py @@ -0,0 +1,97 @@ +import base64, string +import unpaddedbase32, nacl.signing, nacl.encoding +def validate_hash(utils_inst, data, length=64): + ''' + Validate if a string is a valid hash hex digest (does not compare, just checks length and charset) + ''' + retVal = True + if data == False or data == True: + return False + data = data.strip() + if len(data) != length: + retVal = False + else: + try: + int(data, 16) + except ValueError: + retVal = False + + return retVal + +def validate_pub_key(utils_inst, key): + ''' + Validate if a string is a valid base32 encoded Ed25519 key + ''' + if type(key) is type(None): + return False + # Accept keys that have no = padding + key = unpaddedbase32.repad(utils_inst.strToBytes(key)) + + retVal = False + try: + nacl.signing.SigningKey(seed=key, encoder=nacl.encoding.Base32Encoder) + except nacl.exceptions.ValueError: + pass + except base64.binascii.Error as err: + pass + else: + retVal = True + return retVal + +def validate_transport(id): + try: + idLength = len(id) + retVal = True + idNoDomain = '' + peerType = '' + # i2p b32 addresses are 60 characters long (including .b32.i2p) + if idLength == 60: + peerType = 'i2p' + if not id.endswith('.b32.i2p'): + retVal = False + else: + idNoDomain = id.split('.b32.i2p')[0] + # Onion v2's are 22 (including .onion), v3's are 62 with .onion + elif idLength == 22 or idLength == 62: + peerType = 'onion' + if not id.endswith('.onion'): + retVal = False + else: + idNoDomain = id.split('.onion')[0] + else: + retVal = False + if retVal: + if peerType == 'i2p': + try: + id.split('.b32.i2p')[2] + except: + pass + else: + retVal = False + elif peerType == 'onion': + try: + id.split('.onion')[2] + except: + pass + else: + retVal = False + if not idNoDomain.isalnum(): + retVal = False + + # Validate address is valid base32 (when capitalized and minus extension); v2/v3 onions and .b32.i2p use base32 + for x in idNoDomain.upper(): + if x not in string.ascii_uppercase and x not in '234567': + retVal = False + + return retVal + except Exception as e: + return False + +def is_integer_string(data): + '''Check if a string is a valid base10 integer (also returns true if already an int)''' + try: + int(data) + except (ValueError, TypeError) as e: + return False + else: + return True diff --git a/onionr/onionrutils/validatemetadata.py b/onionr/onionrutils/validatemetadata.py index c4bcc4a8..8feb9859 100644 --- a/onionr/onionrutils/validatemetadata.py +++ b/onionr/onionrutils/validatemetadata.py @@ -1,6 +1,7 @@ import json import logger, onionrexceptions from etc import onionrvalues +from onionrutils import stringvalidators def validate_metadata(utils_inst, metadata, blockData): '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string''' # TODO, make this check sane sizes @@ -15,7 +16,7 @@ def validate_metadata(utils_inst, metadata, blockData): pass # Validate metadata dict for invalid keys to sizes that are too large - maxAge = utils_inst._coreconfig.get("general.max_block_age", onionrvalues.OnionrValues().default_expire) + maxAge = utils_inst._core.config.get("general.max_block_age", onionrvalues.OnionrValues().default_expire) if type(metadata) is dict: for i in metadata: try: @@ -33,7 +34,7 @@ def validate_metadata(utils_inst, metadata, blockData): logger.warn('Block metadata key ' + i + ' exceeded maximum size') break if i == 'time': - if not utils_inst.isIntegerString(metadata[i]): + if not stringvalidators.is_integer_string(metadata[i]): logger.warn('Block metadata time stamp is not integer string or int') break isFuture = (metadata[i] - utils_inst.getEpoch()) diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py index f6a0b906..c65917c8 100755 --- a/onionr/static-data/default-plugins/cliui/main.py +++ b/onionr/static-data/default-plugins/cliui/main.py @@ -23,6 +23,7 @@ import threading, time, uuid, subprocess, sys import config, logger from onionrblockapi import Block import onionrplugins +from onionrutils import localcommand plugin_name = 'cliui' PLUGIN_VERSION = '0.0.1' @@ -48,7 +49,7 @@ class OnionrCLIUI: def isRunning(self): while not self.shutdown: - if self.myCore._utils.localCommand('ping', maxWait=5) == 'pong!': + if localcommand.local_command(self.myCore, 'ping', maxWait=5) == 'pong!': self.running = 'Yes' else: self.running = 'No' diff --git a/onionr/static-data/default-plugins/esoteric/peerserver.py b/onionr/static-data/default-plugins/esoteric/peerserver.py index b3ae423e..f48ba0be 100755 --- a/onionr/static-data/default-plugins/esoteric/peerserver.py +++ b/onionr/static-data/default-plugins/esoteric/peerserver.py @@ -19,6 +19,7 @@ ''' import sys, os, json import core +from onionrutils import localcommand from flask import Response, request, redirect, Blueprint, abort, g sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) direct_blueprint = Blueprint('esoteric', __name__) @@ -47,9 +48,9 @@ def sendto(): msg = '' else: msg = json.dumps(msg) - core_inst._utils.localCommand('/esoteric/addrec/%s' % (g.peer,), post=True, postData=msg) + localcommand.local_command(core_inst, '/esoteric/addrec/%s' % (g.peer,), post=True, postData=msg) return Response('success') @direct_blueprint.route('/esoteric/poll') def poll_chat(): - return Response(core_inst._utils.localCommand('/esoteric/gets/%s' % (g.peer,))) \ No newline at end of file + return Response(localcommand.local_command(core_inst, '/esoteric/gets/%s' % (g.peer,))) \ No newline at end of file diff --git a/onionr/static-data/default-plugins/pluginmanager/main.py b/onionr/static-data/default-plugins/pluginmanager/main.py index d4feb69d..2f261bbf 100755 --- a/onionr/static-data/default-plugins/pluginmanager/main.py +++ b/onionr/static-data/default-plugins/pluginmanager/main.py @@ -22,6 +22,7 @@ import logger, config import os, sys, json, time, random, shutil, base64, getpass, datetime, re from onionrblockapi import Block +from onionrutils import importnewblocks plugin_name = 'pluginmanager' @@ -236,7 +237,7 @@ def pluginToBlock(plugin, import_block = True): # hash = pluginapi.get_core().insertBlock(, header = 'plugin', sign = True) if import_block: - pluginapi.get_utils().importNewBlocks() + importnewblocks.import_new_blocks(pluginapi.get_core()) return hash else: diff --git a/onionr/tests/test_stringvalidations.py b/onionr/tests/test_stringvalidations.py index caac34df..4eb7f113 100755 --- a/onionr/tests/test_stringvalidations.py +++ b/onionr/tests/test_stringvalidations.py @@ -6,6 +6,7 @@ TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' print("Test directory:", TEST_DIR) os.environ["ONIONR_HOME"] = TEST_DIR import core, onionr +from onionrutils import stringvalidators core.Core() @@ -13,7 +14,6 @@ class OnionrValidations(unittest.TestCase): def test_peer_validator(self): # Test hidden service domain validities - c = core.Core() valid = ['facebookcorewwwi.onion', 'vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd.onion', '5bvb5ncnfr4dlsfriwczpzcvo65kn7fnnlnt2ln7qvhzna2xaldq.b32.i2p'] @@ -21,11 +21,11 @@ class OnionrValidations(unittest.TestCase): for x in valid: print('testing', x) - self.assertTrue(c._utils.validateID(x)) + self.assertTrue(stringvalidators.validate_transport(x)) for x in invalid: print('testing', x) - self.assertFalse(c._utils.validateID(x)) + self.assertFalse(stringvalidators.validate_transport(x)) def test_pubkey_validator(self): # Test ed25519 public key validity @@ -44,14 +44,13 @@ class OnionrValidations(unittest.TestCase): def test_integer_string(self): valid = ["1", "100", 100, "-5", -5] invalid = ['test', "1d3434", "1e100", None] - c = core.Core() for x in valid: #print('testing', x) - self.assertTrue(c._utils.isIntegerString(x)) + self.assertTrue(stringvalidators.is_integer_string(x)) for x in invalid: #print('testing', x) - self.assertFalse(c._utils.isIntegerString(x)) + self.assertFalse(stringvalidators.is_integer_string(x)) unittest.main() \ No newline at end of file