From 1dd471b91e714750dcfea61be6a9ab85d03da032 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 22 Dec 2018 13:02:09 -0600 Subject: [PATCH] + Reformatted API, more efficient, standard, and secure now * Various bug fixes --- onionr/api.py | 83 +++++++++++++++++++------ onionr/communicator2.py | 15 +++-- onionr/core.py | 2 +- onionr/netcontroller.py | 2 +- onionr/onionr.py | 112 ++++++++++++++++++++-------------- onionr/onionrdaemontools.py | 2 +- onionr/onionrpeers.py | 38 ++++++------ onionr/onionrproofs.py | 1 - onionr/onionrutils.py | 16 ++--- onionr/static-data/index.html | 4 +- requirements.txt | 1 - 11 files changed, 174 insertions(+), 102 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index e2a6604c..f983677d 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -25,8 +25,6 @@ import core from onionrblockapi import Block import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionr -API_VERSION = 0 - def guessMime(path): ''' Guesses the mime type of a file from the input filename @@ -67,6 +65,7 @@ class PublicAPI: self.torAdder = clientAPI._core.hsAddress self.i2pAdder = clientAPI._core.i2pAddress self.bindPort = config.get('client.public.port') + logger.info('Running public api on %s:%s' % (self.host, self.bindPort)) @app.before_request def validateRequest(): @@ -84,7 +83,7 @@ class PublicAPI: resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'" resp.headers['X-Frame-Options'] = 'deny' resp.headers['X-Content-Type-Options'] = "nosniff" - resp.headers['X-API'] = API_VERSION + resp.headers['X-API'] = onionr.API_VERSION return resp @app.route('/') @@ -105,8 +104,9 @@ class PublicAPI: return Response('\n'.join(bList)) @app.route('/getdata/') - def getBlockData(): + def getBlockData(name): resp = '' + data = name if clientAPI._utils.validateHash(data): if data not in self.hideBlocks: if os.path.exists(clientAPI._core.dataDir + 'blocks/' + data + '.dat'): @@ -117,20 +117,64 @@ class PublicAPI: resp = "" return Response(resp) + @app.route('/www/') + def wwwPublic(path): + if not config.get("www.public.run", True): + abort(403) + return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path) + @app.route('/ping') def ping(): return Response("pong!") - + @app.route('/getdbhash') def getDBHash(): return Response(clientAPI._utils.getBlockDBHash()) - + @app.route('/pex') def peerExchange(): - response = ','.join(self._core.listAdders()) + response = ','.join(clientAPI._core.listAdders()) if len(response) == 0: response = 'none' - resp = Response(response) + return Response(response) + + @app.route('/announce', methods=['post']) + def acceptAnnounce(): + resp = 'failure' + powHash = '' + randomData = '' + newNode = '' + ourAdder = clientAPI._core.hsAddress.encode() + try: + newNode = request.form['node'].encode() + except KeyError: + logger.warn('No block specified for upload') + pass + else: + try: + randomData = request.form['random'] + randomData = base64.b64decode(randomData) + except KeyError: + logger.warn('No random data specified for upload') + else: + nodes = newNode + clientAPI._core.hsAddress.encode() + nodes = clientAPI._core._crypto.blake2bHash(nodes) + powHash = clientAPI._core._crypto.blake2bHash(randomData + nodes) + try: + powHash = powHash.decode() + except AttributeError: + pass + if powHash.startswith('0000'): + try: + newNode = newNode.decode() + except AttributeError: + pass + if clientAPI._core.addAddress(newNode): + resp = 'Success' + else: + logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash) + resp = Response(resp) + return resp @app.route('/upload', methods=['post']) def upload(): @@ -156,6 +200,10 @@ class PublicAPI: return resp clientAPI.setPublicAPIInstance(self) + while self.torAdder == '': + clientAPI._core.refreshFirstStartVars() + self.torAdder = clientAPI._core.hsAddress + time.sleep(1) self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None) self.httpServer.serve_forever() @@ -166,7 +214,7 @@ class API: callbacks = {'public' : {}, 'private' : {}} - def __init__(self, debug, API_VERSION): + def __init__(self, onionrInst, debug, API_VERSION): ''' Initialize the api server, preping variables for later use @@ -190,10 +238,11 @@ class API: self.timeBypassToken = base64.b16encode(os.urandom(32)).decode() self.publicAPI = None # gets set when the thread calls our setter... bad hack but kinda necessary with flask - threading.Thread(target=PublicAPI, args=(self,)).start() + #threading.Thread(target=PublicAPI, args=(self,)).start() self.host = setBindIP(self._core.privateApiHostFile) logger.info('Running api on %s:%s' % (self.host, self.bindPort)) self.httpServer = '' + onionrInst.setClientAPIInst(self) @app.before_request def validateRequest(): @@ -205,20 +254,20 @@ class API: abort(403) except KeyError: abort(403) - + @app.after_request def afterReq(resp): resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'" resp.headers['X-Frame-Options'] = 'deny' resp.headers['X-Content-Type-Options'] = "nosniff" - resp.headers['X-API'] = API_VERSION - resp.headers['Server'] = 'nginx' + resp.headers['X-API'] = onionr.API_VERSION + resp.headers['Server'] = '' resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch. return resp @app.route('/ping') def ping(): - return Respose("pong!") + return Response("pong!") @app.route('/') def hello(): @@ -256,10 +305,10 @@ class API: except AttributeError: pass return Response("bye") - + self.httpServer = WSGIServer((self.host, bindPort), app, log=None) self.httpServer.serve_forever() - + def setPublicAPIInstance(self, inst): assert isinstance(inst, PublicAPI) self.publicAPI = inst @@ -277,4 +326,4 @@ class API: else: return True except TypeError: - return False \ No newline at end of file + return False diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 15b0d047..5d045a69 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -19,11 +19,10 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -#import gevent.monkey -#gevent.monkey.patch_all() import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block import onionrdaemontools, onionrsockets, onionrchat, onionr, onionrproofs +import binascii from dependencies import secrets from defusedxml import minidom @@ -103,6 +102,7 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True) OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1) OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1) + OnionrCommunicatorTimers(self, self.detectAPICrash, 5, maxThreads=1) deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1) netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) @@ -232,7 +232,10 @@ class OnionrCommunicatorDaemon: content = content.encode() except AttributeError: pass - content = base64.b64decode(content) # content is base64 encoded in transport + try: + content = base64.b64decode(content) # content is base64 encoded in transport + except binascii.Error: + pass realHash = self._core._crypto.sha3Hash(content) try: realHash = realHash.decode() # bytes on some versions for some reason @@ -423,7 +426,7 @@ class OnionrCommunicatorDaemon: if len(peer) == 0: return False #logger.debug('Performing ' + action + ' with ' + peer + ' on port ' + str(self.proxyPort)) - url = 'http://' + peer + '/' + action + url = 'http://%s/%s' % (peer, action) if len(data) > 0: url += '&data=' + data @@ -546,9 +549,9 @@ class OnionrCommunicatorDaemon: def detectAPICrash(self): '''exit if the api server crashes/stops''' - if self._core._utils.localCommand('ping', silent=False) != 'pong': + if self._core._utils.localCommand('ping', silent=False) not in ('pong', 'pong!'): for i in range(5): - if self._core._utils.localCommand('ping') == 'pong': + if self._core._utils.localCommand('ping') in ('pong', 'pong!'): break # break for loop time.sleep(1) else: diff --git a/onionr/core.py b/onionr/core.py index d2d55811..a1d78e0a 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -154,7 +154,7 @@ class Core: if address == config.get('i2p.ownAddr', None) or address == self.hsAddress: return False - if type(address) is type(None) or len(address) == 0: + if type(address) is None or len(address) == 0: return False if self._utils.validateID(address): conn = sqlite3.connect(self.addressDB, timeout=10) diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py index 7566623e..c87b5e46 100644 --- a/onionr/netcontroller.py +++ b/onionr/netcontroller.py @@ -77,7 +77,7 @@ class NetController: hsVer = '# v2 onions' if config.get('tor.v3onions'): hsVer = 'HiddenServiceVersion 3' - logger.debug('Using v3 onions :)') + logger.debug('Using v3 onions') if os.path.exists(self.torConfigLocation): os.remove(self.torConfigLocation) diff --git a/onionr/onionr.py b/onionr/onionr.py index 0493f1e4..b9dd6260 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -67,6 +67,7 @@ class Onionr: data_exists = Onionr.setupConfig(self.dataDir, self = self) self.onionrCore = core.Core() + #self.deleteRunFiles() self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore) self.clientAPIInst = '' # Client http api instance @@ -399,6 +400,16 @@ class Onionr: logger.info('Syntax: friend add/remove/list [address]') + def deleteRunFiles(self): + try: + os.remove(self.onionrCore.publicApiHostFile) + except FileNotFoundError: + pass + try: + os.remove(self.onionrCore.privateApiHostFile) + except FileNotFoundError: + pass + def banBlock(self): try: ban = sys.argv[2] @@ -695,6 +706,13 @@ class Onionr: os.remove('.onionr-lock') except FileNotFoundError: pass + def setClientAPIInst(self, inst): + self.clientAPIInst = inst + + def getClientApi(self): + while self.clientAPIInst == '': + time.sleep(0.5) + return self.clientAPIInst def daemon(self): ''' @@ -708,64 +726,66 @@ class Onionr: logger.debug('Runcheck file found on daemon start, deleting in advance.') os.remove('data/.runcheck') - apiTarget = api.API - apiThread = Thread(target = apiTarget, args = (self.debug, API_VERSION)) - apiThread.start() - + Thread(target=api.API, args=(self, self.debug, API_VERSION)).start() + Thread(target=api.PublicAPI, args=[self.getClientApi()]).start() try: - time.sleep(3) + time.sleep(0) except KeyboardInterrupt: logger.debug('Got keyboard interrupt, shutting down...') time.sleep(1) self.onionrUtils.localCommand('shutdown') + + apiHost = '' + while apiHost == '': + try: + with open(self.onionrCore.publicApiHostFile, 'r') as hostFile: + apiHost = hostFile.read() + except FileNotFoundError: + pass + time.sleep(0.5) + Onionr.setupConfig('data/', self = self) + + if self._developmentMode: + logger.warn('DEVELOPMENT MODE ENABLED (LESS SECURE)', timestamp = False) + net = NetController(config.get('client.public.port', 59497), apiServerIP=apiHost) + logger.debug('Tor is starting...') + if not net.startTor(): + self.onionrUtils.localCommand('shutdown') + sys.exit(1) + if len(net.myID) > 0 and config.get('general.security_level') == 0: + logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID)) else: - apiHost = '127.0.0.1' - if apiThread.isAlive(): - try: - with open(self.onionrCore.publicApiHostFile, 'r') as hostFile: - apiHost = hostFile.read() - except FileNotFoundError: - pass - Onionr.setupConfig('data/', self = self) + logger.debug('.onion service disabled') + logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey)) + time.sleep(1) - if self._developmentMode: - logger.warn('DEVELOPMENT MODE ENABLED (LESS SECURE)', timestamp = False) - net = NetController(config.get('client.public.port', 59497), apiServerIP=apiHost) - logger.debug('Tor is starting...') - if not net.startTor(): - self.onionrUtils.localCommand('shutdown') - sys.exit(1) - if len(net.myID) > 0 and config.get('general.security_level') == 0: - logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID)) - else: - logger.debug('.onion service disabled') - logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey)) - time.sleep(1) + # TODO: make runable on windows + communicatorProc = subprocess.Popen([communicatorDaemon, 'run', str(net.socksPort)]) - # TODO: make runable on windows - communicatorProc = subprocess.Popen([communicatorDaemon, 'run', str(net.socksPort)]) + # print nice header thing :) + if config.get('general.display_header', True): + self.header() - # print nice header thing :) - if config.get('general.display_header', True): - self.header() + # print out debug info + self.version(verbosity = 5, function = logger.debug) + logger.debug('Python version %s' % platform.python_version()) - # print out debug info - self.version(verbosity = 5, function = logger.debug) - logger.debug('Python version %s' % platform.python_version()) + logger.debug('Started communicator.') - logger.debug('Started communicator.') + events.event('daemon_start', onionr = self) + try: + while True: + time.sleep(5) - events.event('daemon_start', onionr = self) - try: - while True: - time.sleep(5) - - # Break if communicator process ends, so we don't have left over processes - if communicatorProc.poll() is not None: - break - except KeyboardInterrupt: - self.onionrCore.daemonQueueAdd('shutdown') - self.onionrUtils.localCommand('shutdown') + # Break if communicator process ends, so we don't have left over processes + if communicatorProc.poll() is not None: + break + except KeyboardInterrupt: + self.onionrCore.daemonQueueAdd('shutdown') + self.onionrUtils.localCommand('shutdown') + time.sleep(3) + self.deleteRunFiles() + net.killTor() return def killDaemon(self): diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index ebe181c6..9f1ab2a2 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -45,7 +45,7 @@ class DaemonTools: ourID = self.daemon._core.hsAddress.strip() - url = 'http://' + peer + '/public/announce/' + url = 'http://' + peer + '/announce' data = {'node': ourID} combinedNodes = ourID + peer diff --git a/onionr/onionrpeers.py b/onionr/onionrpeers.py index 62716eee..e4793dfb 100644 --- a/onionr/onionrpeers.py +++ b/onionr/onionrpeers.py @@ -79,27 +79,29 @@ def peerCleanup(coreInst): logger.info('Cleaning peers...') config.reload() - minScore = int(config.get('peers.minimum_score', -100)) - maxPeers = int(config.get('peers.max_stored', 5000)) - adders = getScoreSortedPeerList(coreInst) adders.reverse() + + if len(adders) > 1: - for address in adders: - # Remove peers that go below the negative score - if PeerProfiles(address, coreInst).score < minScore: - coreInst.removeAddress(address) - try: - if (int(coreInst._utils.getEpoch()) - int(coreInst.getPeerInfo(address, 'dateSeen'))) >= 600: - expireTime = 600 - else: - expireTime = 86400 - coreInst._blacklist.addToDB(address, dataType=1, expire=expireTime) - except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue - pass - except ValueError: - pass - logger.warn('Removed address ' + address + '.') + minScore = int(config.get('peers.minimum_score', -100)) + maxPeers = int(config.get('peers.max_stored', 5000)) + + for address in adders: + # Remove peers that go below the negative score + if PeerProfiles(address, coreInst).score < minScore: + coreInst.removeAddress(address) + try: + if (int(coreInst._utils.getEpoch()) - int(coreInst.getPeerInfo(address, 'dateSeen'))) >= 600: + expireTime = 600 + else: + expireTime = 86400 + coreInst._blacklist.addToDB(address, dataType=1, expire=expireTime) + except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue + pass + except ValueError: + pass + logger.warn('Removed address ' + address + '.') # Unban probably not malicious peers TODO improve coreInst._blacklist.deleteExpired(dataType=1) diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py index 0bbfee6b..9665a4cb 100644 --- a/onionr/onionrproofs.py +++ b/onionr/onionrproofs.py @@ -238,7 +238,6 @@ class POW: break else: time.sleep(2) - print('boi') except KeyboardInterrupt: self.shutdown() logger.warn('Got keyboard interrupt while waiting for POW result, stopping') diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 75fcb7c2..c6cfdfb9 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -159,17 +159,17 @@ class OnionrUtils: config.reload() self.getTimeBypassToken() # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless. - try: - with open(self._core.privateApiHostFile, 'r') as host: - hostname = host.read() - except FileNotFoundError: - return False + hostname = '' + while hostname == '': + try: + with open(self._core.privateApiHostFile, 'r') as host: + hostname = host.read() + except FileNotFoundError: + print('wat') + time.sleep(1) if data != '': data = '&data=' + urllib.parse.quote_plus(data) payload = 'http://%s:%s/%s%s' % (hostname, config.get('client.client.port'), command, data) - #payload = 'http://%s:%s/client/?action=%s&token=%s&timingToken=%s' % (hostname, config.get('client.client.port'), command, config.get('client.webpassword'), self.timingToken) - #if data != '': - # payload += '&data=' + urllib.parse.quote_plus(data) try: retData = requests.get(payload, headers={'token': config.get('client.webpassword')}).text except Exception as error: diff --git a/onionr/static-data/index.html b/onionr/static-data/index.html index 157f1586..916bfe9c 100644 --- a/onionr/static-data/index.html +++ b/onionr/static-data/index.html @@ -1,7 +1,7 @@

This is an Onionr Node

-

The content on this server is not necessarily created by the server owner, and was not necessarily stored specifically with the owner's knowledge of its contents.

+

The content on this server was not necessarily intentionally stored or created by the owner(s) of this server.

-

Onionr is a decentralized data storage system that anyone can insert data into.

+

Onionr is a decentralized peer-to-peer data storage system.

To learn more about Onionr, see the website at https://Onionr.VoidNet.tech/

diff --git a/requirements.txt b/requirements.txt index 2375324d..6bceb49a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,6 @@ urllib3==1.23 requests==2.20.0 PyNaCl==1.2.1 gevent==1.3.6 -sha3==0.2.1 defusedxml==0.5.0 Flask==1.0.2 PySocks==1.6.8