From eb8c309626444da9b8b64fd42d47ea5ad7114445 Mon Sep 17 00:00:00 2001 From: Benjamin Levy Date: Fri, 20 Jul 2018 17:49:03 -0400 Subject: [PATCH 01/53] Update the Makefile to comply with the DESTDIR/PREFIX convention --- Makefile | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 472ffc2d..a9c5a6f5 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,20 @@ +PREFIX = /usr/local + .DEFAULT_GOAL := setup setup: - sudo pip3 install -r requirements.txt + pip3 install -r requirements.txt install: - sudo rm -rf /usr/share/onionr/ - sudo rm -f /usr/bin/onionr - sudo cp -rp ./onionr /usr/share/onionr - sudo sh -c "echo \"#!/bin/sh\ncd /usr/share/onionr/\n./onionr.py \\\"\\\$$@\\\"\" > /usr/bin/onionr" - sudo chmod +x /usr/bin/onionr - sudo chown -R `whoami` /usr/share/onionr/ + cp -rfp ./onionr $(DESTDIR)$(PREFIX)/share/onionr + echo '#!/bin/sh' > $(DESTDIR)$(PREFIX)/bin/onionr + echo 'cd $(DESTDIR)$(PREFIX)/share/onionr' > $(DESTDIR)$(PREFIX)/bin/onionr + echo './onionr \\\"\\\$$@\\\"\"' > $(DESTDIR)$(PREFIX)/bin/onionr + chmod +x $(DESTDIR)$(PREFIX)/bin/onionr uninstall: - sudo rm -rf /usr/share/onionr - sudo rm -f /usr/bin/onionr + rm -rf $(DESTDIR)$(PREFIX)/share/onionr + rm -f $(DESTDIR)$(PREFIX)/bin/onionr test: @./RUN-LINUX.sh stop From e59d4645e1e7662afc91042090f0712cf95d1084 Mon Sep 17 00:00:00 2001 From: Benjamin Levy Date: Fri, 20 Jul 2018 17:55:03 -0400 Subject: [PATCH 02/53] Fix onionr start script --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a9c5a6f5..cb290800 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ install: cp -rfp ./onionr $(DESTDIR)$(PREFIX)/share/onionr echo '#!/bin/sh' > $(DESTDIR)$(PREFIX)/bin/onionr echo 'cd $(DESTDIR)$(PREFIX)/share/onionr' > $(DESTDIR)$(PREFIX)/bin/onionr - echo './onionr \\\"\\\$$@\\\"\"' > $(DESTDIR)$(PREFIX)/bin/onionr + echo './onionr "$$@"' > $(DESTDIR)$(PREFIX)/bin/onionr chmod +x $(DESTDIR)$(PREFIX)/bin/onionr uninstall: From 2907471b5f570d6493c2611e33c32b184f06cb90 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 5 Aug 2018 01:35:49 -0500 Subject: [PATCH 03/53] * always check needed config on startup + added Dockerfile --- Dockerfile | 22 ++++++++++++++++++++++ onionr/onionr.py | 25 +++++++++++++------------ 2 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..7331dd6f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM ubuntu:xenial + +#Base settings +ENV HOME /root + +#Install needed packages +RUN apt update && apt install -y python3 python3-dev python3-pip tor + +#Add Onionr source +COPY . /root +VOLUME /root/data + +WORKDIR /root + +RUN pip3 install -r requirements.txt + +#Set upstart command +#CMD (! ${ENABLE_TOR} || tor&) && python zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552 +CMD bash + +#Expose ports +EXPOSE 8080 diff --git a/onionr/onionr.py b/onionr/onionr.py index 278ba60d..05f22099 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -135,18 +135,19 @@ class Onionr: self.onionrCore.createAddressDB() # Get configuration - - if not data_exists: - # Generate default config - # Hostname should only be set if different from 127.x.x.x. Important for DNS rebinding attack prevention. - if self.debug: - randomPort = 8080 - else: - while True: - randomPort = random.randint(1024, 65535) - if self.onionrUtils.checkPort(randomPort): - break - config.set('client', {'participate': True, 'hmac': base64.b16encode(os.urandom(32)).decode('utf-8'), 'port': randomPort, 'api_version': API_VERSION}, True) + if type(config.get('client.hmac')) is type(None): + config.set('client.hmac', base64.b16encode(os.urandom(32)).decode('utf-8'), savefile=True) + if type(config.get('client.port')) is type(None): + #while True: + randomPort = random.randint(1024, 65535) + # if self.onionrUtils.checkPort(randomPort): + # break + config.set('client.port', randomPort, savefile=True) + if type(config.get('client.participate')) is type(None): + config.set('client.participate', True, savefile=True) + if type(config.get('client.api_version')) is type(None): + config.set('client.api_version', API_VERSION, savefile=True) + self.cmds = { '': self.showHelpSuggestion, From 1ae27253199522ff0ee3b4e7f8cadd4261441912 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 6 Aug 2018 02:50:08 -0500 Subject: [PATCH 04/53] improved onionr startup and port binding --- onionr/api.py | 1 - onionr/netcontroller.py | 2 +- onionr/onionr.py | 51 ++++++++++++++++++++++++++--------------- onionr/onionrutils.py | 1 + 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index 3ccebff5..d41245d1 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -492,7 +492,6 @@ class API: self.http_server.serve_forever() except KeyboardInterrupt: pass - #app.run(host=self.host, port=bindPort, debug=False, threaded=True) except Exception as e: logger.error(str(e)) logger.fatal('Failed to start client on ' + self.host + ':' + str(bindPort) + ', exiting...') diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py index 386c334b..19f355e4 100644 --- a/onionr/netcontroller.py +++ b/onionr/netcontroller.py @@ -97,7 +97,7 @@ DataDirectory data/tordata/ elif 'Opening Socks listener' in line.decode(): logger.debug(line.decode().replace('\n', '')) else: - logger.fatal('Failed to start Tor. Try killing any other Tor processes owned by this user.') + logger.fatal('Failed to start Tor. Maybe a stray instance of Tor used by Onionr is still running?') return False except KeyboardInterrupt: logger.fatal("Got keyboard interrupt.") diff --git a/onionr/onionr.py b/onionr/onionr.py index 8ed144ee..0de17850 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -138,10 +138,9 @@ class Onionr: if type(config.get('client.hmac')) is type(None): config.set('client.hmac', base64.b16encode(os.urandom(32)).decode('utf-8'), savefile=True) if type(config.get('client.port')) is type(None): - #while True: - randomPort = random.randint(1024, 65535) - # if self.onionrUtils.checkPort(randomPort): - # break + randomPort = 0 + while randomPort < 1024: + randomPort = self.onionrCore._crypto.secrets.randbelow(65535) config.set('client.port', randomPort, savefile=True) if type(config.get('client.participate')) is type(None): config.set('client.participate', True, savefile=True) @@ -544,22 +543,36 @@ class Onionr: Starts the Onionr communication daemon ''' communicatorDaemon = './communicator2.py' - if not os.environ.get("WERKZEUG_RUN_MAIN") == "true": - if self._developmentMode: - logger.warn('DEVELOPMENT MODE ENABLED (THIS IS LESS SECURE!)', timestamp = False) - net = NetController(config.get('client.port', 59496)) - logger.info('Tor is starting...') - if not net.startTor(): - sys.exit(1) - logger.info('Started .onion service: ' + logger.colors.underline + net.myID) - logger.info('Our Public key: ' + self.onionrCore._crypto.pubKey) - time.sleep(1) - #TODO make runable on windows - subprocess.Popen([communicatorDaemon, "run", str(net.socksPort)]) - logger.debug('Started communicator') - events.event('daemon_start', onionr = self) - self.api = api.API(self.debug) + apiThread = Thread(target=api.API, args=(self.debug,)) + apiThread.start() + try: + time.sleep(3) + except KeyboardInterrupt: + logger.info('Got keyboard interrupt') + time.sleep(1) + self.onionrUtils.localCommand('shutdown') + else: + if apiThread.isAlive(): + if self._developmentMode: + logger.warn('DEVELOPMENT MODE ENABLED (THIS IS LESS SECURE!)', timestamp = False) + net = NetController(config.get('client.port', 59496)) + logger.info('Tor is starting...') + if not net.startTor(): + sys.exit(1) + logger.info('Started .onion service: ' + logger.colors.underline + net.myID) + logger.info('Our Public key: ' + self.onionrCore._crypto.pubKey) + time.sleep(1) + #TODO make runable on windows + subprocess.Popen([communicatorDaemon, "run", str(net.socksPort)]) + logger.debug('Started communicator') + events.event('daemon_start', onionr = self) + try: + while True: + time.sleep(5) + except KeyboardInterrupt: + self.onionrCore.daemonQueueAdd('shutdown') + self.onionrUtils.localCommand('shutdown') return def killDaemon(self): diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 531568ee..afa86ef6 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -553,6 +553,7 @@ class OnionrUtils: ''' Do a get request through a local tor or i2p instance ''' + retData = False if proxyType == 'tor': if port == 0: raise onionrexceptions.MissingPort('Socks port required for Tor HTTP get request') From 0ae052336c892b3c8499d33b32d155bf889c0b74 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 7 Aug 2018 02:31:53 -0500 Subject: [PATCH 05/53] + onionr now introduces automatically + added daemon tools file + added .dockerignore --- .dockerignore | 2 ++ onionr/communicator2.py | 13 +++++++++++-- onionr/core.py | 5 ++--- onionr/onionrdaemontools.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 .dockerignore create mode 100644 onionr/onionrdaemontools.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..d76a75c1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +onionr/data/**/* +onionr/data diff --git a/onionr/communicator2.py b/onionr/communicator2.py index b641b76e..6a37a376 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -21,6 +21,7 @@ ''' import sys, os, core, config, json, requests, time, logger, threading, base64, onionr import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block +import onionrdaemontools from defusedxml import minidom class OnionrCommunicatorDaemon: @@ -69,6 +70,11 @@ class OnionrCommunicatorDaemon: # Loads in and starts the enabled plugins plugins.reload() + # daemon tools are misc daemon functions, e.g. announce to online peers + # intended only for use by OnionrCommunicatorDaemon + #self.daemonTools = onionrdaemontools.DaemonTools(self) + self.daemonTools = onionrdaemontools.DaemonTools(self) + if debug or developmentMode: OnionrCommunicatorTimers(self, self.heartbeat, 10) @@ -78,6 +84,7 @@ class OnionrCommunicatorDaemon: # Set timers, function reference, seconds # requiresPeer True means the timer function won't fire if we have no connected peers + # TODO: make some of these timer counts configurable OnionrCommunicatorTimers(self, self.daemonCommands, 5) OnionrCommunicatorTimers(self, self.detectAPICrash, 5) peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60) @@ -86,11 +93,13 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58) OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True) OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True) + announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 305, requiresPeer=True) cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requiresPeer=True) # set loop to execute instantly to load up peer pool (replaced old pool init wait) peerPoolTimer.count = (peerPoolTimer.frequency - 1) cleanupTimer.count = (cleanupTimer.frequency - 60) + announceTimer.count = (cleanupTimer.frequency - 60) # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking try: @@ -278,7 +287,7 @@ class OnionrCommunicatorDaemon: def addBootstrapListToPeerList(self, peerList): '''Add the bootstrap list to the peer list (no duplicates)''' for i in self._core.bootstrapList: - if i not in peerList and i not in self.offlinePeers and i != self._core.hsAdder: + if i not in peerList and i not in self.offlinePeers and i != self._core.hsAddress: peerList.append(i) def connectNewPeer(self, peer='', useBootstrap=False): @@ -441,7 +450,7 @@ class OnionrCommunicatorDaemon: announceAmount = 2 for peer in self.onlinePeers: announceCount += 1 - if self.peerAction(peer, 'announce', self._core.hsAdder) == 'Success': + if self.peerAction(peer, 'announce', self._core.hsAddress) == 'Success': logger.info('Successfully introduced node to ' + peer) break else: diff --git a/onionr/core.py b/onionr/core.py index eb45e182..64ad6439 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -40,7 +40,7 @@ class Core: self.blockDB = 'data/blocks.db' self.blockDataLocation = 'data/blocks/' self.addressDB = 'data/address.db' - self.hsAdder = '' + self.hsAddress = '' self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt' self.bootstrapList = [] self.requirements = onionrvalues.OnionrValues() @@ -57,7 +57,7 @@ class Core: if os.path.exists('data/hs/hostname'): with open('data/hs/hostname', 'r') as hs: - self.hsAdder = hs.read().strip() + self.hsAddress = hs.read().strip() # Load bootstrap address list if os.path.exists(self.bootstrapFileLocation): @@ -158,7 +158,6 @@ class Core: conn.close() events.event('address_remove', data = {'address': address}, onionr = None) - return True else: return False diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py new file mode 100644 index 00000000..e615aacb --- /dev/null +++ b/onionr/onionrdaemontools.py @@ -0,0 +1,29 @@ +''' + Onionr - P2P Microblogging Platform & Social network. + + Contains the CommunicatorUtils class which contains useful functions for the communicator daemon +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import onionrexceptions, onionrpeers +class DaemonTools: + def __init__(self, daemon): + self.daemon = daemon + + def announceNode(self): + '''Announce our node to our peers''' + peer = self.daemon.pickOnlinePeer() + self.daemon.peerAction(peer, 'announce', self.daemon._core.hsAddress) + self.daemon.decrementThreadCount('announceNode') \ No newline at end of file From bc95d8855d372203362e1191435b7b3cc570c9b2 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Wed, 8 Aug 2018 14:26:02 -0500 Subject: [PATCH 06/53] + added POW announce for node (now POST) * fixed bug where core hsAddress was not available on first startup --- onionr/api.py | 52 ++++++++++++++++++++++++++++++------- onionr/communicator2.py | 17 ++++-------- onionr/core.py | 6 +++++ onionr/onionr.py | 2 +- onionr/onionrdaemontools.py | 35 ++++++++++++++++++++++--- 5 files changed, 86 insertions(+), 26 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index d41245d1..1f68a49d 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -404,6 +404,46 @@ class API: resp = Response(resp) return resp + + @app.route('/public/announce/', methods=['POST']) + def acceptAnnounce(): + self.validateHost('public') + resp = 'failure' + powHash = '' + randomData = '' + newNode = '' + ourAdder = self._core.hsAddress.encode() + try: + newNode = request.form['node'].encode() + except KeyError: + logger.warn('No block specified for upload') + pass + else: + try: + randomData = request.form['random'] + randomData = base64.b64decode(randomData) + except KeyError: + logger.warn('No random data specified for upload') + else: + nodes = newNode + self._core.hsAddress.encode() + nodes = self._core._crypto.blake2bHash(nodes) + powHash = self._core._crypto.blake2bHash(randomData + nodes) + try: + powHash = powHash.decode() + except AttributeError: + pass + if powHash.startswith('0000'): + try: + newNode = newNode.decode() + except AttributeError: + pass + if self._core.addAddress(newNode): + resp = 'Success' + else: + logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash) + resp = Response(resp) + return resp + @app.route('/public/') def public_handler(): # Public means it is publicly network accessible @@ -428,15 +468,6 @@ class API: resp = Response(self._utils.getBlockDBHash()) elif action == 'getBlockHashes': resp = Response('\n'.join(self._core.getBlockList())) - elif action == 'announce': - if data != '': - # TODO: require POW for this - if self._core.addAddress(data): - resp = Response('Success') - else: - resp = Response('') - else: - resp = Response('') # setData should be something the communicator initiates, not this api elif action == 'getData': resp = '' @@ -488,6 +519,9 @@ class API: logger.info('Starting client on ' + self.host + ':' + str(bindPort) + '...', timestamp=False) try: + while len(self._core.hsAddress) == 0: + self._core.refreshFirstStartVars() + time.sleep(0.5) self.http_server = WSGIServer((self.host, bindPort), app) self.http_server.serve_forever() except KeyboardInterrupt: diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 6a37a376..b3af1d0f 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -93,7 +93,7 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58) OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True) OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True) - announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 305, requiresPeer=True) + announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 305, requiresPeer=True, maxThreads=1) cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requiresPeer=True) # set loop to execute instantly to load up peer pool (replaced old pool init wait) @@ -446,17 +446,10 @@ class OnionrCommunicatorDaemon: def announce(self, peer): '''Announce to peers our address''' - announceCount = 0 - announceAmount = 2 - for peer in self.onlinePeers: - announceCount += 1 - if self.peerAction(peer, 'announce', self._core.hsAddress) == 'Success': - logger.info('Successfully introduced node to ' + peer) - break - else: - if announceCount == announceAmount: - logger.warn('Could not introduce node. Try again soon') - break + if self.daemonTools.announceNode(): + logger.info('Successfully introduced node to ' + peer) + else: + logger.warn('Could not introduce node.') def detectAPICrash(self): '''exit if the api server crashes/stops''' diff --git a/onionr/core.py b/onionr/core.py index 64ad6439..b3857393 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -78,6 +78,12 @@ class Core: sys.exit(1) return + def refreshFirstStartVars(self): + '''Hack to refresh some vars which may not be set on first start''' + if os.path.exists('data/hs/hostname'): + with open('data/hs/hostname', 'r') as hs: + self.hsAddress = hs.read().strip() + def addPeer(self, peerID, powID, name=''): ''' Adds a public key to the key database (misleading function name) diff --git a/onionr/onionr.py b/onionr/onionr.py index 0de17850..c4c4529d 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -40,7 +40,7 @@ except ImportError: raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)") ONIONR_TAGLINE = 'Anonymous P2P Platform - GPLv3 - https://Onionr.VoidNet.Tech' -ONIONR_VERSION = '0.1.1' # for debugging and stuff +ONIONR_VERSION = '0.2.0' # for debugging and stuff ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION) API_VERSION = '4' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes know how to communicate without learning too much information about you. diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index e615aacb..50679184 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -17,13 +17,40 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import onionrexceptions, onionrpeers +import onionrexceptions, onionrpeers, onionrproofs, base64 class DaemonTools: def __init__(self, daemon): self.daemon = daemon + self.announceCache = {} def announceNode(self): '''Announce our node to our peers''' - peer = self.daemon.pickOnlinePeer() - self.daemon.peerAction(peer, 'announce', self.daemon._core.hsAddress) - self.daemon.decrementThreadCount('announceNode') \ No newline at end of file + retData = False + + # Announce to random online peers + for i in self.daemon.onlinePeers: + if not i in self.announceCache: + peer = i + break + else: + peer = self.daemon.pickOnlinePeer() + + ourID = self.daemon._core.hsAddress.strip() + + url = 'http://' + peer + '/public/announce/' + data = {'node': ourID} + + combinedNodes = ourID + peer + + if peer in self.announceCache: + data['random'] = self.announceCache[peer] + else: + proof = onionrproofs.DataPOW(combinedNodes, forceDifficulty=4) + data['random'] = base64.b64encode(proof.waitForResult()[1]) + self.announceCache[peer] = data['random'] + + logger.info('Announcing node to ' + url) + if self.daemon._core._utils.doPostRequest(url, data) == 'Success': + retData = True + self.daemon.decrementThreadCount('announceNode') + return retData \ No newline at end of file From be77e0283564818b59493561070a2c543bac8ec8 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Wed, 8 Aug 2018 20:48:31 -0500 Subject: [PATCH 07/53] added missing logger import --- onionr/onionrdaemontools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index 50679184..8410cb80 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import onionrexceptions, onionrpeers, onionrproofs, base64 +import onionrexceptions, onionrpeers, onionrproofs, base64, logger class DaemonTools: def __init__(self, daemon): self.daemon = daemon From 1e3768416305c3136154a60744681202e7b23648 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 10 Aug 2018 02:03:49 -0500 Subject: [PATCH 08/53] work on blacklisting data --- onionr/onionrblacklist.py | 62 +++++++++++++++++++++++++++++++++++++++ onionr/onionrutils.py | 13 ++++++++ 2 files changed, 75 insertions(+) create mode 100644 onionr/onionrblacklist.py diff --git a/onionr/onionrblacklist.py b/onionr/onionrblacklist.py new file mode 100644 index 00000000..7261cf4f --- /dev/null +++ b/onionr/onionrblacklist.py @@ -0,0 +1,62 @@ +''' + Onionr - P2P Microblogging Platform & Social network. + + This file handles maintenence of a blacklist database, for blocks and peers +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import sqlite3, os +class OnionrBlackList: + def __init__(self, coreInst): + self.blacklistDB = 'data/blacklist.db' + self._core = coreInst + + if not os.path.exists(self.blacklistDB): + self.generateDB() + + return + + def inBlacklist(self, data): + return + + def _dbExecute(self, toExec): + conn = sqlite3.connect(self.blacklistDB) + c = conn.cursor() + c.execute(toExec) + conn.commit() + conn.close() + + def deleteBeforeDate(self, date): + # TODO, delete blacklist entries before date + return + + def generateDB(self): + self._dbExecute'''CREATE TABLE blacklist( + hash text primary key not null, + type text + ); + ''') + return + + def clearDB(self): + self._dbExecute('''delete from blacklist;);''') + + + def addToDB(self, data): + hashed = self._core._crypto.sha3Hash(data) + if not hashed.isalnum(): + raise Exception("Hashed data is not alpha numeric") + insert = (hashed,) + self._dbExecute('insert into blacklist (hash) VALUES(' + data + ');') diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index afa86ef6..e25e692c 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -597,6 +597,19 @@ class OnionrUtils: else: self.powSalt = retData return retData + + def strToBytes(self, data): + try: + data = data.encode() + except AttributeError: + pass + return data + def bytesToStr(self, data): + try: + data = data.decode() + except AttributeError: + pass + return data def size(path='.'): ''' From 12d39393b4670fbbb463df863ad48d509a343ffa Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 10 Aug 2018 17:13:58 -0500 Subject: [PATCH 09/53] * cache pip requirements in dockerfile * import blacklist into core + began work on blacklist command * work on blacklist module * modified dockerignore --- .dockerignore | 2 ++ Dockerfile | 15 ++++++++------- onionr/api.py | 1 - onionr/core.py | 2 ++ onionr/onionr.py | 15 +++++++++++++++ onionr/onionrblacklist.py | 30 +++++++++++++++++++++++------- 6 files changed, 50 insertions(+), 15 deletions(-) diff --git a/.dockerignore b/.dockerignore index d76a75c1..4a0c253e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,4 @@ onionr/data/**/* onionr/data +RUN-WINDOWS.bat +MY-RUN.sh diff --git a/Dockerfile b/Dockerfile index 7331dd6f..7b7feb20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,16 +6,17 @@ ENV HOME /root #Install needed packages RUN apt update && apt install -y python3 python3-dev python3-pip tor -#Add Onionr source -COPY . /root -VOLUME /root/data - -WORKDIR /root - +WORKDIR /srv/ +ADD ./requirements.txt /srv/requirements.txt RUN pip3 install -r requirements.txt + +WORKDIR /root/ +#Add Onionr source +COPY . /root/ +VOLUME /root/data/ + #Set upstart command -#CMD (! ${ENABLE_TOR} || tor&) && python zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552 CMD bash #Expose ports diff --git a/onionr/api.py b/onionr/api.py index 1f68a49d..90a3b83e 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -506,7 +506,6 @@ class API: def authFail(err): self.requestFailed = True resp = Response("403") - return resp @app.errorhandler(401) diff --git a/onionr/core.py b/onionr/core.py index b3857393..76f55652 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -21,6 +21,7 @@ import sqlite3, os, sys, time, math, base64, tarfile, getpass, simplecrypt, hash from onionrblockapi import Block import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues +import onionrblacklist if sys.version_info < (3, 6): try: @@ -71,6 +72,7 @@ class Core: self._utils = onionrutils.OnionrUtils(self) # Initialize the crypto object self._crypto = onionrcrypto.OnionrCrypto(self) + self._blacklist = onionrblacklist.OnionrBlackList(self) except Exception as error: logger.error('Failed to initialize core Onionr library.', error=error) diff --git a/onionr/onionr.py b/onionr/onionr.py index c4c4529d..c6357089 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -186,6 +186,8 @@ class Onionr: 'addaddress': self.addAddress, 'list-peers': self.listPeers, + 'blacklist-block': self.banBlock, + 'add-file': self.addFile, 'addfile': self.addFile, 'listconn': self.listConn, @@ -258,6 +260,19 @@ class Onionr: def getCommands(self): return self.cmds + def banBlock(self): + try: + ban = sys.argv[2] + except IndexError: + while True: + ban = logger.readline('Enter a block hash:') + if self.onionrUtils.validateHash(ban): + if not self.onionrCore._blacklist.inBlacklist(ban): + self.onionrCore._blacklist.addToDB(ban) + + return + + def listConn(self): self.onionrCore.daemonQueueAdd('connectedPeers') diff --git a/onionr/onionrblacklist.py b/onionr/onionrblacklist.py index 7261cf4f..52d0957b 100644 --- a/onionr/onionrblacklist.py +++ b/onionr/onionrblacklist.py @@ -29,21 +29,29 @@ class OnionrBlackList: return def inBlacklist(self, data): - return + hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data)) + retData = False + if not hashed.isalnum(): + raise Exception("Hashed data is not alpha numeric") + + for i in self._dbExecute("select * from blacklist where hash='%s'" % (hashed,)): + retData = True # this only executes if an entry is present by that hash + break + return retData def _dbExecute(self, toExec): conn = sqlite3.connect(self.blacklistDB) c = conn.cursor() - c.execute(toExec) + retData = c.execute(toExec) conn.commit() - conn.close() + return retData def deleteBeforeDate(self, date): # TODO, delete blacklist entries before date return def generateDB(self): - self._dbExecute'''CREATE TABLE blacklist( + self._dbExecute('''CREATE TABLE blacklist( hash text primary key not null, type text ); @@ -53,10 +61,18 @@ class OnionrBlackList: def clearDB(self): self._dbExecute('''delete from blacklist;);''') - + def getList(self): + data = self._dbExecute('select * from blacklist') + myList = [] + for i in data: + myList.append(i[0]) + return myList + def addToDB(self, data): - hashed = self._core._crypto.sha3Hash(data) + '''Add to the blacklist. Intended to be block hash, block data, peers, or transport addresses''' + # we hash the data so we can remove data entirely from our node's disk + hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data)) if not hashed.isalnum(): raise Exception("Hashed data is not alpha numeric") insert = (hashed,) - self._dbExecute('insert into blacklist (hash) VALUES(' + data + ');') + self._dbExecute("insert into blacklist (hash) VALUES('%s');" % (hashed,)) From 5f21d15cdd44283adf37459bf46b26ff6a3eb519 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 11 Aug 2018 00:23:59 -0500 Subject: [PATCH 10/53] + blocks can now be blacklisted * Peers sync a little better --- onionr/communicator2.py | 3 ++- onionr/core.py | 2 +- onionr/onionr.py | 22 +++++++++++++++------- onionr/onionrblacklist.py | 1 - 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index b3af1d0f..53edda76 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -173,7 +173,7 @@ class OnionrCommunicatorDaemon: # if newline seperated string is valid hash if not i in existingBlocks: # if block does not exist on disk and is not already in block queue - if i not in self.blockQueue: + if i not in self.blockQueue and not self._core._blacklist.inBlacklist(i): self.blockQueue.append(i) self.decrementThreadCount('lookupBlocks') return @@ -289,6 +289,7 @@ class OnionrCommunicatorDaemon: for i in self._core.bootstrapList: if i not in peerList and i not in self.offlinePeers and i != self._core.hsAddress: peerList.append(i) + self._core.addAddress(i) def connectNewPeer(self, peer='', useBootstrap=False): '''Adds a new random online peer to self.onlinePeers''' diff --git a/onionr/core.py b/onionr/core.py index 76f55652..440f31e0 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -172,7 +172,7 @@ class Core: def removeBlock(self, block): ''' - remove a block from this node + remove a block from this node (does not automatically blacklist) ''' if self._utils.validateHash(block): conn = sqlite3.connect(self.blockDB) diff --git a/onionr/onionr.py b/onionr/onionr.py index c6357089..1736c3f9 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -232,6 +232,7 @@ class Onionr: 'listconn': 'list connected peers', 'kex': 'exchange keys with peers (done automatically)', 'pex': 'exchange addresses with peers (done automatically)', + 'blacklist-block': 'deletes a block by hash and permanently removes it from your node', 'introduce': 'Introduce your node to the public Onionr network', } @@ -264,15 +265,22 @@ class Onionr: try: ban = sys.argv[2] except IndexError: - while True: - ban = logger.readline('Enter a block hash:') - if self.onionrUtils.validateHash(ban): - if not self.onionrCore._blacklist.inBlacklist(ban): - self.onionrCore._blacklist.addToDB(ban) - + ban = logger.readline('Enter a block hash:') + if self.onionrUtils.validateHash(ban): + if not self.onionrCore._blacklist.inBlacklist(ban): + try: + self.onionrCore._blacklist.addToDB(ban) + self.onionrCore.removeBlock(ban) + except Exception as error: + logger.error('Could not blacklist block', error=error) + else: + logger.info('Block blacklisted') + else: + logger.warn('That block is already blacklisted') + else: + logger.error('Invalid block hash') return - def listConn(self): self.onionrCore.daemonQueueAdd('connectedPeers') diff --git a/onionr/onionrblacklist.py b/onionr/onionrblacklist.py index 52d0957b..056aafb6 100644 --- a/onionr/onionrblacklist.py +++ b/onionr/onionrblacklist.py @@ -25,7 +25,6 @@ class OnionrBlackList: if not os.path.exists(self.blacklistDB): self.generateDB() - return def inBlacklist(self, data): From aab7d4296fdbc356f0da59a603c61465f8ca84a3 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 12 Aug 2018 22:48:33 -0500 Subject: [PATCH 11/53] improved blacklisting --- onionr/api.py | 16 +++++++++------- onionr/blockimporter.py | 6 ++++++ onionr/core.py | 19 +++++++++++++------ onionr/onionrexceptions.py | 3 +++ onionr/onionrpeers.py | 2 +- 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index 90a3b83e..34c320d9 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -24,7 +24,7 @@ from gevent.wsgi import WSGIServer import sys, random, threading, hmac, hashlib, base64, time, math, os, json from core import Core from onionrblockapi import Block -import onionrutils, onionrcrypto, blockimporter, onionrevents as events, logger, config +import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config class API: ''' @@ -119,9 +119,7 @@ class API: ''' Simply define the request as not having yet failed, before every request. ''' - self.requestFailed = False - return @app.after_request @@ -397,10 +395,14 @@ class API: pass else: if sys.getsizeof(data) < 100000000: - if blockimporter.importBlockFromData(data, self._core): - resp = 'success' - else: - logger.warn('Error encountered importing uploaded block') + try: + if blockimporter.importBlockFromData(data, self._core): + resp = 'success' + else: + logger.warn('Error encountered importing uploaded block') + except onionrexceptions.BlacklistedBlock: + logger.debug('uploaded block is blacklisted') + pass resp = Response(resp) return resp diff --git a/onionr/blockimporter.py b/onionr/blockimporter.py index a2695093..f5698d00 100644 --- a/onionr/blockimporter.py +++ b/onionr/blockimporter.py @@ -20,6 +20,12 @@ import core, onionrexceptions, logger def importBlockFromData(content, coreInst): retData = False + + dataHash = coreInst._getSha3Hash(content) + + if coreInst._blacklist.inBlacklist(dataHash): + raise onionrexceptions.BlacklistedBlock('%s is a blacklisted block' % (dataHash,)) + if not isinstance(coreInst, core.Core): raise Exception("coreInst must be an Onionr core instance") diff --git a/onionr/core.py b/onionr/core.py index 440f31e0..8f517e82 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -133,7 +133,6 @@ class Core: for i in c.execute("SELECT * FROM adders where address = '" + address + "';"): try: if i[0] == address: - logger.warn('Not adding existing address') conn.close() return False except ValueError: @@ -311,16 +310,24 @@ class Core: return data - def setData(self, data): - ''' - Set the data assciated with a hash - ''' - data = data + def _getSha3Hash(self, data): hasher = hashlib.sha3_256() if not type(data) is bytes: data = data.encode() hasher.update(data) dataHash = hasher.hexdigest() + return dataHash + + def setData(self, data): + ''' + Set the data assciated with a hash + ''' + data = data + if not type(data) is bytes: + data = data.encode() + + dataHash = self._getSha3Hash(data) + if type(dataHash) is bytes: dataHash = dataHash.decode() blockFileName = self.blockDataLocation + dataHash + '.dat' diff --git a/onionr/onionrexceptions.py b/onionr/onionrexceptions.py index 2817d419..7b0de6c7 100644 --- a/onionr/onionrexceptions.py +++ b/onionr/onionrexceptions.py @@ -38,6 +38,9 @@ class InvalidPubkey(Exception): class InvalidMetadata(Exception): pass +class BlacklistedBlock(Exception): + pass + class InvalidHexHash(Exception): '''When a string is not a valid hex string of appropriate length for a hash value''' pass diff --git a/onionr/onionrpeers.py b/onionr/onionrpeers.py index 21697033..2762277c 100644 --- a/onionr/onionrpeers.py +++ b/onionr/onionrpeers.py @@ -72,7 +72,7 @@ def getScoreSortedPeerList(coreInst): return peerList def peerCleanup(coreInst): - '''Removes peers who have been offline too long''' + '''Removes peers who have been offline too long or score too low''' if not type(coreInst is core.Core): raise TypeError('coreInst must be instance of core.Core') From a4c86630cf87991c9d7515c0968fcc07e1d49e2b Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 13 Aug 2018 15:07:14 -0500 Subject: [PATCH 12/53] reduced peer score minimum, blacklist peers --- onionr/onionrpeers.py | 1 + onionr/static-data/default_config.json | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/onionr/onionrpeers.py b/onionr/onionrpeers.py index 2762277c..a146a17c 100644 --- a/onionr/onionrpeers.py +++ b/onionr/onionrpeers.py @@ -89,4 +89,5 @@ def peerCleanup(coreInst): # Remove peers that go below the negative score if PeerProfiles(address, coreInst).score < minScore: coreInst.removeAddress(address) + coreInst._blacklist.addToDB(address) logger.warn('Removed address ' + address + '.') \ No newline at end of file diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 81ba610f..c298c137 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -53,11 +53,11 @@ "allocations":{ "disk": 9000000000, "netTotal": 1000000000, - "blockCache" : 5000000, - "blockCacheTotal" : 50000000 + "blockCache": 5000000, + "blockCacheTotal": 50000000 }, "peers":{ - "minimumScore": -4000, + "minimumScore": -1000, "maxStoredPeers": 100, "maxConnect": 3 } From 9f4024104abed48ce1b64a42b8ece271d643380b Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 14 Aug 2018 00:02:34 -0500 Subject: [PATCH 13/53] work on peer blacklisting/profiling\nupdated readme --- onionr/onionrblacklist.py | 17 ++++++++++++++--- onionr/onionrutils.py | 3 ++- readme.md | 34 +++++++++++++++++++++------------- 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/onionr/onionrblacklist.py b/onionr/onionrblacklist.py index 056aafb6..9affb377 100644 --- a/onionr/onionrblacklist.py +++ b/onionr/onionrblacklist.py @@ -52,7 +52,9 @@ class OnionrBlackList: def generateDB(self): self._dbExecute('''CREATE TABLE blacklist( hash text primary key not null, - type text + dataType text, + blacklistDate int, + expire int ); ''') return @@ -67,11 +69,20 @@ class OnionrBlackList: myList.append(i[0]) return myList - def addToDB(self, data): + def addToDB(self, data, dataType=0, expire=0): '''Add to the blacklist. Intended to be block hash, block data, peers, or transport addresses''' # we hash the data so we can remove data entirely from our node's disk hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data)) if not hashed.isalnum(): raise Exception("Hashed data is not alpha numeric") + try: + int(dataType) + except ValueError: + raise Exception("dataType is not int") + try: + int(expire) + except ValueError: + raise Exception("expire is not int") + #TODO check for length sanity insert = (hashed,) - self._dbExecute("insert into blacklist (hash) VALUES('%s');" % (hashed,)) + self._dbExecute("insert into blacklist (hash, dataType, expire) VALUES('%s', %s, %s);" % (hashed, dataType, expire)) diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index e25e692c..1f2111a6 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -126,7 +126,8 @@ class OnionrUtils: retVal = False if newAdderList != False: for adder in newAdderList.split(','): - if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress(): + adder = adder.strip() + if not adder in self._core.listAdders(randomOrder = False) and adder != self.getMyAddress() and not self._core._blacklist.inBlacklist(adder): if self._core.addAddress(adder): logger.info('Added %s to db.' % adder, timestamp = True) retVal = True diff --git a/readme.md b/readme.md index 926d3276..7486bcac 100644 --- a/readme.md +++ b/readme.md @@ -5,29 +5,39 @@ Anonymous P2P platform, using Tor & I2P. -Major work in progress. +***Experimental, not safe or easy to use yet*** -***THIS SOFTWARE IS NOT USABLE OR SECURE YET.*** +
**The main repo for this software is at https://gitlab.com/beardog/Onionr/** -**Roadmap/features:** + +# Summary + +Onionr is a decentralized, peer-to-peer data storage network, designed to be anonymous and resistant to (meta)data analysis and spam. + +Onionr can be used for mail, as a social network, instant messenger, file sharing software, or for encrypted group discussion. + +# Roadmap/features Check the [Gitlab Project](https://gitlab.com/beardog/Onionr/milestones/1) to see progress towards the alpha release. +## Core internal features + * [X] Fully p2p/decentralized, no trackers or other single points of failure -* [X] High level of anonymity -* [ ] End to end encryption where applicable +* [X] End to end encryption of user data * [X] Optional non-encrypted blocks, useful for blog posts or public file sharing -* [ ] Easy API system for integration to websites +* [X] Easy API system for integration to websites +* [ ] Metadata analysis resistance (being improved) -# Development -This software is in heavy development. If for some reason you want to get involved, get in touch first. +## Other features -**Onionr API and functionality is subject to non-backwards compatible change during development** +**Onionr API and functionality is subject to non-backwards compatible change during pre-alpha development** -# Donate +## Help out + +Everyone is welcome to help out. Please get in touch first if you are making non-trivial changes. If you can't help with programming, you can write documentation or guides. Bitcoin/Bitcoin Cash: 1onion55FXzm6h8KQw3zFw2igpHcV7LPq @@ -35,6 +45,4 @@ Bitcoin/Bitcoin Cash: 1onion55FXzm6h8KQw3zFw2igpHcV7LPq The Tor Project, I2P developers, and anyone else do not own, create, or endorse this project, and are not otherwise involved. -The badges (besides travis-ci build) are by Maik Ellerbrock is licensed under a Creative Commons Attribution 4.0 International License. - -The onion in the Onionr logo is adapted from [this](https://commons.wikimedia.org/wiki/File:Red_Onion_on_White.JPG) image by Colin on Wikimedia under a Creative Commons Attribution-Share Alike 3.0 Unported license. The Onionr logo is under the same license. +The badges (besides travis-ci build) are by Maik Ellerbrock is licensed under a Creative Commons Attribution 4.0 International License. \ No newline at end of file From ba2e5d7da9a8598432d22ae676cc3ead53db4cda Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 14 Aug 2018 23:20:08 -0500 Subject: [PATCH 14/53] ability to create expiring bans --- onionr/onionrblacklist.py | 31 +++++++++++++++++++++++++++---- onionr/onionrpeers.py | 7 +++++-- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/onionr/onionrblacklist.py b/onionr/onionrblacklist.py index 9affb377..bb990cce 100644 --- a/onionr/onionrblacklist.py +++ b/onionr/onionrblacklist.py @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import sqlite3, os +import sqlite3, os, logger class OnionrBlackList: def __init__(self, coreInst): self.blacklistDB = 'data/blacklist.db' @@ -48,11 +48,29 @@ class OnionrBlackList: def deleteBeforeDate(self, date): # TODO, delete blacklist entries before date return + + def deleteExpired(self, dataType=0): + '''Delete expired entries''' + deleteList = [] + curTime = self._core._utils.getEpoch() + + try: + int(dataType) + except AttributeError: + raise TypeError("dataType must be int") + + for i in self._dbExecute('select * from blacklist where dataType=%s' % (dataType,)): + if i[1] == dataType: + if (curTime - i[2]) >= i[3]: + deleteList.append(i[0]) + + for thing in deleteList: + self._dbExecute("delete from blacklist where hash='%s'" % (thing,)) def generateDB(self): self._dbExecute('''CREATE TABLE blacklist( hash text primary key not null, - dataType text, + dataType int, blacklistDate int, expire int ); @@ -70,7 +88,11 @@ class OnionrBlackList: return myList def addToDB(self, data, dataType=0, expire=0): - '''Add to the blacklist. Intended to be block hash, block data, peers, or transport addresses''' + '''Add to the blacklist. Intended to be block hash, block data, peers, or transport addresses + 0=block + 1=peer + 2=pubkey + ''' # we hash the data so we can remove data entirely from our node's disk hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data)) if not hashed.isalnum(): @@ -85,4 +107,5 @@ class OnionrBlackList: raise Exception("expire is not int") #TODO check for length sanity insert = (hashed,) - self._dbExecute("insert into blacklist (hash, dataType, expire) VALUES('%s', %s, %s);" % (hashed, dataType, expire)) + blacklistDate = self._core._utils.getEpoch() + self._dbExecute("insert into blacklist (hash, dataType, blacklistDate, expire) VALUES('%s', %s, %s, %s);" % (hashed, dataType, blacklistDate, expire)) diff --git a/onionr/onionrpeers.py b/onionr/onionrpeers.py index a146a17c..cd0745ea 100644 --- a/onionr/onionrpeers.py +++ b/onionr/onionrpeers.py @@ -89,5 +89,8 @@ def peerCleanup(coreInst): # Remove peers that go below the negative score if PeerProfiles(address, coreInst).score < minScore: coreInst.removeAddress(address) - coreInst._blacklist.addToDB(address) - logger.warn('Removed address ' + address + '.') \ No newline at end of file + coreInst._blacklist.addToDB(address, dataType=1, expire=300) + logger.warn('Removed address ' + address + '.') + + # Unban probably not malicious peers TODO improve + coreInst._blacklist.deleteExpired(dataType=1) \ No newline at end of file From 06dc97869e37508718d43a9892c8cdefb656d867 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 16 Aug 2018 00:01:40 -0500 Subject: [PATCH 15/53] working on preventing replay attacks with block content --- onionr/blockimporter.py | 2 +- onionr/communicator2.py | 2 +- onionr/core.py | 86 ++++++-------------------------------- onionr/onionrexceptions.py | 3 ++ onionr/onionrutils.py | 21 +++++++++- 5 files changed, 37 insertions(+), 77 deletions(-) diff --git a/onionr/blockimporter.py b/onionr/blockimporter.py index f5698d00..1b37c3cd 100644 --- a/onionr/blockimporter.py +++ b/onionr/blockimporter.py @@ -36,7 +36,7 @@ def importBlockFromData(content, coreInst): metas = coreInst._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata metadata = metas[0] - if coreInst._utils.validateMetadata(metadata): # check if metadata is valid + if coreInst._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid if coreInst._crypto.verifyPow(content): # check if POW is enough/correct logger.info('Block passed proof, saving.') blockHash = coreInst.setData(content) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 53edda76..019dedb8 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -206,7 +206,7 @@ class OnionrCommunicatorDaemon: metas = self._core._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata metadata = metas[0] #meta = metas[1] - if self._core._utils.validateMetadata(metadata): # check if metadata is valid + if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce if self._core._crypto.verifyPow(content): # check if POW is enough/correct logger.info('Block passed proof, saving.') self._core.setData(content) diff --git a/onionr/core.py b/onionr/core.py index 8f517e82..f3834ed3 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -22,7 +22,7 @@ from onionrblockapi import Block import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues import onionrblacklist - +import dbcreator if sys.version_info < (3, 6): try: import sha3 @@ -46,6 +46,8 @@ class Core: self.bootstrapList = [] self.requirements = onionrvalues.OnionrValues() self.torPort = torPort + self.dataNonceFile = 'data/block-nonces.dat' + self.dbCreate = dbcreator.DBCreator(self) self.usageFile = 'data/disk-usage.txt' @@ -188,89 +190,20 @@ class Core: def createAddressDB(self): ''' Generate the address database - - types: - 1: I2P b32 address - 2: Tor v2 (like facebookcorewwwi.onion) - 3: Tor v3 ''' - conn = sqlite3.connect(self.addressDB) - c = conn.cursor() - c.execute('''CREATE TABLE adders( - address text, - type int, - knownPeer text, - speed int, - success int, - DBHash text, - powValue text, - failure int, - lastConnect int, - lastConnectAttempt int, - trust int - ); - ''') - conn.commit() - conn.close() + self.dbCreate.createAddressDB() def createPeerDB(self): ''' Generate the peer sqlite3 database and populate it with the peers table. ''' - # generate the peer database - conn = sqlite3.connect(self.peerDB) - c = conn.cursor() - c.execute('''CREATE TABLE peers( - ID text not null, - name text, - adders text, - blockDBHash text, - forwardKey text, - dateSeen not null, - bytesStored int, - trust int, - pubkeyExchanged int, - hashID text, - pow text not null); - ''') - conn.commit() - conn.close() - return + self.dbCreate.createPeerDB() def createBlockDB(self): ''' Create a database for blocks - - hash - the hash of a block - dateReceived - the date the block was recieved, not necessarily when it was created - decrypted - if we can successfully decrypt the block (does not describe its current state) - dataType - data type of the block - dataFound - if the data has been found for the block - dataSaved - if the data has been saved for the block - sig - optional signature by the author (not optional if author is specified) - author - multi-round partial sha3-256 hash of authors public key - dateClaimed - timestamp claimed inside the block, only as trustworthy as the block author is ''' - if os.path.exists(self.blockDB): - raise Exception("Block database already exists") - conn = sqlite3.connect(self.blockDB) - c = conn.cursor() - c.execute('''CREATE TABLE hashes( - hash text not null, - dateReceived int, - decrypted int, - dataType text, - dataFound int, - dataSaved int, - sig text, - author text, - dateClaimed int - ); - ''') - conn.commit() - conn.close() - - return + self.dbCreate.createBlockDB() def addToBlockDB(self, newHash, selfInsert=False, dataSaved=False): ''' @@ -702,6 +635,7 @@ class Core: signature = '' signer = '' metadata = {} + # metadata is full block metadata, meta is internal, user specified metadata # only use header if not set in provided meta if not header is None: @@ -749,6 +683,12 @@ class Core: metadata['sig'] = signature metadata['signer'] = signer metadata['time'] = str(self._utils.getEpoch()) + + nonce = self._utils.bytesToStr(self._crypto.sha3Hash(data)) + + # TODO check in advance + with open(self.dataNonceFile, 'a') as nonceFile: + nonceFile.write(nonce + '\n') # send block data (and metadata) to POW module to get tokenized block data proof = onionrproofs.POW(metadata, data) diff --git a/onionr/onionrexceptions.py b/onionr/onionrexceptions.py index 7b0de6c7..b26a97d7 100644 --- a/onionr/onionrexceptions.py +++ b/onionr/onionrexceptions.py @@ -41,6 +41,9 @@ class InvalidMetadata(Exception): class BlacklistedBlock(Exception): pass +class DataExists(Exception): + pass + class InvalidHexHash(Exception): '''When a string is not a valid hex string of appropriate length for a hash value''' pass diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 1f2111a6..331c8e8b 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -334,7 +334,7 @@ class OnionrUtils: return retVal - def validateMetadata(self, metadata): + def validateMetadata(self, metadata, blockData): '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string''' # TODO, make this check sane sizes retData = False @@ -364,7 +364,24 @@ class OnionrUtils: break else: # if metadata loop gets no errors, it does not break, therefore metadata is valid - retData = True + # make sure we do not have another block with the same data content (prevent data duplication and replay attacks) + try: + with open(self._core.dataNonceFile, 'r') as nonceFile: + nonce = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(blockData)) + if nonce in nonceFile.read(): + retData = False # we've seen that nonce before, so we can't pass metadata + raise onionrexceptions.DataExists + except FileNotFoundError: + retData = True + except onionrexceptions.DataExists: + # do not set retData to True, because nonce has been seen before + pass + else: + retData = True + if retData: + # Executes if data not seen + with open(self._core.dataNonceFile, 'a') as nonceFile: + nonceFile.write(nonce + '\n') else: logger.warn('In call to utils.validateMetadata, metadata must be JSON string or a dictionary object') From cb90c24e647795abaaf75be87a9d70f13d7d7603 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 16 Aug 2018 09:03:59 -0500 Subject: [PATCH 16/53] added dbcreator --- onionr/dbcreator.py | 109 ++++++++++++++++++++++++++++++++++++++++++ onionr/onionrutils.py | 2 +- 2 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 onionr/dbcreator.py diff --git a/onionr/dbcreator.py b/onionr/dbcreator.py new file mode 100644 index 00000000..19a9a7bd --- /dev/null +++ b/onionr/dbcreator.py @@ -0,0 +1,109 @@ +''' + Onionr - P2P Anonymous Data Storage & Sharing + + DBCreator, creates sqlite3 databases used by Onionr +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import sqlite3, os +class DBCreator: + def __init__(self, coreInst): + self.core = coreInst + + def createAddressDB(self): + ''' + Generate the address database + + types: + 1: I2P b32 address + 2: Tor v2 (like facebookcorewwwi.onion) + 3: Tor v3 + ''' + conn = sqlite3.connect(self.core.addressDB) + c = conn.cursor() + c.execute('''CREATE TABLE adders( + address text, + type int, + knownPeer text, + speed int, + success int, + DBHash text, + powValue text, + failure int, + lastConnect int, + lastConnectAttempt int, + trust int + ); + ''') + conn.commit() + conn.close() + + def createPeerDB(self): + ''' + Generate the peer sqlite3 database and populate it with the peers table. + ''' + # generate the peer database + conn = sqlite3.connect(self.core.peerDB) + c = conn.cursor() + c.execute('''CREATE TABLE peers( + ID text not null, + name text, + adders text, + blockDBHash text, + forwardKey text, + dateSeen not null, + bytesStored int, + trust int, + pubkeyExchanged int, + hashID text, + pow text not null); + ''') + conn.commit() + conn.close() + return + + def createBlockDB(self): + ''' + Create a database for blocks + + hash - the hash of a block + dateReceived - the date the block was recieved, not necessarily when it was created + decrypted - if we can successfully decrypt the block (does not describe its current state) + dataType - data type of the block + dataFound - if the data has been found for the block + dataSaved - if the data has been saved for the block + sig - optional signature by the author (not optional if author is specified) + author - multi-round partial sha3-256 hash of authors public key + dateClaimed - timestamp claimed inside the block, only as trustworthy as the block author is + ''' + if os.path.exists(self.core.blockDB): + raise Exception("Block database already exists") + conn = sqlite3.connect(self.core.blockDB) + c = conn.cursor() + c.execute('''CREATE TABLE hashes( + hash text not null, + dateReceived int, + decrypted int, + dataType text, + dataFound int, + dataSaved int, + sig text, + author text, + dateClaimed int + ); + ''') + conn.commit() + conn.close() + return \ No newline at end of file diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 331c8e8b..7e0abd94 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -365,9 +365,9 @@ class OnionrUtils: else: # if metadata loop gets no errors, it does not break, therefore metadata is valid # make sure we do not have another block with the same data content (prevent data duplication and replay attacks) + nonce = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(blockData)) try: with open(self._core.dataNonceFile, 'r') as nonceFile: - nonce = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(blockData)) if nonce in nonceFile.read(): retData = False # we've seen that nonce before, so we can't pass metadata raise onionrexceptions.DataExists From 5b20930d5c52e839f2b648c250413ba26b285c6a Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 16 Aug 2018 19:02:14 -0500 Subject: [PATCH 17/53] hopefully fixed block sync issue --- docs/whitepaper.md | 10 ++++++++-- onionr/api.py | 2 +- onionr/communicator2.py | 1 + 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/whitepaper.md b/docs/whitepaper.md index 789f8e47..e791b83c 100644 --- a/docs/whitepaper.md +++ b/docs/whitepaper.md @@ -86,6 +86,12 @@ Blocks are stored indefinitely until the allocated space is filled, at which poi ## Block Timestamping -Onionr can provide evidence when a block was inserted by requesting other users to sign a hash of the current time with the block data hash: sha3_256(time + sha3_256(block data)). +Onionr can provide evidence of when a block was inserted by requesting other users to sign a hash of the current time with the block data hash: sha3_256(time + sha3_256(block data)). -This can be done either by the creator of the block prior to generation, or by any node after insertion. \ No newline at end of file +This can be done either by the creator of the block prior to generation, or by any node after insertion. + +In addition, randomness beacons such as the one operated by [NIST](https://beacon.nist.gov/home) or the hash of the latest blocks in a cryptocurrency network could be used to affirm that a block was at least not *created* before a given time. + +# Direct Connections + +We propose a system to \ No newline at end of file diff --git a/onionr/api.py b/onionr/api.py index 34c320d9..37c6b531 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -474,7 +474,7 @@ class API: elif action == 'getData': resp = '' if self._utils.validateHash(data): - if not os.path.exists('data/blocks/' + data + '.db'): + if os.path.exists('data/blocks/' + data + '.dat'): block = Block(hash=data.encode(), core=self._core) resp = base64.b64encode(block.getRaw().encode()).decode() if len(resp) == 0: diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 019dedb8..5a6aeee6 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -216,6 +216,7 @@ class OnionrCommunicatorDaemon: logger.warn('POW failed for block ' + blockHash) else: logger.warn('Metadata for ' + blockHash + ' is invalid.') + self._core._blacklist.addToDB(blockHash) else: # if block didn't meet expected hash tempHash = self._core._crypto.sha3Hash(content) # lazy hack, TODO use var From 91c1e223244439e6d130eabd232182d7d7f749e2 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 16 Aug 2018 22:30:36 -0500 Subject: [PATCH 18/53] do not duplicate blacklist entries --- onionr/communicator2.py | 7 +++++-- onionr/onionrblacklist.py | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 5a6aeee6..1f2166ac 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -215,8 +215,11 @@ class OnionrCommunicatorDaemon: else: logger.warn('POW failed for block ' + blockHash) else: - logger.warn('Metadata for ' + blockHash + ' is invalid.') - self._core._blacklist.addToDB(blockHash) + if self._core._blacklist.inBlacklist(realHash): + logger.warn('%s is blacklisted' % (realHash,)) + else: + logger.warn('Metadata for ' + blockHash + ' is invalid.') + self._core._blacklist.addToDB(blockHash) else: # if block didn't meet expected hash tempHash = self._core._crypto.sha3Hash(content) # lazy hack, TODO use var diff --git a/onionr/onionrblacklist.py b/onionr/onionrblacklist.py index bb990cce..86823283 100644 --- a/onionr/onionrblacklist.py +++ b/onionr/onionrblacklist.py @@ -95,6 +95,10 @@ class OnionrBlackList: ''' # we hash the data so we can remove data entirely from our node's disk hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data)) + + if self.inBlacklist(hashed): + return + if not hashed.isalnum(): raise Exception("Hashed data is not alpha numeric") try: From 40ea61707f39731fafd0cee501a1c8d0cdfead76 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 16 Aug 2018 23:21:21 -0500 Subject: [PATCH 19/53] do not try to download already saved blocks from upload --- Makefile | 2 +- onionr/communicator2.py | 4 ++++ onionr/static-data/default_config.json | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c51fc72b..b872b042 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ test: soft-reset: @echo "Soft-resetting Onionr..." - rm -f onionr/data/blocks/*.dat onionr/data/*.db | true > /dev/null 2>&1 + rm -f onionr/data/blocks/*.dat onionr/data/*.db onionr/data/block-nonces.dat | true > /dev/null 2>&1 @./RUN-LINUX.sh version | grep -v "Failed" --color=always reset: diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 1f2166ac..42be9e45 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -186,6 +186,10 @@ class OnionrCommunicatorDaemon: if blockHash in self.currentDownloading: logger.debug('ALREADY DOWNLOADING ' + blockHash) continue + if blockHash in self._core.getBlockList(): + logger.debug('%s is already saved' % (blockHash,)) + self.blockQueue.remove(blockHash) + continue self.currentDownloading.append(blockHash) logger.info("Attempting to download %s..." % blockHash) peerUsed = self.pickOnlinePeer() diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index c298c137..fcbf733e 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -57,8 +57,8 @@ "blockCacheTotal": 50000000 }, "peers":{ - "minimumScore": -1000, - "maxStoredPeers": 100, - "maxConnect": 3 + "minimumScore": -100, + "maxStoredPeers": 500, + "maxConnect": 5 } } From cfbc834eb5e843f0f805618f0a02bfe8a72265ff Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 17 Aug 2018 16:50:16 -0500 Subject: [PATCH 20/53] * fixed locale issue in docker * fixed uploaded blocks not saving type properly --- Dockerfile | 9 +++++++-- onionr/blockimporter.py | 4 ++-- onionr/static-data/default-plugins/pms/main.py | 2 ++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7b7feb20..90f16342 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,13 +4,18 @@ FROM ubuntu:xenial ENV HOME /root #Install needed packages -RUN apt update && apt install -y python3 python3-dev python3-pip tor +RUN apt update && apt install -y python3 python3-dev python3-pip tor locales + +RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ + locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 WORKDIR /srv/ ADD ./requirements.txt /srv/requirements.txt RUN pip3 install -r requirements.txt - WORKDIR /root/ #Add Onionr source COPY . /root/ diff --git a/onionr/blockimporter.py b/onionr/blockimporter.py index 1b37c3cd..2c29927f 100644 --- a/onionr/blockimporter.py +++ b/onionr/blockimporter.py @@ -21,7 +21,7 @@ import core, onionrexceptions, logger def importBlockFromData(content, coreInst): retData = False - dataHash = coreInst._getSha3Hash(content) + dataHash = coreInst._crypto.sha3Hash(content) if coreInst._blacklist.inBlacklist(dataHash): raise onionrexceptions.BlacklistedBlock('%s is a blacklisted block' % (dataHash,)) @@ -40,7 +40,7 @@ def importBlockFromData(content, coreInst): if coreInst._crypto.verifyPow(content): # check if POW is enough/correct logger.info('Block passed proof, saving.') blockHash = coreInst.setData(content) - blockHash = coreInst.addToBlockDB(blockHash, dataSaved=True) + coreInst.addToBlockDB(blockHash, dataSaved=True) coreInst._utils.processBlockMetadata(blockHash) # caches block metadata values to block database retData = True return retData \ No newline at end of file diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index 3da4d268..4b7199fd 100644 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -22,6 +22,8 @@ import logger, config, threading, time, readline, datetime from onionrblockapi import Block import onionrexceptions +import locale +locale.setlocale(locale.LC_ALL, '') plugin_name = 'pms' PLUGIN_VERSION = '0.0.1' From 9655bfd872f4007449bc185204c22c6a0eaef191 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 17 Aug 2018 23:42:30 -0500 Subject: [PATCH 21/53] * do not reinsert blocks * warn mail about bad sigs --- onionr/core.py | 20 ++++++++++++------- onionr/onionrpeers.py | 7 +++++-- .../static-data/default-plugins/pms/main.py | 11 +++++----- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index f3834ed3..9d97e3f1 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -624,6 +624,18 @@ class Core: ''' retData = False + # check nonce + dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data)) + try: + with open(self.dataNonceFile, 'r') as nonces: + if dataNonce in nonces: + return retData + except FileNotFoundError: + pass + # record nonce + with open(self.dataNonceFile, 'a') as nonceFile: + nonceFile.write(dataNonce + '\n') + if meta is None: meta = dict() @@ -683,13 +695,7 @@ class Core: metadata['sig'] = signature metadata['signer'] = signer metadata['time'] = str(self._utils.getEpoch()) - - nonce = self._utils.bytesToStr(self._crypto.sha3Hash(data)) - - # TODO check in advance - with open(self.dataNonceFile, 'a') as nonceFile: - nonceFile.write(nonce + '\n') - + # send block data (and metadata) to POW module to get tokenized block data proof = onionrproofs.POW(metadata, data) payload = proof.waitForResult() diff --git a/onionr/onionrpeers.py b/onionr/onionrpeers.py index cd0745ea..710f698d 100644 --- a/onionr/onionrpeers.py +++ b/onionr/onionrpeers.py @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import core, config, logger +import core, config, logger, sqlite3 class PeerProfiles: ''' PeerProfiles @@ -89,7 +89,10 @@ def peerCleanup(coreInst): # Remove peers that go below the negative score if PeerProfiles(address, coreInst).score < minScore: coreInst.removeAddress(address) - coreInst._blacklist.addToDB(address, dataType=1, expire=300) + try: + coreInst._blacklist.addToDB(address, dataType=1, expire=300) + except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue + pass logger.warn('Removed address ' + address + '.') # Unban probably not malicious peers TODO improve diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index 4b7199fd..300c9625 100644 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -109,14 +109,13 @@ class OnionrMail: pass else: readBlock.verifySig() - print('Message recieved from', readBlock.signer) + print('Message recieved from %s' % (readBlock.signer,)) print('Valid signature:', readBlock.validSig) if not readBlock.validSig: - logger.warn('This message has an INVALID signature. Anyone could have sent this message.') - logger.readline('Press enter to continue to message.') - - print(draw_border(self.myCore._utils.escapeAnsi(readBlock.bcontent.decode().strip()))) - + logger.warn('This message has an INVALID signature. ANYONE could have sent this message.') + cancel = logger.readline('Press enter to continue to message, or -q to not open the message (recommended).') + if cancel != '-q': + print(draw_border(self.myCore._utils.escapeAnsi(readBlock.bcontent.decode().strip()))) return def draftMessage(self): From a900c8eb0e478e657c92c827957e247b1b75b5a9 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 18 Aug 2018 09:40:59 -0500 Subject: [PATCH 22/53] disabled site viewer --- onionr/api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onionr/api.py b/onionr/api.py index 37c6b531..1ce5eca9 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -239,6 +239,7 @@ class API: resp = Response('Goodbye') elif action == 'ping': resp = Response('pong') + '''#TODO Remove and replace with iframe elif action == 'site': block = data siteData = self._core.getData(data) @@ -247,6 +248,7 @@ class API: self.mimeType = 'text/html' response = siteData.split(b'-', 2)[-1] resp = Response(response) + ''' elif action == "insertBlock": response = {'success' : False, 'reason' : 'An unknown error occurred'} From 3a666d23e7fe40efaeffc9afe01531ae77ddf639 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 18 Aug 2018 10:09:55 -0500 Subject: [PATCH 23/53] disabled site viewer --- onionr/api.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index 1ce5eca9..d0407388 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -239,16 +239,6 @@ class API: resp = Response('Goodbye') elif action == 'ping': resp = Response('pong') - '''#TODO Remove and replace with iframe - elif action == 'site': - block = data - siteData = self._core.getData(data) - response = 'not found' - if siteData != '' and siteData != False: - self.mimeType = 'text/html' - response = siteData.split(b'-', 2)[-1] - resp = Response(response) - ''' elif action == "insertBlock": response = {'success' : False, 'reason' : 'An unknown error occurred'} From 0f8581b399b63ed8214011d8d6d902b1d2d1a9b8 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 18 Aug 2018 14:38:15 -0500 Subject: [PATCH 24/53] fixed crash in mail --- onionr/static-data/default-plugins/pms/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index 300c9625..27f56438 100644 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -108,6 +108,7 @@ class OnionrMail: except KeyError: pass else: + cancel = '' readBlock.verifySig() print('Message recieved from %s' % (readBlock.signer,)) print('Valid signature:', readBlock.validSig) From 9d827493e40dbf295a7b6b10f0595cede861c8d7 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 18 Aug 2018 23:07:09 -0500 Subject: [PATCH 25/53] + v3 onion config + use newer ubuntu --- Dockerfile | 4 ++-- onionr/communicator2.py | 2 ++ onionr/netcontroller.py | 9 +++++++-- onionr/onionrcrypto.py | 4 ++++ onionr/onionrutils.py | 4 +++- onionr/static-data/default_config.json | 2 +- 6 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 90f16342..c83de87d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ -FROM ubuntu:xenial +FROM ubuntu:bionic #Base settings ENV HOME /root #Install needed packages -RUN apt update && apt install -y python3 python3-dev python3-pip tor locales +RUN apt update && apt install -y python3 python3-dev python3-pip tor locales nano RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ locale-gen diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 42be9e45..38ba2692 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -318,6 +318,8 @@ class OnionrCommunicatorDaemon: self.addBootstrapListToPeerList(peerList) for address in peerList: + if not config.get('tor.v3onions') and len(address) == 62: + continue if len(address) == 0 or address in tried or address in self.onlinePeers: continue if self.shutdown: diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py index 19f355e4..3749ce7a 100644 --- a/onionr/netcontroller.py +++ b/onionr/netcontroller.py @@ -18,7 +18,7 @@ along with this program. If not, see . ''' -import subprocess, os, random, sys, logger, time, signal +import subprocess, os, random, sys, logger, time, signal, config from onionrblockapi import Block class NetController: @@ -33,6 +33,7 @@ class NetController: self.hsPort = hsPort self._torInstnace = '' self.myID = '' + config.reload() ''' if os.path.exists(self.torConfigLocation): torrc = open(self.torConfigLocation, 'r') @@ -47,11 +48,15 @@ class NetController: ''' Generate a torrc file for our tor instance ''' - + hsVer = '# v2 onions' + if config.get('tor.v3onions'): + hsVer = 'HiddenServiceVersion 3' + logger.info('Using v3 onions :)') if os.path.exists(self.torConfigLocation): os.remove(self.torConfigLocation) torrcData = '''SocksPort ''' + str(self.socksPort) + ''' HiddenServiceDir data/hs/ +\n''' + hsVer + '''\n HiddenServicePort 80 127.0.0.1:''' + str(self.hsPort) + ''' DataDirectory data/tordata/ ''' diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index 40dca90f..00c5a604 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -247,6 +247,10 @@ class OnionrCrypto: return result def sha3Hash(self, data): + try: + data = data.encode() + except AttributeError: + pass hasher = hashlib.sha3_256() hasher.update(data) return hasher.hexdigest() diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 7e0abd94..951f546a 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -43,7 +43,7 @@ class OnionrUtils: self.avoidDupe = [] # list used to prevent duplicate requests per peer for certain actions self.peerProcessing = {} # dict of current peer actions: peer, actionList - + config.reload() return def getTimeBypassToken(self): @@ -128,6 +128,8 @@ class OnionrUtils: for adder in newAdderList.split(','): adder = adder.strip() if not adder in self._core.listAdders(randomOrder = False) and adder != self.getMyAddress() and not self._core._blacklist.inBlacklist(adder): + if not config.get('tor.v3onions') and len(address) == 62: + continue if self._core.addAddress(adder): logger.info('Added %s to db.' % adder, timestamp = True) retVal = True diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index fcbf733e..5458db4a 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -41,7 +41,7 @@ }, "tor" : { - + "v3onions": false }, "i2p":{ From a33d45f43073a076ea37171032b54290f0965e01 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 19 Aug 2018 20:32:21 -0500 Subject: [PATCH 26/53] fixed broken address merge --- onionr/onionrutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 951f546a..6d22992c 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -128,7 +128,7 @@ class OnionrUtils: for adder in newAdderList.split(','): adder = adder.strip() if not adder in self._core.listAdders(randomOrder = False) and adder != self.getMyAddress() and not self._core._blacklist.inBlacklist(adder): - if not config.get('tor.v3onions') and len(address) == 62: + if not config.get('tor.v3onions') and len(adder) == 62: continue if self._core.addAddress(adder): logger.info('Added %s to db.' % adder, timestamp = True) From 53577a4c10e8ab2b3002aa324ce66f8da4b45b09 Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 21 Aug 2018 15:01:50 -0500 Subject: [PATCH 27/53] work on netcheck and configuration --- onionr/communicator2.py | 3 +++ onionr/onionrdaemontools.py | 11 ++++++++++- onionr/onionrutils.py | 27 +++++++++++++++++++++++--- onionr/static-data/default_config.json | 2 +- 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 38ba2692..a575a350 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -27,6 +27,8 @@ from defusedxml import minidom class OnionrCommunicatorDaemon: def __init__(self, debug, developmentMode): + self.isOnline = True # Assume we're connected to the internet + # list of timer instances self.timers = [] @@ -93,6 +95,7 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58) OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True) OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True) + netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 305, requiresPeer=True, maxThreads=1) cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requiresPeer=True) diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index 8410cb80..36264600 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -53,4 +53,13 @@ class DaemonTools: if self.daemon._core._utils.doPostRequest(url, data) == 'Success': retData = True self.daemon.decrementThreadCount('announceNode') - return retData \ No newline at end of file + return retData + + def netCheck(self): + '''Check if we are connected to the internet or not when we can't connect to any peers''' + if len(self.daemon.onlinePeers) != 0: + if not self.daemon._core._utils.checkNetwork(): + logger.warn('Network check failed, are you connected to the internet?') + self.daemon.isOnline = False + + self.daemon.decrementThreadCount('netCheck') \ No newline at end of file diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 6d22992c..6fffbee8 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -131,8 +131,12 @@ class OnionrUtils: if not config.get('tor.v3onions') and len(adder) == 62: continue if self._core.addAddress(adder): - logger.info('Added %s to db.' % adder, timestamp = True) - retVal = True + # Check if we have the maxmium amount of allowed stored peers + if config.get('peers.maxStoredPeers') > len(self._core.listAdders): + logger.info('Added %s to db.' % adder, timestamp = True) + retVal = True + else: + logger.warn('Reached the maximum amount of peers in the net database as allowed by your config.') else: pass #logger.debug('%s is either our address or already in our DB' % adder) @@ -630,6 +634,23 @@ class OnionrUtils: except AttributeError: pass return data + + def checkNetwork(self): + '''Check if we are connected to the internet (through Tor)''' + retData = False + connectURLs = [] + try: + with open('static-data/connect-check.txt', 'r') as connectTest: + connectURLs = connectTest.read().split(',') + + for url in connectURLs: + if self.doGetRequest(url) != False: + retData = True + break + + except FileNotFoundError: + pass + return retData def size(path='.'): ''' @@ -655,4 +676,4 @@ def humanSize(num, suffix='B'): if abs(num) < 1024.0: return "%.1f %s%s" % (num, unit, suffix) num /= 1024.0 - return "%.1f %s%s" % (num, 'Yi', suffix) + return "%.1f %s%s" % (num, 'Yi', suffix) \ No newline at end of file diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 5458db4a..86d44499 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -58,7 +58,7 @@ }, "peers":{ "minimumScore": -100, - "maxStoredPeers": 500, + "maxStoredPeers": 5000, "maxConnect": 5 } } From dd5cb991552573efe758c46a88c22c538595e182 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Wed, 22 Aug 2018 23:59:41 -0500 Subject: [PATCH 28/53] * do not save blocks if disk allocation reached * improved some commenting * bug fixes --- onionr/blockimporter.py | 12 ++++++++---- onionr/communicator2.py | 26 ++++++++++++++++++-------- onionr/core.py | 23 ++++++++++++++--------- onionr/onionrexceptions.py | 5 +++++ onionr/onionrutils.py | 5 ++--- 5 files changed, 47 insertions(+), 24 deletions(-) diff --git a/onionr/blockimporter.py b/onionr/blockimporter.py index 2c29927f..ce1cd1fe 100644 --- a/onionr/blockimporter.py +++ b/onionr/blockimporter.py @@ -39,8 +39,12 @@ def importBlockFromData(content, coreInst): if coreInst._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid if coreInst._crypto.verifyPow(content): # check if POW is enough/correct logger.info('Block passed proof, saving.') - blockHash = coreInst.setData(content) - coreInst.addToBlockDB(blockHash, dataSaved=True) - coreInst._utils.processBlockMetadata(blockHash) # caches block metadata values to block database - retData = True + try: + blockHash = coreInst.setData(content) + except onionrexceptions.DiskAllocationReached: + pass + else: + coreInst.addToBlockDB(blockHash, dataSaved=True) + coreInst._utils.processBlockMetadata(blockHash) # caches block metadata values to block database + retData = True return retData \ No newline at end of file diff --git a/onionr/communicator2.py b/onionr/communicator2.py index a575a350..da610f4b 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -117,14 +117,14 @@ class OnionrCommunicatorDaemon: pass logger.info('Goodbye.') - self._core._utils.localCommand('shutdown') + self._core._utils.localCommand('shutdown') # shutdown the api time.sleep(0.5) def lookupKeys(self): '''Lookup new keys''' logger.debug('Looking up new keys...') tryAmount = 1 - for i in range(tryAmount): + for i in range(tryAmount): # amount of times to ask peers for new keys # Download new key list from random online peers peer = self.pickOnlinePeer() newKeys = self.peerAction(peer, action='kex') @@ -151,6 +151,10 @@ class OnionrCommunicatorDaemon: existingBlocks = self._core.getBlockList() triedPeers = [] # list of peers we've tried this time around for i in range(tryAmount): + # check if disk allocation is used + if self._core._utils.storageCounter.isFull(): + logger.warn('Not looking up new blocks due to maximum amount of allowed disk space used') + break peer = self.pickOnlinePeer() # select random online peer # if we've already tried all the online peers this time around, stop if peer in triedPeers: @@ -165,7 +169,7 @@ class OnionrCommunicatorDaemon: if newDBHash != self._core.getAddressInfo(peer, 'DBHash'): self._core.setAddressInfo(peer, 'DBHash', newDBHash) try: - newBlocks = self.peerAction(peer, 'getBlockHashes') + newBlocks = self.peerAction(peer, 'getBlockHashes') # get list of new block hashes except Exception as error: logger.warn("could not get new blocks with " + peer, error=error) newBlocks = False @@ -177,7 +181,7 @@ class OnionrCommunicatorDaemon: if not i in existingBlocks: # if block does not exist on disk and is not already in block queue if i not in self.blockQueue and not self._core._blacklist.inBlacklist(i): - self.blockQueue.append(i) + self.blockQueue.append(i) # add blocks to download queue self.decrementThreadCount('lookupBlocks') return @@ -185,7 +189,9 @@ class OnionrCommunicatorDaemon: '''download new blocks in queue''' for blockHash in self.blockQueue: if self.shutdown: + # Exit loop if shutting down break + # Do not download blocks being downloaded or that are already saved (edge cases) if blockHash in self.currentDownloading: logger.debug('ALREADY DOWNLOADING ' + blockHash) continue @@ -193,7 +199,7 @@ class OnionrCommunicatorDaemon: logger.debug('%s is already saved' % (blockHash,)) self.blockQueue.remove(blockHash) continue - self.currentDownloading.append(blockHash) + self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block logger.info("Attempting to download %s..." % blockHash) peerUsed = self.pickOnlinePeer() content = self.peerAction(peerUsed, 'getData', data=blockHash) # block content from random peer (includes metadata) @@ -216,9 +222,13 @@ class OnionrCommunicatorDaemon: if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce if self._core._crypto.verifyPow(content): # check if POW is enough/correct logger.info('Block passed proof, saving.') - self._core.setData(content) - self._core.addToBlockDB(blockHash, dataSaved=True) - self._core._utils.processBlockMetadata(blockHash) # caches block metadata values to block database + try: + self._core.setData(content) + except onionrexceptions.DiskAllocationReached: + logger.error("Reached disk allocation allowance, cannot save additional blocks.") + else: + self._core.addToBlockDB(blockHash, dataSaved=True) + self._core._utils.processBlockMetadata(blockHash) # caches block metadata values to block database else: logger.warn('POW failed for block ' + blockHash) else: diff --git a/onionr/core.py b/onionr/core.py index 9d97e3f1..14f4c3ba 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -50,6 +50,7 @@ class Core: self.dbCreate = dbcreator.DBCreator(self) self.usageFile = 'data/disk-usage.txt' + self.config = config if not os.path.exists('data/'): os.mkdir('data/') @@ -256,6 +257,8 @@ class Core: Set the data assciated with a hash ''' data = data + dataSize = sys.getsizeof(data) + if not type(data) is bytes: data = data.encode() @@ -268,15 +271,17 @@ class Core: pass # TODO: properly check if block is already saved elsewhere #raise Exception("Data is already set for " + dataHash) else: - blockFile = open(blockFileName, 'wb') - blockFile.write(data) - blockFile.close() - - conn = sqlite3.connect(self.blockDB) - c = conn.cursor() - c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = '" + dataHash + "';") - conn.commit() - conn.close() + if self._utils.storageCounter.addBytes(dataSize) != False: + blockFile = open(blockFileName, 'wb') + blockFile.write(data) + blockFile.close() + conn = sqlite3.connect(self.blockDB) + c = conn.cursor() + c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = '" + dataHash + "';") + conn.commit() + conn.close() + else: + raise onionrexceptions.DiskAllocationReached return dataHash diff --git a/onionr/onionrexceptions.py b/onionr/onionrexceptions.py index b26a97d7..040bc9be 100644 --- a/onionr/onionrexceptions.py +++ b/onionr/onionrexceptions.py @@ -58,3 +58,8 @@ class MissingPort(Exception): class InvalidAddress(Exception): pass + +# file exceptions + +class DiskAllocationReached: + pass \ No newline at end of file diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 6fffbee8..bf57df66 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -23,7 +23,7 @@ import nacl.signing, nacl.encoding from onionrblockapi import Block import onionrexceptions from defusedxml import minidom -import pgpwords +import pgpwords, storagecounter if sys.version_info < (3, 6): try: import sha3 @@ -40,9 +40,9 @@ class OnionrUtils: self._core = coreInstance self.timingToken = '' - self.avoidDupe = [] # list used to prevent duplicate requests per peer for certain actions self.peerProcessing = {} # dict of current peer actions: peer, actionList + self.storageCounter = storagecounter.StorageCounter(self._core) config.reload() return @@ -647,7 +647,6 @@ class OnionrUtils: if self.doGetRequest(url) != False: retData = True break - except FileNotFoundError: pass return retData From a6719abed78a221f6f263ea2b70d593603f1f109 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 23 Aug 2018 09:01:17 -0500 Subject: [PATCH 29/53] added storagecounter.py --- onionr/storagecounter.py | 61 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 onionr/storagecounter.py diff --git a/onionr/storagecounter.py b/onionr/storagecounter.py new file mode 100644 index 00000000..50456a24 --- /dev/null +++ b/onionr/storagecounter.py @@ -0,0 +1,61 @@ +''' + Onionr - P2P Microblogging Platform & Social network. + + Keeps track of how much disk space we're using +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import config + +class StorageCounter: + def __init__(self, coreInst): + self._core = coreInst + self.dataFile = self._core.usageFile + return + + def isFull(self): + retData = False + if self._core.config.get('allocations.disk') >= self.getAmount(): + retData = True + return retData + + def _update(self, data): + with open(self.dataFile, 'w') as dataFile: + dataFile.write(str(data)) + def getAmount(self, data): + '''Return how much disk space we're using (according to record)''' + retData = 0 + try: + with open(self.dataFile, 'w') as dataFile: + retData = int(dataFile.read()) + except FileNotFoundError: + pass + return retData + + def addBytes(self, amount): + '''Record that we are now using more disk space, unless doing so would exceed configured max''' + newAmount = amount + self.getAmount() + retData = newAmount + if newAmount > self._core.config.get('allocations.disk'): + retData = False + else: + self._update(newAmount) + return retData + + def removeBytes(self, amount): + '''Record that we are now using less disk space''' + newAmount = self.getAmount() - amount + self._update(newAmount) + return newAmount \ No newline at end of file From 1217e4a83c36d80975e83535481b67b99b972415 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 23 Aug 2018 09:45:51 -0500 Subject: [PATCH 30/53] removed bad argument in storagecounter --- onionr/storagecounter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/storagecounter.py b/onionr/storagecounter.py index 50456a24..b2ccd0d1 100644 --- a/onionr/storagecounter.py +++ b/onionr/storagecounter.py @@ -34,7 +34,7 @@ class StorageCounter: def _update(self, data): with open(self.dataFile, 'w') as dataFile: dataFile.write(str(data)) - def getAmount(self, data): + def getAmount(self): '''Return how much disk space we're using (according to record)''' retData = 0 try: From cd39ae68b6700fc56d8e097c97a33766b3c48283 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 23 Aug 2018 09:51:53 -0500 Subject: [PATCH 31/53] r not w --- onionr/storagecounter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/storagecounter.py b/onionr/storagecounter.py index b2ccd0d1..024eb4f5 100644 --- a/onionr/storagecounter.py +++ b/onionr/storagecounter.py @@ -38,7 +38,7 @@ class StorageCounter: '''Return how much disk space we're using (according to record)''' retData = 0 try: - with open(self.dataFile, 'w') as dataFile: + with open(self.dataFile, 'r') as dataFile: retData = int(dataFile.read()) except FileNotFoundError: pass From 25e4444bdafb16ef708952a6d278dad7b596284c Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 23 Aug 2018 09:54:37 -0500 Subject: [PATCH 32/53] fix comparison error in storagecounter --- onionr/storagecounter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/storagecounter.py b/onionr/storagecounter.py index 024eb4f5..481d7821 100644 --- a/onionr/storagecounter.py +++ b/onionr/storagecounter.py @@ -27,7 +27,7 @@ class StorageCounter: def isFull(self): retData = False - if self._core.config.get('allocations.disk') >= self.getAmount(): + if self._core.config.get('allocations.disk') <= self.getAmount(): retData = True return retData From 6d31fa4229b1b871b0cabe394e237431a61295ad Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 23 Aug 2018 09:59:00 -0500 Subject: [PATCH 33/53] fix peer amount check not calling method --- onionr/onionrutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index bf57df66..b48160a3 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -132,7 +132,7 @@ class OnionrUtils: continue if self._core.addAddress(adder): # Check if we have the maxmium amount of allowed stored peers - if config.get('peers.maxStoredPeers') > len(self._core.listAdders): + if config.get('peers.maxStoredPeers') > len(self._core.listAdders()): logger.info('Added %s to db.' % adder, timestamp = True) retVal = True else: From e34c08b0368e14c545e7e5c9008fc1b1d3bcc141 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 23 Aug 2018 12:48:49 -0500 Subject: [PATCH 34/53] sync improvements, bug fixes, config changes --- onionr/communicator2.py | 22 +++++++--------------- onionr/core.py | 14 ++++++++++++-- onionr/onionr.py | 10 ++++++++++ onionr/onionrdaemontools.py | 10 ++++++++-- onionr/onionrexceptions.py | 2 +- onionr/onionrutils.py | 4 ---- onionr/static-data/default_config.json | 2 +- onionr/storagecounter.py | 2 +- 8 files changed, 40 insertions(+), 26 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index da610f4b..057defed 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -80,10 +80,6 @@ class OnionrCommunicatorDaemon: if debug or developmentMode: OnionrCommunicatorTimers(self, self.heartbeat, 10) - # Print nice header thing :) - if config.get('general.display_header', True) and not self.shutdown: - self.header() - # Set timers, function reference, seconds # requiresPeer True means the timer function won't fire if we have no connected peers # TODO: make some of these timer counts configurable @@ -93,6 +89,7 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.lookupBlocks, 7, requiresPeer=True, maxThreads=1) OnionrCommunicatorTimers(self, self.getBlocks, 10, requiresPeer=True) OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58) + OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 650) OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True) OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True) netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) @@ -152,6 +149,8 @@ class OnionrCommunicatorDaemon: triedPeers = [] # list of peers we've tried this time around for i in range(tryAmount): # check if disk allocation is used + if not self.isOnline: + break if self._core._utils.storageCounter.isFull(): logger.warn('Not looking up new blocks due to maximum amount of allowed disk space used') break @@ -188,8 +187,8 @@ class OnionrCommunicatorDaemon: def getBlocks(self): '''download new blocks in queue''' for blockHash in self.blockQueue: - if self.shutdown: - # Exit loop if shutting down + if self.shutdown or not self.isOnline: + # Exit loop if shutting down or offline break # Do not download blocks being downloaded or that are already saved (edge cases) if blockHash in self.currentDownloading: @@ -221,11 +220,11 @@ class OnionrCommunicatorDaemon: #meta = metas[1] if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce if self._core._crypto.verifyPow(content): # check if POW is enough/correct - logger.info('Block passed proof, saving.') + logger.info('Block passed proof, attemping save.') try: self._core.setData(content) except onionrexceptions.DiskAllocationReached: - logger.error("Reached disk allocation allowance, cannot save additional blocks.") + logger.error("Reached disk allocation allowance, cannot save this block.") else: self._core.addToBlockDB(blockHash, dataSaved=True) self._core._utils.processBlockMetadata(blockHash) # caches block metadata values to block database @@ -489,13 +488,6 @@ class OnionrCommunicatorDaemon: self.shutdown = True self.decrementThreadCount('detectAPICrash') - def header(self, message = logger.colors.fg.pink + logger.colors.bold + 'Onionr' + logger.colors.reset + logger.colors.fg.pink + ' has started.'): - if os.path.exists('static-data/header.txt'): - with open('static-data/header.txt', 'rb') as file: - # only to stdout, not file or log or anything - sys.stderr.write(file.read().decode().replace('P', logger.colors.fg.pink).replace('W', logger.colors.reset + logger.colors.bold).replace('G', logger.colors.fg.green).replace('\n', logger.colors.reset + '\n').replace('B', logger.colors.bold).replace('V', onionr.ONIONR_VERSION)) - logger.info(logger.colors.fg.lightgreen + '-> ' + str(message) + logger.colors.reset + logger.colors.fg.lightgreen + ' <-\n') - class OnionrCommunicatorTimers: def __init__(self, daemonInstance, timerFunction, frequency, makeThread=True, threadAmount=1, maxThreads=5, requiresPeer=False): self.timerFunction = timerFunction diff --git a/onionr/core.py b/onionr/core.py index 14f4c3ba..13c89c9e 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -183,8 +183,16 @@ class Core: c.execute('Delete from hashes where hash=?;', t) conn.commit() conn.close() + blockFile = 'data/blocks/' + block + '.dat' + dataSize = 0 try: - os.remove('data/blocks/' + block + '.dat') + ''' Get size of data when loaded as an object/var, rather than on disk, + to avoid conflict with getsizeof when saving blocks + ''' + with open(blockFile, 'r') as data: + dataSize = sys.getsizeof(data.read()) + self._utils.storageCounter.removeBytes(dataSize) + os.remove(blockFile) except FileNotFoundError: pass @@ -280,6 +288,8 @@ class Core: c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = '" + dataHash + "';") conn.commit() conn.close() + with open(self.dataNonceFile, 'a') as nonceFile: + nonceFile.write(dataHash + '\n') else: raise onionrexceptions.DiskAllocationReached @@ -544,7 +554,7 @@ class Core: if unsaved: execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();' else: - execute = 'SELECT hash FROM hashes ORDER BY dateReceived DESC;' + execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;' rows = list() for row in c.execute(execute): for i in row: diff --git a/onionr/onionr.py b/onionr/onionr.py index 1736c3f9..52b12e48 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -588,6 +588,9 @@ class Onionr: time.sleep(1) #TODO make runable on windows subprocess.Popen([communicatorDaemon, "run", str(net.socksPort)]) + # Print nice header thing :) + if config.get('general.display_header', True): + self.header() logger.debug('Started communicator') events.event('daemon_start', onionr = self) try: @@ -759,5 +762,12 @@ class Onionr: print('Opening %s ...' % url) webbrowser.open(url, new = 1, autoraise = True) + def header(self, message = logger.colors.fg.pink + logger.colors.bold + 'Onionr' + logger.colors.reset + logger.colors.fg.pink + ' has started.'): + if os.path.exists('static-data/header.txt'): + with open('static-data/header.txt', 'rb') as file: + # only to stdout, not file or log or anything + sys.stderr.write(file.read().decode().replace('P', logger.colors.fg.pink).replace('W', logger.colors.reset + logger.colors.bold).replace('G', logger.colors.fg.green).replace('\n', logger.colors.reset + '\n').replace('B', logger.colors.bold).replace('V', ONIONR_VERSION)) + logger.info(logger.colors.fg.lightgreen + '-> ' + str(message) + logger.colors.reset + logger.colors.fg.lightgreen + ' <-\n') + if __name__ == "__main__": Onionr() diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index 36264600..f269f028 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -61,5 +61,11 @@ class DaemonTools: if not self.daemon._core._utils.checkNetwork(): logger.warn('Network check failed, are you connected to the internet?') self.daemon.isOnline = False - - self.daemon.decrementThreadCount('netCheck') \ No newline at end of file + self.daemon.decrementThreadCount('netCheck') + + def cleanOldBlocks(self): + '''Delete old blocks if our disk allocation is full/near full''' + if self.daemon._core._utils.storageCounter.isFull(): + + + self.daemon.decrementThreadCount('cleanOldBlocks') \ No newline at end of file diff --git a/onionr/onionrexceptions.py b/onionr/onionrexceptions.py index 040bc9be..8044508f 100644 --- a/onionr/onionrexceptions.py +++ b/onionr/onionrexceptions.py @@ -61,5 +61,5 @@ class InvalidAddress(Exception): # file exceptions -class DiskAllocationReached: +class DiskAllocationReached(Exception): pass \ No newline at end of file diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index b48160a3..d1383cab 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -384,10 +384,6 @@ class OnionrUtils: pass else: retData = True - if retData: - # Executes if data not seen - with open(self._core.dataNonceFile, 'a') as nonceFile: - nonceFile.write(nonce + '\n') else: logger.warn('In call to utils.validateMetadata, metadata must be JSON string or a dictionary object') diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 86d44499..4b53f391 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -51,7 +51,7 @@ }, "allocations":{ - "disk": 9000000000, + "disk": 800, "netTotal": 1000000000, "blockCache": 5000000, "blockCacheTotal": 50000000 diff --git a/onionr/storagecounter.py b/onionr/storagecounter.py index 481d7821..a5c11f21 100644 --- a/onionr/storagecounter.py +++ b/onionr/storagecounter.py @@ -27,7 +27,7 @@ class StorageCounter: def isFull(self): retData = False - if self._core.config.get('allocations.disk') <= self.getAmount(): + if self._core.config.get('allocations.disk') <= (self.getAmount() + 100): retData = True return retData From 638436ee10c535999d4334be4e55e5ff4c8b596a Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 23 Aug 2018 13:02:48 -0500 Subject: [PATCH 35/53] sync improvements, bug fixes, config changes --- onionr/onionrdaemontools.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index f269f028..e36945c0 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -65,7 +65,9 @@ class DaemonTools: def cleanOldBlocks(self): '''Delete old blocks if our disk allocation is full/near full''' - if self.daemon._core._utils.storageCounter.isFull(): - - + while self.daemon._core._utils.storageCounter.isFull(): + oldest = self.daemon._core.getBlockList[0] + self.daemon._core._blacklist.addToDB(oldest) + self.daemon._core.removeBlock(oldest) + logger.info('Deleted block: %s' % (oldest,)) self.daemon.decrementThreadCount('cleanOldBlocks') \ No newline at end of file From 1faae80aafdc4907a158bc7cb78e763762a993eb Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 23 Aug 2018 13:24:32 -0500 Subject: [PATCH 36/53] sync improvements, bug fixes, config changes --- onionr/core.py | 2 ++ onionr/onionrdaemontools.py | 2 +- onionr/storagecounter.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index 13c89c9e..c6495f2f 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -175,6 +175,8 @@ class Core: def removeBlock(self, block): ''' remove a block from this node (does not automatically blacklist) + + **You may want blacklist.addToDB(blockHash) ''' if self._utils.validateHash(block): conn = sqlite3.connect(self.blockDB) diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index e36945c0..ff4a9b3f 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -66,7 +66,7 @@ class DaemonTools: def cleanOldBlocks(self): '''Delete old blocks if our disk allocation is full/near full''' while self.daemon._core._utils.storageCounter.isFull(): - oldest = self.daemon._core.getBlockList[0] + oldest = self.daemon._core.getBlockList()[0] self.daemon._core._blacklist.addToDB(oldest) self.daemon._core.removeBlock(oldest) logger.info('Deleted block: %s' % (oldest,)) diff --git a/onionr/storagecounter.py b/onionr/storagecounter.py index a5c11f21..b88bc4f5 100644 --- a/onionr/storagecounter.py +++ b/onionr/storagecounter.py @@ -27,7 +27,7 @@ class StorageCounter: def isFull(self): retData = False - if self._core.config.get('allocations.disk') <= (self.getAmount() + 100): + if self._core.config.get('allocations.disk') <= (self.getAmount() + 500): retData = True return retData From e346c092282fcfa0de3af8858528ddf8e6499942 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 23 Aug 2018 14:46:23 -0500 Subject: [PATCH 37/53] bug fixes --- onionr/communicator2.py | 11 ++++++++--- onionr/onionrdaemontools.py | 2 +- onionr/onionrpeers.py | 6 +++++- onionr/onionrutils.py | 4 ++-- onionr/static-data/default_config.json | 2 +- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 057defed..42bbbce8 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -89,7 +89,7 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.lookupBlocks, 7, requiresPeer=True, maxThreads=1) OnionrCommunicatorTimers(self, self.getBlocks, 10, requiresPeer=True) OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58) - OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 650) + OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 65) OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True) OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True) netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) @@ -152,7 +152,7 @@ class OnionrCommunicatorDaemon: if not self.isOnline: break if self._core._utils.storageCounter.isFull(): - logger.warn('Not looking up new blocks due to maximum amount of allowed disk space used') + logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used') break peer = self.pickOnlinePeer() # select random online peer # if we've already tried all the online peers this time around, stop @@ -187,6 +187,7 @@ class OnionrCommunicatorDaemon: def getBlocks(self): '''download new blocks in queue''' for blockHash in self.blockQueue: + removeFromQueue = True if self.shutdown or not self.isOnline: # Exit loop if shutting down or offline break @@ -198,6 +199,8 @@ class OnionrCommunicatorDaemon: logger.debug('%s is already saved' % (blockHash,)) self.blockQueue.remove(blockHash) continue + if self._core._utils.storageCounter.isFull(): + break self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block logger.info("Attempting to download %s..." % blockHash) peerUsed = self.pickOnlinePeer() @@ -225,6 +228,7 @@ class OnionrCommunicatorDaemon: self._core.setData(content) except onionrexceptions.DiskAllocationReached: logger.error("Reached disk allocation allowance, cannot save this block.") + removeFromQueue = False else: self._core.addToBlockDB(blockHash, dataSaved=True) self._core._utils.processBlockMetadata(blockHash) # caches block metadata values to block database @@ -246,7 +250,8 @@ class OnionrCommunicatorDaemon: # Punish peer for sharing invalid block (not always malicious, but is bad regardless) onionrpeers.PeerProfiles(peerUsed, self._core).addScore(-50) logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash) - self.blockQueue.remove(blockHash) # remove from block queue both if success or false + if removeFromQueue: + self.blockQueue.remove(blockHash) # remove from block queue both if success or false self.currentDownloading.remove(blockHash) self.decrementThreadCount('getBlocks') return diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index ff4a9b3f..7509d421 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -58,7 +58,7 @@ class DaemonTools: def netCheck(self): '''Check if we are connected to the internet or not when we can't connect to any peers''' if len(self.daemon.onlinePeers) != 0: - if not self.daemon._core._utils.checkNetwork(): + if not self.daemon._core._utils.checkNetwork(torPort=self.daemon.proxyPort): logger.warn('Network check failed, are you connected to the internet?') self.daemon.isOnline = False self.daemon.decrementThreadCount('netCheck') diff --git a/onionr/onionrpeers.py b/onionr/onionrpeers.py index 710f698d..82224d0a 100644 --- a/onionr/onionrpeers.py +++ b/onionr/onionrpeers.py @@ -90,7 +90,11 @@ def peerCleanup(coreInst): if PeerProfiles(address, coreInst).score < minScore: coreInst.removeAddress(address) try: - coreInst._blacklist.addToDB(address, dataType=1, expire=300) + if (self.coreInst._utils.getEpoch() - coreInst.getPeerInfo(address, 4)) >= 600: + expireTime = 600 + else: + expireTime = 86400 + coreInst._blacklist.addToDB(address, dataType=1, expire=expireTime) except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue pass logger.warn('Removed address ' + address + '.') diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index d1383cab..2a47aaba 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -631,7 +631,7 @@ class OnionrUtils: pass return data - def checkNetwork(self): + def checkNetwork(self, torPort=0): '''Check if we are connected to the internet (through Tor)''' retData = False connectURLs = [] @@ -640,7 +640,7 @@ class OnionrUtils: connectURLs = connectTest.read().split(',') for url in connectURLs: - if self.doGetRequest(url) != False: + if self.doGetRequest(url, port=torPort) != False: retData = True break except FileNotFoundError: diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 4b53f391..9fba656b 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -51,7 +51,7 @@ }, "allocations":{ - "disk": 800, + "disk": 10000000000, "netTotal": 1000000000, "blockCache": 5000000, "blockCacheTotal": 50000000 From b46bd42d9e6aa21711b9c17267adf927c0552e1a Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 24 Aug 2018 17:42:09 -0500 Subject: [PATCH 38/53] fixed invalid argument call --- onionr/communicator2.py | 4 +++- onionr/core.py | 2 ++ onionr/storagecounter.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 42bbbce8..df428c03 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -199,6 +199,8 @@ class OnionrCommunicatorDaemon: logger.debug('%s is already saved' % (blockHash,)) self.blockQueue.remove(blockHash) continue + if self._core._blacklist.inBlacklist(blockHash): + continue if self._core._utils.storageCounter.isFull(): break self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block @@ -223,7 +225,7 @@ class OnionrCommunicatorDaemon: #meta = metas[1] if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce if self._core._crypto.verifyPow(content): # check if POW is enough/correct - logger.info('Block passed proof, attemping save.') + logger.info('Block passed proof, attempting save.') try: self._core.setData(content) except onionrexceptions.DiskAllocationReached: diff --git a/onionr/core.py b/onionr/core.py index c6495f2f..9d0b831d 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -52,6 +52,8 @@ class Core: self.usageFile = 'data/disk-usage.txt' self.config = config + self.maxBlockSize = 10000000 # max block size in bytes + if not os.path.exists('data/'): os.mkdir('data/') if not os.path.exists('data/blocks/'): diff --git a/onionr/storagecounter.py b/onionr/storagecounter.py index b88bc4f5..4468dacc 100644 --- a/onionr/storagecounter.py +++ b/onionr/storagecounter.py @@ -27,7 +27,7 @@ class StorageCounter: def isFull(self): retData = False - if self._core.config.get('allocations.disk') <= (self.getAmount() + 500): + if self._core.config.get('allocations.disk') <= (self.getAmount() + 1000): retData = True return retData From b6c015255deecec4525898d168cb1380c6eea0f6 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 24 Aug 2018 18:57:53 -0500 Subject: [PATCH 39/53] do not use self in peercleanup --- onionr/onionrpeers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/onionrpeers.py b/onionr/onionrpeers.py index 82224d0a..a1f4aa8b 100644 --- a/onionr/onionrpeers.py +++ b/onionr/onionrpeers.py @@ -90,7 +90,7 @@ def peerCleanup(coreInst): if PeerProfiles(address, coreInst).score < minScore: coreInst.removeAddress(address) try: - if (self.coreInst._utils.getEpoch() - coreInst.getPeerInfo(address, 4)) >= 600: + if (coreInst._utils.getEpoch() - coreInst.getPeerInfo(address, 4)) >= 600: expireTime = 600 else: expireTime = 86400 From 7a7b62725958d32be04ea1d2a374dc0ef49f839d Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 25 Aug 2018 09:33:38 -0500 Subject: [PATCH 40/53] use config for some timers --- onionr/communicator2.py | 4 ++-- onionr/static-data/default_config.json | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index df428c03..1c8329da 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -86,8 +86,8 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.daemonCommands, 5) OnionrCommunicatorTimers(self, self.detectAPICrash, 5) peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60) - OnionrCommunicatorTimers(self, self.lookupBlocks, 7, requiresPeer=True, maxThreads=1) - OnionrCommunicatorTimers(self, self.getBlocks, 10, requiresPeer=True) + OnionrCommunicatorTimers(self, self.lookupBlocks, self._core.config.get('timers.lookupBlocks'), requiresPeer=True, maxThreads=1) + OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.getBlocks'), requiresPeer=True) OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58) OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 65) OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True) diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 9fba656b..6e35245e 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -60,5 +60,9 @@ "minimumScore": -100, "maxStoredPeers": 5000, "maxConnect": 5 + }, + "timers":{ + "lookupBlocks": 25, + "getBlocks": 30 } } From 7a0cfe34f39d5cfe3dfdc68f9035363515a9a0d3 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 26 Aug 2018 22:44:32 -0500 Subject: [PATCH 41/53] work on adding/removing friends, user info --- onionr/communicator2.py | 2 +- onionr/core.py | 16 ++++++----- onionr/dbcreator.py | 1 - onionr/onionr.py | 44 +++++++++++++++++++++++++++--- onionr/onionrblacklist.py | 2 +- onionr/onionrblockapi.py | 2 +- onionr/onionrcrypto.py | 20 -------------- onionr/onionrdaemontools.py | 2 +- onionr/onionrexceptions.py | 5 +++- onionr/onionrpeers.py | 2 +- onionr/onionrusers.py | 51 +++++++++++++++++++++++++++++++++++ onionr/onionrutils.py | 15 +++++++++++ onionr/static-data/index.html | 2 +- 13 files changed, 127 insertions(+), 37 deletions(-) create mode 100644 onionr/onionrusers.py diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 38ba2692..9c44ed21 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 ''' - Onionr - P2P Microblogging Platform & Social network. + Onionr - P2P Anonymous Storage Network This file contains both the OnionrCommunicate class for communcating with peers and code to operate as a daemon, getting commands from the command queue database (see core.Core.daemonQueue) diff --git a/onionr/core.py b/onionr/core.py index 9d97e3f1..f7d36072 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -1,5 +1,5 @@ ''' - Onionr - P2P Microblogging Platform & Social network + Onionr - P2P Anonymous Storage Network Core Onionr library, useful for external programs. Handles peer & data processing ''' @@ -102,7 +102,7 @@ class Core: conn = sqlite3.connect(self.peerDB) hashID = self._crypto.pubKeyHashID(peerID) c = conn.cursor() - t = (peerID, name, 'unknown', hashID, powID) + t = (peerID, name, 'unknown', hashID, powID, 0) for i in c.execute("SELECT * FROM PEERS where id = '" + peerID + "';"): try: @@ -113,7 +113,7 @@ class Core: pass except IndexError: pass - c.execute('INSERT INTO peers (id, name, dateSeen, pow, hashID) VALUES(?, ?, ?, ?, ?);', t) + c.execute('INSERT INTO peers (id, name, dateSeen, pow, hashID, trust) VALUES(?, ?, ?, ?, ?, ?);', t) conn.commit() conn.close() @@ -403,19 +403,23 @@ class Core: conn.close() return addressList - def listPeers(self, randomOrder=True, getPow=False): + def listPeers(self, randomOrder=True, getPow=False, trust=0): ''' Return a list of public keys (misleading function name) randomOrder determines if the list should be in a random order + trust sets the minimum trust to list ''' conn = sqlite3.connect(self.peerDB) c = conn.cursor() payload = "" + if trust not in (0, 1, 2): + logger.error('Tried to select invalid trust.') + return if randomOrder: - payload = 'SELECT * FROM peers ORDER BY RANDOM();' + payload = 'SELECT * FROM peers where trust >= %s ORDER BY RANDOM();' % (trust,) else: - payload = 'SELECT * FROM peers;' + payload = 'SELECT * FROM peers where trust >= %s;' % (trust,) peerList = [] for i in c.execute(payload): try: diff --git a/onionr/dbcreator.py b/onionr/dbcreator.py index 19a9a7bd..5f3d2c79 100644 --- a/onionr/dbcreator.py +++ b/onionr/dbcreator.py @@ -61,7 +61,6 @@ class DBCreator: ID text not null, name text, adders text, - blockDBHash text, forwardKey text, dateSeen not null, bytesStored int, diff --git a/onionr/onionr.py b/onionr/onionr.py index 1736c3f9..5408b2c4 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 ''' - Onionr - P2P Microblogging Platform & Social network. + Onionr - P2P Anonymous Storage Network Onionr is the name for both the protocol and the original/reference software. @@ -32,7 +32,7 @@ import onionrutils from onionrutils import OnionrUtils from netcontroller import NetController from onionrblockapi import Block -import onionrproofs +import onionrproofs, onionrexceptions, onionrusers try: from urllib3.contrib.socks import SOCKSProxyManager @@ -210,7 +210,9 @@ class Onionr: 'getpass': self.printWebPassword, 'get-pass': self.printWebPassword, 'getpasswd': self.printWebPassword, - 'get-passwd': self.printWebPassword + 'get-passwd': self.printWebPassword, + + 'friend': self.friendCmd } self.cmdhelp = { @@ -234,6 +236,7 @@ class Onionr: 'pex': 'exchange addresses with peers (done automatically)', 'blacklist-block': 'deletes a block by hash and permanently removes it from your node', 'introduce': 'Introduce your node to the public Onionr network', + 'friend': '[add|remove] [public key/id]' } # initialize plugins @@ -260,6 +263,41 @@ class Onionr: def getCommands(self): return self.cmds + + def friendCmd(self): + '''List, add, or remove friend(s)''' + friend = '' + try: + action = sys.argv[2] + except IndexError: + logger.info('Syntax: friend add/remove/list [address]') + else: + action = action.lower() + if action == 'list': + for friend in self.onionrCore.listPeers(randomOrder=False, trust=1): + logger.info(friend) + elif action in ('add', 'remove'): + try: + friend = sys.argv[3] + if not self.onionrUtils.validatePubKey(friend): + raise onionrexceptions.InvalidPubkey('Public key is invalid') + if friend not in self.onionrCore.listPeers(): + raise onionrexceptions.KeyNotKnown + friend = onionrusers.OnionrUser(self.onionrCore, friend) + except IndexError: + logger.error('Friend ID is required.') + except onionrexceptions.KeyNotKnown: + logger.error('That peer is not in our database') + else: + if action == 'add': + friend.setTrust(1) + logger.info('Added %s as friend.' % (friend.publicKey,)) + else: + friend.setTrust(0) + logger.info('Removed %s as friend.' % (friend.publicKey,)) + else: + logger.info('Syntax: friend add/remove/list [address]') + def banBlock(self): try: diff --git a/onionr/onionrblacklist.py b/onionr/onionrblacklist.py index 86823283..863ddc37 100644 --- a/onionr/onionrblacklist.py +++ b/onionr/onionrblacklist.py @@ -1,5 +1,5 @@ ''' - Onionr - P2P Microblogging Platform & Social network. + Onionr - P2P Anonymous Storage Network This file handles maintenence of a blacklist database, for blocks and peers ''' diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index e255c0ae..97b34730 100644 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -1,5 +1,5 @@ ''' - Onionr - P2P Microblogging Platform & Social network. + Onionr - P2P Anonymous Storage Network This class contains the OnionrBlocks class which is a class for working with Onionr blocks ''' diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index 00c5a604..9e055716 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -155,26 +155,6 @@ class OnionrCrypto: decrypted = anonBox.decrypt(data, encoder=encoding) return decrypted - def symmetricPeerEncrypt(self, data, peer): - '''Salsa20 encrypt data to peer (with mac) - this function does not accept a key, it is a wrapper for encryption with a peer - ''' - key = self._core.getPeerInfo(4) - if type(key) != bytes: - key = self._core.getPeerInfo(2) - encrypted = self.symmetricEncrypt(data, key, encodedKey=True) - return encrypted - - def symmetricPeerDecrypt(self, data, peer): - '''Salsa20 decrypt data from peer (with mac) - this function does not accept a key, it is a wrapper for encryption with a peer - ''' - key = self._core.getPeerInfo(4) - if type(key) != bytes: - key = self._core.getPeerInfo(2) - decrypted = self.symmetricDecrypt(data, key, encodedKey=True) - return decrypted - def symmetricEncrypt(self, data, key, encodedKey=False, returnEncoded=True): '''Encrypt data to a 32-byte key (Salsa20-Poly1305 MAC)''' if encodedKey: diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index 8410cb80..c7b738e1 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -1,5 +1,5 @@ ''' - Onionr - P2P Microblogging Platform & Social network. + Onionr - P2P Anonymous Storage Network Contains the CommunicatorUtils class which contains useful functions for the communicator daemon ''' diff --git a/onionr/onionrexceptions.py b/onionr/onionrexceptions.py index b26a97d7..dbdab69a 100644 --- a/onionr/onionrexceptions.py +++ b/onionr/onionrexceptions.py @@ -1,5 +1,5 @@ ''' - Onionr - P2P Microblogging Platform & Social network. + Onionr - P2P Anonymous Storage Network This file contains exceptions for onionr ''' @@ -34,6 +34,9 @@ class OnlinePeerNeeded(Exception): class InvalidPubkey(Exception): pass +class KeyNotKnown(Exception): + pass + # block exceptions class InvalidMetadata(Exception): pass diff --git a/onionr/onionrpeers.py b/onionr/onionrpeers.py index 710f698d..6a34adf7 100644 --- a/onionr/onionrpeers.py +++ b/onionr/onionrpeers.py @@ -1,5 +1,5 @@ ''' - Onionr - P2P Microblogging Platform & Social network. + Onionr - P2P Anonymous Storage Network This file contains both the PeerProfiles class for network profiling of Onionr nodes ''' diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py new file mode 100644 index 00000000..9138e1ce --- /dev/null +++ b/onionr/onionrusers.py @@ -0,0 +1,51 @@ +''' + Onionr - P2P Anonymous Storage Network + + Contains abstractions for interacting with users of Onionr +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +class OnionrUser: + def __init__(self, coreInst, publicKey): + self.trust = 0 + self._core = coreInst + self.publicKey = publicKey + + self.trust = self._core.getPeerInfo(self.publicKey, 'trust') + return + + def setTrust(self, newTrust): + '''Set the peers trust. 0 = not trusted, 1 = friend, 2 = ultimate''' + self._core.setPeerInfo(self.publicKey, 'trust', newTrust) + + def isFriend(self): + if self._core.getPeerInfo(self.publicKey, 6) == 1: + return True + return False + + def encrypt(self, data): + encrypted = coreInst._crypto.pubKeyEncrypt(data, self.publicKey, encodedData=True) + return encrypted + + def decrypt(self, data): + decrypted = coreInst._crypto.pubKeyDecrypt(data, self.publicKey, encodedData=True) + return decrypted + + def forwardEncrypt(self, data): + return + + def forwardDecrypt(self, encrypted): + return + \ No newline at end of file diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 6d22992c..8e8f5ab8 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -264,9 +264,24 @@ class OnionrUtils: if myBlock.isEncrypted: myBlock.decrypt() blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks + signer = myBlock.getSigner() + try: if len(blockType) <= 10: self._core.updateBlockInfo(blockHash, 'dataType', blockType) + + if blockType == 'userInfo': + if myBlock.verifySig(): + peerName = myBlock.getMetadata('name') + try: + if len(peerName) > 20: + raise onionrexceptions.InvalidMetdata('Peer name specified is too large') + except TypeError: + pass + except onionrexceptions.InvalidMetadata: + pass + else: + self._core.setPeerInfo(signer, 'name', peerName) except TypeError: pass diff --git a/onionr/static-data/index.html b/onionr/static-data/index.html index 93e48beb..a48dad37 100644 --- a/onionr/static-data/index.html +++ b/onionr/static-data/index.html @@ -1,6 +1,6 @@

This is an Onionr Node

-

The content on this server is not necessarily created by the server owner, and was not necessarily stored with the owner's knowledge.

+

The content on this server is not necessarily created by the server owner, and was not necessarily stored specifically with the owner's knowledge of its contents.

Onionr is a decentralized, distributed data storage system, that anyone can insert data into.

From c907558dd147d3a17ffa6aec3efbf04efd87a39b Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 27 Aug 2018 23:45:31 -0500 Subject: [PATCH 42/53] more work on user info --- onionr/onionr.py | 5 +- onionr/onionrusers.py | 12 ++++- onionr/onionrutils.py | 2 +- .../default-plugins/cliui/info.json | 5 ++ .../static-data/default-plugins/cliui/main.py | 51 +++++++++++++++++++ 5 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 onionr/static-data/default-plugins/cliui/info.json create mode 100644 onionr/static-data/default-plugins/cliui/main.py diff --git a/onionr/onionr.py b/onionr/onionr.py index 5408b2c4..333340ad 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -275,7 +275,10 @@ class Onionr: action = action.lower() if action == 'list': for friend in self.onionrCore.listPeers(randomOrder=False, trust=1): - logger.info(friend) + if friend == self.onionrCore._crypto.pubKey: + continue + friendProfile = onionrusers.OnionrUser(self.onionrCore, friend) + logger.info(friend + ' - ' + friendProfile.getName()) elif action in ('add', 'remove'): try: friend = sys.argv[3] diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index 9138e1ce..d258a8da 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -31,9 +31,19 @@ class OnionrUser: self._core.setPeerInfo(self.publicKey, 'trust', newTrust) def isFriend(self): - if self._core.getPeerInfo(self.publicKey, 6) == 1: + if self._core.getPeerInfo(self.publicKey, 'trust') == 1: return True return False + + def getName(self): + retData = 'anonymous' + name = self._core.getPeerInfo(self.publicKey, 'name') + try: + if len(name) > 0: + retData = name + except ValueError: + pass + return retData def encrypt(self, data): encrypted = coreInst._crypto.pubKeyEncrypt(data, self.publicKey, encodedData=True) diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 8e8f5ab8..81ebf44a 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -264,7 +264,7 @@ class OnionrUtils: if myBlock.isEncrypted: myBlock.decrypt() blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks - signer = myBlock.getSigner() + signer = myBlock.signer try: if len(blockType) <= 10: diff --git a/onionr/static-data/default-plugins/cliui/info.json b/onionr/static-data/default-plugins/cliui/info.json new file mode 100644 index 00000000..e06855f6 --- /dev/null +++ b/onionr/static-data/default-plugins/cliui/info.json @@ -0,0 +1,5 @@ +{ + "name" : "cliui", + "version" : "1.0", + "author" : "onionr" +} diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py new file mode 100644 index 00000000..40bcfd6f --- /dev/null +++ b/onionr/static-data/default-plugins/cliui/main.py @@ -0,0 +1,51 @@ +''' + Onionr - P2P Anonymous Storage Network + + This is an interactive menu-driven CLI interface for Onionr +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' + +# Imports some useful libraries +import logger, config, threading, time +from onionrblockapi import Block + +plugin_name = 'cliui' +PLUGIN_VERSION = '0.0.1' + +class OnionrCLIUI: + def __init__(self, apiInst): + self.api = apiInst + self.myCore = apiInst.get_core() + return + def start(self): + name = input("Enter your name") + self.myCore.insertBlock("userInfo", sign=True, header='userInfo', meta={'name': name}) + return + +def on_init(api, data = None): + ''' + This event is called after Onionr is initialized, but before the command + inputted is executed. Could be called when daemon is starting or when + just the client is running. + ''' + + # Doing this makes it so that the other functions can access the api object + # by simply referencing the variable `pluginapi`. + pluginapi = api + ui = OnionrCLIUI(api) + api.commands.register('interactive', ui.start) + api.commands.register_help('interactive', 'Open the CLI interface') + return From c0c0f838b6371ca8d6818104ad519a56ee1e591a Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 28 Aug 2018 20:09:27 -0500 Subject: [PATCH 43/53] better user info syncing and show names in mail --- .gitmodules | 3 --- onionr/onionr.py | 8 ++++++-- onionr/onionrusers.py | 14 +++++++++++++- onionr/onionrutils.py | 5 +++-- onionr/static-data/default-plugins/cliui/main.py | 4 ++-- onionr/static-data/default-plugins/pms/main.py | 11 +++++++++-- 6 files changed, 33 insertions(+), 12 deletions(-) delete mode 100644 .gitmodules diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index f2ab3397..00000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "onionr/bitpeer"] - path = onionr/bitpeer - url = https://github.com/beardog108/bitpeer.py diff --git a/onionr/onionr.py b/onionr/onionr.py index 333340ad..dcf76318 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -265,17 +265,21 @@ class Onionr: return self.cmds def friendCmd(self): - '''List, add, or remove friend(s)''' + '''List, add, or remove friend(s) + Changes their peer DB entry. + ''' friend = '' try: + # Get the friend command action = sys.argv[2] except IndexError: logger.info('Syntax: friend add/remove/list [address]') else: action = action.lower() if action == 'list': + # List out peers marked as our friend for friend in self.onionrCore.listPeers(randomOrder=False, trust=1): - if friend == self.onionrCore._crypto.pubKey: + if friend == self.onionrCore._crypto.pubKey: # do not list our key continue friendProfile = onionrusers.OnionrUser(self.onionrCore, friend) logger.info(friend + ' - ' + friendProfile.getName()) diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index d258a8da..03807ea9 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' +import onionrblockapi, logger class OnionrUser: def __init__(self, coreInst, publicKey): self.trust = 0 @@ -58,4 +59,15 @@ class OnionrUser: def forwardDecrypt(self, encrypted): return - \ No newline at end of file + + def findAndSetID(self): + '''Find any info about the user from existing blocks and cache it to their DB entry''' + infoBlocks = [] + for bHash in self._core.getBlocksByType('userInfo'): + block = onionrblockapi.Block(bHash, core=self._core) + if block.signer == self.publicKey: + if block.verifySig(): + newName = block.getMetadata('name') + if newName.isalnum(): + logger.info('%s is now using the name %s.' % (self.publicKey, newName)) + self._core.setPeerInfo(self.publicKey, 'name', newName) \ No newline at end of file diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 81ebf44a..075077e5 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -23,7 +23,7 @@ import nacl.signing, nacl.encoding from onionrblockapi import Block import onionrexceptions from defusedxml import minidom -import pgpwords +import pgpwords, onionrusers if sys.version_info < (3, 6): try: import sha3 @@ -95,7 +95,6 @@ class OnionrUtils: except IndexError: logger.warn('No pow token') continue - #powHash = self._core._crypto.blake2bHash(base64.b64decode(key[1]) + self._core._crypto.blake2bHash(key[0].encode())) value = base64.b64decode(key[1]) hashedKey = self._core._crypto.blake2bHash(key[0]) powHash = self._core._crypto.blake2bHash(value + hashedKey) @@ -106,6 +105,7 @@ class OnionrUtils: if powHash.startswith(b'0000'): if not key[0] in self._core.listPeers(randomOrder=False) and type(key) != None and key[0] != self._core._crypto.pubKey: if self._core.addPeer(key[0], key[1]): + onionrusers.OnionrUser(self._core, key[0]).findAndSetID() retVal = True else: logger.warn("Failed to add key") @@ -282,6 +282,7 @@ class OnionrUtils: pass else: self._core.setPeerInfo(signer, 'name', peerName) + logger.info('%s is now using the name %s.' % (signer, peerName)) except TypeError: pass diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py index 40bcfd6f..1d0db8b0 100644 --- a/onionr/static-data/default-plugins/cliui/main.py +++ b/onionr/static-data/default-plugins/cliui/main.py @@ -19,7 +19,7 @@ ''' # Imports some useful libraries -import logger, config, threading, time +import logger, config, threading, time, uuid from onionrblockapi import Block plugin_name = 'cliui' @@ -32,7 +32,7 @@ class OnionrCLIUI: return def start(self): name = input("Enter your name") - self.myCore.insertBlock("userInfo", sign=True, header='userInfo', meta={'name': name}) + self.myCore.insertBlock("userInfo-" + str(uuid.uuid1()), sign=True, header='userInfo', meta={'name': name}) return def on_init(api, data = None): diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index 27f56438..dc57a82c 100644 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -21,7 +21,7 @@ # Imports some useful libraries import logger, config, threading, time, readline, datetime from onionrblockapi import Block -import onionrexceptions +import onionrexceptions, onionrusers import locale locale.setlocale(locale.LC_ALL, '') @@ -81,8 +81,15 @@ class OnionrMail: continue blockCount += 1 pmBlockMap[blockCount] = blockHash + + block = Block(blockHash, core=self.myCore) + senderKey = block.getMetadata('signer') + senderDisplay = onionrusers.OnionrUser(self.myCore, senderKey) + if senderDisplay == 'anonymous': + senderDisplay = senderKey + blockDate = pmBlocks[blockHash].getDate().strftime("%m/%d %H:%M") - print('%s. %s: %s' % (blockCount, blockDate, blockHash)) + print('%s. %s - %s: %s' % (blockCount, blockDate, senderDisplay, blockHash)) try: choice = logger.readline('Enter a block number, -r to refresh, or -q to stop: ').strip().lower() From f4cc1a6f8f195112354a8c29bebdbf9a853fa530 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 28 Aug 2018 22:02:32 -0500 Subject: [PATCH 44/53] updated dockerfile, fixed bug in showing name in mail --- Dockerfile | 2 +- onionr/onionrusers.py | 9 ++++++--- onionr/onionrutils.py | 10 ++++++---- onionr/static-data/default-plugins/cliui/main.py | 2 +- onionr/static-data/default-plugins/pms/main.py | 11 ++++++++--- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index c83de87d..e6132726 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ FROM ubuntu:bionic ENV HOME /root #Install needed packages -RUN apt update && apt install -y python3 python3-dev python3-pip tor locales nano +RUN apt update && apt install -y python3 python3-dev python3-pip tor locales nano sqlite3 RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ locale-gen diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index 03807ea9..208419aa 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import onionrblockapi, logger +import onionrblockapi, logger, onionrexceptions class OnionrUser: def __init__(self, coreInst, publicKey): self.trust = 0 @@ -69,5 +69,8 @@ class OnionrUser: if block.verifySig(): newName = block.getMetadata('name') if newName.isalnum(): - logger.info('%s is now using the name %s.' % (self.publicKey, newName)) - self._core.setPeerInfo(self.publicKey, 'name', newName) \ No newline at end of file + logger.info('%s is now using the name %s.' % (self.publicKey, self._core._utils.escapeAnsi(newName))) + self._core.setPeerInfo(self.publicKey, 'name', newName) + print("DEBUG PLS") + else: + raise onionrexceptions.InvalidPubkey \ No newline at end of file diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 075077e5..29d3f350 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -95,7 +95,10 @@ class OnionrUtils: except IndexError: logger.warn('No pow token') continue - value = base64.b64decode(key[1]) + try: + value = base64.b64decode(key[1]) + except binascii.Error: + continue hashedKey = self._core._crypto.blake2bHash(key[0]) powHash = self._core._crypto.blake2bHash(value + hashedKey) try: @@ -264,8 +267,7 @@ class OnionrUtils: if myBlock.isEncrypted: myBlock.decrypt() blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks - signer = myBlock.signer - + signer = self.bytesToStr(myBlock.signer) try: if len(blockType) <= 10: self._core.updateBlockInfo(blockHash, 'dataType', blockType) @@ -282,7 +284,7 @@ class OnionrUtils: pass else: self._core.setPeerInfo(signer, 'name', peerName) - logger.info('%s is now using the name %s.' % (signer, peerName)) + logger.info('%s is now using the name %s.' % (signer, self.escapeAnsi(peerName))) except TypeError: pass diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py index 1d0db8b0..4d4adb3d 100644 --- a/onionr/static-data/default-plugins/cliui/main.py +++ b/onionr/static-data/default-plugins/cliui/main.py @@ -31,7 +31,7 @@ class OnionrCLIUI: self.myCore = apiInst.get_core() return def start(self): - name = input("Enter your name") + name = input("Enter your name: ") self.myCore.insertBlock("userInfo-" + str(uuid.uuid1()), sign=True, header='userInfo', meta={'name': name}) return diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index dc57a82c..46a614b7 100644 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -82,9 +82,14 @@ class OnionrMail: blockCount += 1 pmBlockMap[blockCount] = blockHash - block = Block(blockHash, core=self.myCore) - senderKey = block.getMetadata('signer') - senderDisplay = onionrusers.OnionrUser(self.myCore, senderKey) + block = pmBlocks[blockHash] + senderKey = block.signer + try: + senderKey = senderKey.decode() + except AttributeError: + pass + print("DEBUG:", senderKey) + senderDisplay = onionrusers.OnionrUser(self.myCore, senderKey).getName() if senderDisplay == 'anonymous': senderDisplay = senderKey From 53505832cc8d6a9f2e7aa89d790eab921c86f596 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Wed, 29 Aug 2018 23:05:38 -0500 Subject: [PATCH 45/53] preparing for config merge --- onionr/communicator2.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 9c44ed21..9e4c70f5 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -84,7 +84,6 @@ class OnionrCommunicatorDaemon: # Set timers, function reference, seconds # requiresPeer True means the timer function won't fire if we have no connected peers - # TODO: make some of these timer counts configurable OnionrCommunicatorTimers(self, self.daemonCommands, 5) OnionrCommunicatorTimers(self, self.detectAPICrash, 5) peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60) @@ -277,7 +276,7 @@ class OnionrCommunicatorDaemon: '''Manages the self.onlinePeers attribute list, connects to more peers if we have none connected''' logger.info('Refreshing peer pool.') - maxPeers = 6 + maxPeers = config.get('peers.maxConnect') needed = maxPeers - len(self.onlinePeers) for i in range(needed): From 716fb7335c4f61bf9a160097f97b76f7d9f9ccbd Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Wed, 29 Aug 2018 23:49:28 -0500 Subject: [PATCH 46/53] removed debug statements --- onionr/onionrpeers.py | 2 +- onionr/onionrusers.py | 1 - onionr/static-data/default-plugins/pms/main.py | 3 +-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/onionr/onionrpeers.py b/onionr/onionrpeers.py index 207f5825..041a69b9 100644 --- a/onionr/onionrpeers.py +++ b/onionr/onionrpeers.py @@ -90,7 +90,7 @@ def peerCleanup(coreInst): if PeerProfiles(address, coreInst).score < minScore: coreInst.removeAddress(address) try: - if (coreInst._utils.getEpoch() - coreInst.getPeerInfo(address, 4)) >= 600: + if (coreInst._utils.getEpoch() - coreInst.getPeerInfo(address, 'dateSeen')) >= 600: expireTime = 600 else: expireTime = 86400 diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index 208419aa..29b9375b 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -71,6 +71,5 @@ class OnionrUser: if newName.isalnum(): logger.info('%s is now using the name %s.' % (self.publicKey, self._core._utils.escapeAnsi(newName))) self._core.setPeerInfo(self.publicKey, 'name', newName) - print("DEBUG PLS") else: raise onionrexceptions.InvalidPubkey \ No newline at end of file diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index 46a614b7..f569ec9b 100644 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -88,13 +88,12 @@ class OnionrMail: senderKey = senderKey.decode() except AttributeError: pass - print("DEBUG:", senderKey) senderDisplay = onionrusers.OnionrUser(self.myCore, senderKey).getName() if senderDisplay == 'anonymous': senderDisplay = senderKey blockDate = pmBlocks[blockHash].getDate().strftime("%m/%d %H:%M") - print('%s. %s - %s: %s' % (blockCount, blockDate, senderDisplay, blockHash)) + print('%s. %s - %s: %s' % (blockCount, blockDate, senderDisplay[:12], blockHash)) try: choice = logger.readline('Enter a block number, -r to refresh, or -q to stop: ').strip().lower() From 1b16c809fd318e22980915a5a86a921ad69be10e Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 31 Aug 2018 17:53:48 -0500 Subject: [PATCH 47/53] work on user connections --- onionr/communicator2.py | 23 +++++++++++++++++++---- onionr/onionrdaemontools.py | 35 +++++++++++++++++++++++++++++++++-- onionr/onionrpeers.py | 4 +++- 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 168499ee..2b8667be 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -51,6 +51,8 @@ class OnionrCommunicatorDaemon: # lists of connected peers and peers we know we can't reach currently self.onlinePeers = [] self.offlinePeers = [] + self.cooldownPeer = {} + self.connectTimes = {} self.peerProfiles = [] # list of peer's profiles (onionrpeers.PeerProfile instances) # amount of threads running by name, used to prevent too many @@ -84,13 +86,14 @@ class OnionrCommunicatorDaemon: # requiresPeer True means the timer function won't fire if we have no connected peers OnionrCommunicatorTimers(self, self.daemonCommands, 5) OnionrCommunicatorTimers(self, self.detectAPICrash, 5) - peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60) + peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60, maxThreads=1) OnionrCommunicatorTimers(self, self.lookupBlocks, self._core.config.get('timers.lookupBlocks'), requiresPeer=True, maxThreads=1) OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.getBlocks'), requiresPeer=True) OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58) OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 65) OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True) OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True) + OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True) netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 305, requiresPeer=True, maxThreads=1) cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requiresPeer=True) @@ -295,7 +298,7 @@ class OnionrCommunicatorDaemon: '''Manages the self.onlinePeers attribute list, connects to more peers if we have none connected''' logger.info('Refreshing peer pool.') - maxPeers = config.get('peers.maxConnect') + maxPeers = int(config.get('peers.maxConnect')) needed = maxPeers - len(self.onlinePeers) for i in range(needed): @@ -338,7 +341,7 @@ class OnionrCommunicatorDaemon: for address in peerList: if not config.get('tor.v3onions') and len(address) == 62: continue - if len(address) == 0 or address in tried or address in self.onlinePeers: + if len(address) == 0 or address in tried or address in self.onlinePeers or address in self.cooldownPeer: continue if self.shutdown: return @@ -347,6 +350,7 @@ class OnionrCommunicatorDaemon: time.sleep(0.1) if address not in self.onlinePeers: self.onlinePeers.append(address) + self.connectTimes[address] = self._core._utils.getEpoch() retData = address # add peer to profile list if they're not in it @@ -361,6 +365,17 @@ class OnionrCommunicatorDaemon: logger.debug('Failed to connect to ' + address) return retData + def removeOnlinePeer(self, peer): + '''Remove an online peer''' + try: + del self.connectTimes[peer] + except KeyError: + pass + try: + self.onlinePeers.remove(peer) + except ValueError: + pass + def peerCleanup(self): '''This just calls onionrpeers.cleanupPeers, which removes dead or bad peers (offline too long, too slow)''' onionrpeers.peerCleanup(self._core) @@ -392,7 +407,7 @@ class OnionrCommunicatorDaemon: if retData == False: try: self.getPeerProfileInstance(peer).addScore(-10) - self.onlinePeers.remove(peer) + self.removeOnlinePeer(peer) self.getOnlinePeers() # Will only add a new peer to pool if needed except ValueError: pass diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index 7c6fc62e..bc3e7d9e 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import onionrexceptions, onionrpeers, onionrproofs, base64, logger +import onionrexceptions, onionrpeers, onionrproofs, base64, logger, secrets class DaemonTools: def __init__(self, daemon): self.daemon = daemon @@ -70,4 +70,35 @@ class DaemonTools: self.daemon._core._blacklist.addToDB(oldest) self.daemon._core.removeBlock(oldest) logger.info('Deleted block: %s' % (oldest,)) - self.daemon.decrementThreadCount('cleanOldBlocks') \ No newline at end of file + self.daemon.decrementThreadCount('cleanOldBlocks') + + def cooldownPeer(self): + '''Randomly add an online peer to cooldown, so we can connect a new one''' + onlinePeerAmount = len(self.daemon.onlinePeers) + minTime = 300 + cooldownTime = 600 + toCool = '' + tempConnectTimes = dict(self.daemon.connectTimes) + + # Remove peers from cooldown that have been there long enough + tempCooldown = dict(self.daemon.cooldownPeer) + for peer in tempCooldown: + if (self.daemon._core._utils.getEpoch() - tempCooldown[peer]) >= cooldownTime: + del self.daemon.cooldownPeer[peer] + + # Cool down a peer, if we have max connections alive for long enough + if onlinePeerAmount >= self.daemon._core.config.get('peers.maxConnect'): + finding = True + while finding: + try: + toCool = min(tempConnectTimes, key=tempConnectTimes.get) + if (self.daemon._core._utils.getEpoch() - tempConnectTimes[toCool]) < minTime: + del tempConnectTimes[toCool] + else: + finding = False + except ValueError: + break + else: + self.daemon.removeOnlinePeer(toCool) + self.daemon.cooldownPeer[toCool] = self.daemon._core._utils.getEpoch() + self.daemon.decrementThreadCount('cooldownPeer') \ No newline at end of file diff --git a/onionr/onionrpeers.py b/onionr/onionrpeers.py index 041a69b9..322db9ba 100644 --- a/onionr/onionrpeers.py +++ b/onionr/onionrpeers.py @@ -90,13 +90,15 @@ def peerCleanup(coreInst): if PeerProfiles(address, coreInst).score < minScore: coreInst.removeAddress(address) try: - if (coreInst._utils.getEpoch() - coreInst.getPeerInfo(address, 'dateSeen')) >= 600: + if (int(coreInst._utils.getEpoch()) - int(coreInst.getPeerInfo(address, 'dateSeen'))) >= 600: expireTime = 600 else: expireTime = 86400 coreInst._blacklist.addToDB(address, dataType=1, expire=expireTime) except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue pass + except ValueError: + pass logger.warn('Removed address ' + address + '.') # Unban probably not malicious peers TODO improve From bed6475e1b5272166b8f48f69bb58db5055b0e49 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 31 Aug 2018 19:51:14 -0500 Subject: [PATCH 48/53] use secrets from dependencies --- onionr/onionrdaemontools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index bc3e7d9e..bbd7af64 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -17,7 +17,8 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import onionrexceptions, onionrpeers, onionrproofs, base64, logger, secrets +import onionrexceptions, onionrpeers, onionrproofs, base64, logger +from dependencies import secrets class DaemonTools: def __init__(self, daemon): self.daemon = daemon From 1203bb2b7a5d672dbc06335bb40cdf06165ed6e9 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 31 Aug 2018 22:29:57 -0500 Subject: [PATCH 49/53] * Adjusted connect configuration * work on cli-ui --- onionr/communicator2.py | 3 +- .../static-data/default-plugins/cliui/main.py | 37 ++++++++++++++++++- onionr/static-data/default_config.json | 2 +- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 2b8667be..f203dd70 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -408,7 +408,8 @@ class OnionrCommunicatorDaemon: try: self.getPeerProfileInstance(peer).addScore(-10) self.removeOnlinePeer(peer) - self.getOnlinePeers() # Will only add a new peer to pool if needed + if action != 'ping': + self.getOnlinePeers() # Will only add a new peer to pool if needed except ValueError: pass else: diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py index 4d4adb3d..dcf370b5 100644 --- a/onionr/static-data/default-plugins/cliui/main.py +++ b/onionr/static-data/default-plugins/cliui/main.py @@ -19,7 +19,7 @@ ''' # Imports some useful libraries -import logger, config, threading, time, uuid +import logger, config, threading, time, uuid, subprocess from onionrblockapi import Block plugin_name = 'cliui' @@ -30,7 +30,42 @@ class OnionrCLIUI: self.api = apiInst self.myCore = apiInst.get_core() return + + def subCommand(self, command): + try: + subprocess.run(["./onionr.py", command]) + except KeyboardInterrupt: + pass + def start(self): + '''Main CLI UI interface menu''' + showMenu = True + while showMenu: + print('''\n1. Flow (Anonymous public chat, use at your own risk) +2. Mail (Secure email-like service) +3. File Sharing +4. User Settings +5. Quit + ''') + try: + choice = input(">").strip().lower() + except KeyboardInterrupt: + choice = "quit" + + if choice in ("flow", "1"): + self.subCommand("flow") + elif choice in ("2", "mail"): + self.subCommand("mail") + elif choice in ("4", "user settings", "settings"): + try: + self.setName() + except (KeyboardInterrupt, EOFError) as e: + pass + elif choice in ("5", "quit"): + showMenu = False + return + + def setName(self): name = input("Enter your name: ") self.myCore.insertBlock("userInfo-" + str(uuid.uuid1()), sign=True, header='userInfo', meta={'name': name}) return diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 6e35245e..5ebcfa4a 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -59,7 +59,7 @@ "peers":{ "minimumScore": -100, "maxStoredPeers": 5000, - "maxConnect": 5 + "maxConnect": 10 }, "timers":{ "lookupBlocks": 25, From 9fd985ea49f25b25eb7ffd57f59fbd6bcd662951 Mon Sep 17 00:00:00 2001 From: Kevin Date: Sat, 1 Sep 2018 15:38:44 -0500 Subject: [PATCH 50/53] work on the cliui --- .../static-data/default-plugins/cliui/main.py | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py index dcf370b5..0f0953c2 100644 --- a/onionr/static-data/default-plugins/cliui/main.py +++ b/onionr/static-data/default-plugins/cliui/main.py @@ -37,19 +37,37 @@ class OnionrCLIUI: except KeyboardInterrupt: pass + def refresh(self): + for i in range(100): + print('') + def start(self): '''Main CLI UI interface menu''' showMenu = True + isOnline = "No" + firstRun = True + while showMenu: - print('''\n1. Flow (Anonymous public chat, use at your own risk) + if firstRun: + print("please wait while Onionr starts...") + subprocess.Popen(["./onionr.py", "start"], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + time.sleep(30) + firstRun = False + + if self.myCore._utils.localCommand('ping') == 'pong': + isOnline = "Yes" + print('''Online Status: ''' + isOnline + ''' + +1. Flow (Anonymous public chat, use at your own risk) 2. Mail (Secure email-like service) 3. File Sharing 4. User Settings -5. Quit +5. Start/Stop Daemon +6. Quit ''') try: choice = input(">").strip().lower() - except KeyboardInterrupt: + except (KeyboardInterrupt, EOFError): choice = "quit" if choice in ("flow", "1"): @@ -61,8 +79,20 @@ class OnionrCLIUI: self.setName() except (KeyboardInterrupt, EOFError) as e: pass - elif choice in ("5", "quit"): + elif choice in ("5", "daemon"): + if isOnline == "Yes": + print("Onionr daemon will shutdown...") + #self.myCore._utils.localCommand("shutdown") + self.myCore.daemonQueueAdd('shutdown') + else: + print("Starting Daemon...") + subprocess.Popen(["./onionr.py", "start"], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + elif choice in ("6", "quit"): showMenu = False + elif choice == "": + pass + else: + print("Invalid choice") return def setName(self): From c142ab770a03668bf76d0b9e217795d3effbdb83 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 1 Sep 2018 21:59:03 -0500 Subject: [PATCH 51/53] work on cliui --- onionr/core.py | 13 ++++++---- .../static-data/default-plugins/cliui/main.py | 24 +++++++++++++++---- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index ca9e6c7f..9cd06043 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -377,18 +377,21 @@ class Core: ''' Add a command to the daemon queue, used by the communication daemon (communicator.py) ''' + retData = True # Intended to be used by the web server date = self._utils.getEpoch() conn = sqlite3.connect(self.queueDB) c = conn.cursor() t = (command, data, date) - c.execute('INSERT INTO commands (command, data, date) VALUES(?, ?, ?)', t) - conn.commit() - conn.close() - + try: + c.execute('INSERT INTO commands (command, data, date) VALUES(?, ?, ?)', t) + conn.commit() + conn.close() + except sqlite3.OperationalError: + retData = False events.event('queue_push', data = {'command': command, 'data': data}, onionr = None) - return + return retData def clearDaemonQueue(self): ''' diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py index 0f0953c2..81af13c2 100644 --- a/onionr/static-data/default-plugins/cliui/main.py +++ b/onionr/static-data/default-plugins/cliui/main.py @@ -47,6 +47,9 @@ class OnionrCLIUI: isOnline = "No" firstRun = True + if self.myCore._utils.localCommand('ping') == 'pong': + firstRun = False + while showMenu: if firstRun: print("please wait while Onionr starts...") @@ -56,14 +59,18 @@ class OnionrCLIUI: if self.myCore._utils.localCommand('ping') == 'pong': isOnline = "Yes" - print('''Online Status: ''' + isOnline + ''' + else: + isOnline = "No" + + print(''' +Daemon Running: ''' + isOnline + ''' 1. Flow (Anonymous public chat, use at your own risk) 2. Mail (Secure email-like service) 3. File Sharing 4. User Settings 5. Start/Stop Daemon -6. Quit +6. Quit (Does not shutdown daemon) ''') try: choice = input(">").strip().lower() @@ -74,6 +81,8 @@ class OnionrCLIUI: self.subCommand("flow") elif choice in ("2", "mail"): self.subCommand("mail") + elif choice in ("3", "file sharing", "file"): + print("Not supported yet") elif choice in ("4", "user settings", "settings"): try: self.setName() @@ -84,6 +93,9 @@ class OnionrCLIUI: print("Onionr daemon will shutdown...") #self.myCore._utils.localCommand("shutdown") self.myCore.daemonQueueAdd('shutdown') + while self.myCore._utils.localCommand('ping') == 'pong': + self.myCore.daemonQueueAdd('shutdown') + time.sleep(8) else: print("Starting Daemon...") subprocess.Popen(["./onionr.py", "start"], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) @@ -96,8 +108,12 @@ class OnionrCLIUI: return def setName(self): - name = input("Enter your name: ") - self.myCore.insertBlock("userInfo-" + str(uuid.uuid1()), sign=True, header='userInfo', meta={'name': name}) + try: + name = input("Enter your name: ") + if name != "": + self.myCore.insertBlock("userInfo-" + str(uuid.uuid1()), sign=True, header='userInfo', meta={'name': name}) + except KeyboardInterrupt: + pass return def on_init(api, data = None): From 083003191f2dc1c36464ef3dd767adf9adbc5da6 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 1 Sep 2018 23:55:24 -0500 Subject: [PATCH 52/53] work on cliui --- onionr/core.py | 1 + onionr/static-data/default-plugins/cliui/main.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index 9cd06043..ab8b640b 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -389,6 +389,7 @@ class Core: conn.close() except sqlite3.OperationalError: retData = False + self.daemonQueue() events.event('queue_push', data = {'command': command, 'data': data}, onionr = None) return retData diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py index 81af13c2..961bbddb 100644 --- a/onionr/static-data/default-plugins/cliui/main.py +++ b/onionr/static-data/default-plugins/cliui/main.py @@ -53,7 +53,7 @@ class OnionrCLIUI: while showMenu: if firstRun: print("please wait while Onionr starts...") - subprocess.Popen(["./onionr.py", "start"], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + daemon = subprocess.Popen(["./onionr.py", "start"], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) time.sleep(30) firstRun = False @@ -93,12 +93,13 @@ Daemon Running: ''' + isOnline + ''' print("Onionr daemon will shutdown...") #self.myCore._utils.localCommand("shutdown") self.myCore.daemonQueueAdd('shutdown') - while self.myCore._utils.localCommand('ping') == 'pong': - self.myCore.daemonQueueAdd('shutdown') - time.sleep(8) + try: + daemon.kill() + except UnboundLocalError: + pass else: print("Starting Daemon...") - subprocess.Popen(["./onionr.py", "start"], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + daemon = subprocess.Popen(["./onionr.py", "start"], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) elif choice in ("6", "quit"): showMenu = False elif choice == "": From 1318bb3ea6cdecfed16c40aecd7cd34327eee154 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 3 Sep 2018 10:28:08 -0500 Subject: [PATCH 53/53] use new gevent version to support python 3.7 --- onionr/api.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index d0407388..adfabf88 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -20,7 +20,7 @@ import flask from flask import request, Response, abort, send_from_directory from multiprocessing import Process -from gevent.wsgi import WSGIServer +from gevent.pywsgi import WSGIServer import sys, random, threading, hmac, hashlib, base64, time, math, os, json from core import Core from onionrblockapi import Block diff --git a/requirements.txt b/requirements.txt index 4653eaf0..97f8969e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ urllib3==1.23 requests==2.18.4 PyNaCl==1.2.1 -gevent==1.2.2 +gevent==1.3.6 sha3==0.2.1 defusedxml==0.5.0 simple_crypt==4.1.7