From 8cbc16224d27f067b5d3b6734793846bf43b6156 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 2 Sep 2018 15:18:53 -0500 Subject: [PATCH 01/70] work on cliui --- onionr/static-data/default-plugins/cliui/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py index 961bbddb..4095f814 100644 --- a/onionr/static-data/default-plugins/cliui/main.py +++ b/onionr/static-data/default-plugins/cliui/main.py @@ -91,7 +91,6 @@ Daemon Running: ''' + isOnline + ''' elif choice in ("5", "daemon"): if isOnline == "Yes": print("Onionr daemon will shutdown...") - #self.myCore._utils.localCommand("shutdown") self.myCore.daemonQueueAdd('shutdown') try: daemon.kill() From 6b33749b37d078b8a9123078d65c9cc96c4c9ea0 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 2 Sep 2018 15:19:27 -0500 Subject: [PATCH 02/70] adding board plugin --- .../default-plugins/boards/info.json | 5 ++ .../default-plugins/boards/main.py | 62 +++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 onionr/static-data/default-plugins/boards/info.json create mode 100644 onionr/static-data/default-plugins/boards/main.py diff --git a/onionr/static-data/default-plugins/boards/info.json b/onionr/static-data/default-plugins/boards/info.json new file mode 100644 index 00000000..f3e80e1c --- /dev/null +++ b/onionr/static-data/default-plugins/boards/info.json @@ -0,0 +1,5 @@ +{ + "name" : "boards", + "version" : "1.0", + "author" : "onionr" +} diff --git a/onionr/static-data/default-plugins/boards/main.py b/onionr/static-data/default-plugins/boards/main.py new file mode 100644 index 00000000..61d66378 --- /dev/null +++ b/onionr/static-data/default-plugins/boards/main.py @@ -0,0 +1,62 @@ +''' + Onionr - P2P Anonymous Storage Network + + This is an interactive menu-driven CLI interface for Onionr +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' + +# Imports some useful libraries +import logger, config, sys +from onionrblockapi import Block +try: + import tkinter +except (ModuleNotFoundError, ImportError) as e: + TK_ENABLED = False +else: + TK_ENABLED = True + + +plugin_name = 'cliui' +PLUGIN_VERSION = '0.0.1' + +class OnionrBoards: + def __init__(self, apiInst): + self.api = apiInst + self.myCore = apiInst.get_core() + + self.gui = tkinter.Tk() + + return + + def start(self): + return + +def on_init(api, data = None): + ''' + This event is called after Onionr is initialized, but before the command + inputted is executed. Could be called when daemon is starting or when + just the client is running. + ''' + + # Doing this makes it so that the other functions can access the api object + # by simply referencing the variable `pluginapi`. + pluginapi = api + ui = OnionrBoards(api) + api.commands.register('boards', ui.start) + api.commands.register_help('boards', 'Open the board viewer') + + + return From da3e0fdc4ec3ddf53f660839fa1352d00953f210 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 3 Sep 2018 22:28:56 -0500 Subject: [PATCH 03/70] better catch tkinter import --- onionr/static-data/default-plugins/boards/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/static-data/default-plugins/boards/main.py b/onionr/static-data/default-plugins/boards/main.py index 61d66378..3f00098a 100644 --- a/onionr/static-data/default-plugins/boards/main.py +++ b/onionr/static-data/default-plugins/boards/main.py @@ -23,7 +23,7 @@ import logger, config, sys from onionrblockapi import Block try: import tkinter -except (ModuleNotFoundError, ImportError) as e: +except (ModuleNotFoundError, ImportError, NameError) as e: TK_ENABLED = False else: TK_ENABLED = True From 0050b60f1a4ac951eb25aff0e8ba15b9d30da541 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 3 Sep 2018 22:30:15 -0500 Subject: [PATCH 04/70] better catch tkinter import --- onionr/static-data/default-plugins/boards/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/static-data/default-plugins/boards/main.py b/onionr/static-data/default-plugins/boards/main.py index 3f00098a..1b8b71b2 100644 --- a/onionr/static-data/default-plugins/boards/main.py +++ b/onionr/static-data/default-plugins/boards/main.py @@ -23,7 +23,7 @@ import logger, config, sys from onionrblockapi import Block try: import tkinter -except (ModuleNotFoundError, ImportError, NameError) as e: +except (ImportError, NameError) as e: TK_ENABLED = False else: TK_ENABLED = True From c1d4040807a11aa2fab6b0b668fa2c76e242f49e Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 3 Sep 2018 22:38:08 -0500 Subject: [PATCH 05/70] better catch tkinter import --- onionr/static-data/default-plugins/boards/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/onionr/static-data/default-plugins/boards/main.py b/onionr/static-data/default-plugins/boards/main.py index 1b8b71b2..573e6237 100644 --- a/onionr/static-data/default-plugins/boards/main.py +++ b/onionr/static-data/default-plugins/boards/main.py @@ -37,7 +37,8 @@ class OnionrBoards: self.api = apiInst self.myCore = apiInst.get_core() - self.gui = tkinter.Tk() + if TK_ENABLED: + self.gui = tkinter.Tk() return From cf37823fd7747c1d3acae3c1a2866196e0234060 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 4 Sep 2018 13:56:05 -0500 Subject: [PATCH 06/70] removed board plugin for now, added getfile command --- RUN-LINUX.sh | 1 + onionr/onionr.py | 29 ++++++++- onionr/onionrblockapi.py | 1 + onionr/onionrusers.py | 2 +- .../default-plugins/boards/info.json | 5 -- .../default-plugins/boards/main.py | 63 ------------------- 6 files changed, 30 insertions(+), 71 deletions(-) delete mode 100644 onionr/static-data/default-plugins/boards/info.json delete mode 100644 onionr/static-data/default-plugins/boards/main.py diff --git a/RUN-LINUX.sh b/RUN-LINUX.sh index 8f9a4b37..286a0f7f 100755 --- a/RUN-LINUX.sh +++ b/RUN-LINUX.sh @@ -1,3 +1,4 @@ #!/bin/sh +cd "$(dirname "$0")" cd onionr/ ./onionr.py "$@" diff --git a/onionr/onionr.py b/onionr/onionr.py index 21b3bd20..c8bb2873 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -50,6 +50,7 @@ class Onionr: Main Onionr class. This is for the CLI program, and does not handle much of the logic. In general, external programs and plugins should not use this class. ''' + self.userRunDir = os.getcwd() # Directory user runs the program from try: os.chdir(sys.path[0]) except FileNotFoundError: @@ -190,6 +191,10 @@ class Onionr: 'add-file': self.addFile, 'addfile': self.addFile, + + 'get-file': self.getFile, + 'getfile': self.getFile, + 'listconn': self.listConn, 'import-blocks': self.onionrUtils.importNewBlocks, @@ -230,6 +235,7 @@ class Onionr: 'add-peer': 'Adds a peer to database', 'list-peers': 'Displays a list of peers', 'add-file': 'Create an Onionr block from a file', + 'get-file': 'Get a file from Onionr blocks', 'import-blocks': 'import blocks from the disk (Onionr is transport-agnostic!)', 'listconn': 'list connected peers', 'kex': 'exchange keys with peers (done automatically)', @@ -780,6 +786,24 @@ class Onionr: return columns + def getFile(self): + ''' + Get a file from onionr blocks + ''' + if len(sys.argv) >= 3: + fileName = sys.argv[2] + print(fileName) + contents = None + bHash = sys.argv[3] + if os.path.exists(fileName): + logger.error("File already exists") + return + if not self.onionrUtils.validateHash(bHash): + logger.error('Block hash is invalid') + return + Block.mergeChain(bHash, fileName) + return + def addFile(self): ''' Adds a file to the onionr network @@ -790,8 +814,9 @@ class Onionr: contents = None if not os.path.exists(filename): - logger.warn('That file does not exist. Improper path?') - + logger.error('That file does not exist. Improper path (specify full path)?') + return + logger.info('Adding file... this might take a long time.') try: blockhash = Block.createChain(file = filename) logger.info('File %s saved in block %s.' % (filename, blockhash)) diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index 97b34730..78d3e71e 100644 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -177,6 +177,7 @@ class Block: # signed data is jsonMeta + block content (no linebreak) self.signedData = (None if not self.isSigned() else self.getHeader('meta') + self.getContent()) self.date = self.getCore().getBlockDate(self.getHash()) + self.claimedTime = self.getHeader('time', None) if not self.getDate() is None: self.date = datetime.datetime.fromtimestamp(self.getDate()) diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index 29b9375b..7340fed3 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import onionrblockapi, logger, onionrexceptions +import onionrblockapi, logger, onionrexceptions, json class OnionrUser: def __init__(self, coreInst, publicKey): self.trust = 0 diff --git a/onionr/static-data/default-plugins/boards/info.json b/onionr/static-data/default-plugins/boards/info.json deleted file mode 100644 index f3e80e1c..00000000 --- a/onionr/static-data/default-plugins/boards/info.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name" : "boards", - "version" : "1.0", - "author" : "onionr" -} diff --git a/onionr/static-data/default-plugins/boards/main.py b/onionr/static-data/default-plugins/boards/main.py deleted file mode 100644 index 573e6237..00000000 --- a/onionr/static-data/default-plugins/boards/main.py +++ /dev/null @@ -1,63 +0,0 @@ -''' - Onionr - P2P Anonymous Storage Network - - This is an interactive menu-driven CLI interface for Onionr -''' -''' - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -''' - -# Imports some useful libraries -import logger, config, sys -from onionrblockapi import Block -try: - import tkinter -except (ImportError, NameError) as e: - TK_ENABLED = False -else: - TK_ENABLED = True - - -plugin_name = 'cliui' -PLUGIN_VERSION = '0.0.1' - -class OnionrBoards: - def __init__(self, apiInst): - self.api = apiInst - self.myCore = apiInst.get_core() - - if TK_ENABLED: - self.gui = tkinter.Tk() - - return - - def start(self): - return - -def on_init(api, data = None): - ''' - This event is called after Onionr is initialized, but before the command - inputted is executed. Could be called when daemon is starting or when - just the client is running. - ''' - - # Doing this makes it so that the other functions can access the api object - # by simply referencing the variable `pluginapi`. - pluginapi = api - ui = OnionrBoards(api) - api.commands.register('boards', ui.start) - api.commands.register_help('boards', 'Open the board viewer') - - - return From 67be0bebc2fccb47dbf9d4f867898ae205b693b9 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 4 Sep 2018 23:06:17 -0500 Subject: [PATCH 07/70] added tor control and stem --- onionr/netcontroller.py | 35 +++++++++++++++---- .../static-data/default-plugins/cliui/main.py | 1 + requirements.txt | 1 + 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py index 3749ce7a..28214b95 100644 --- a/onionr/netcontroller.py +++ b/onionr/netcontroller.py @@ -18,7 +18,8 @@ along with this program. If not, see . ''' -import subprocess, os, random, sys, logger, time, signal, config +import subprocess, os, random, sys, logger, time, signal, config, base64 +from stem.control import Controller from onionrblockapi import Block class NetController: @@ -33,6 +34,14 @@ class NetController: self.hsPort = hsPort self._torInstnace = '' self.myID = '' + + if os.path.exists('./tor'): + self.torBinary = './tor' + elif os.path.exists('/usr/bin/tor'): + self.torBinary = '/usr/bin/tor' + else: + self.torBinary = 'tor' + config.reload() ''' if os.path.exists(self.torConfigLocation): @@ -52,13 +61,27 @@ class NetController: if config.get('tor.v3onions'): hsVer = 'HiddenServiceVersion 3' logger.info('Using v3 onions :)') + if os.path.exists(self.torConfigLocation): os.remove(self.torConfigLocation) + + # Set the Tor control password. Meant to make it harder to manipulate our Tor instance + plaintext = base64.b64encode(os.urandom(50)).decode() + config.set('tor.controlpassword', plaintext, savefile=True) + + hashedPassword = subprocess.Popen([self.torBinary, '--hash-password', plaintext], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + for line in iter(hashedPassword.stdout.readline, b''): + password = line.decode() + break + torrcData = '''SocksPort ''' + str(self.socksPort) + ''' HiddenServiceDir data/hs/ \n''' + hsVer + '''\n HiddenServicePort 80 127.0.0.1:''' + str(self.hsPort) + ''' DataDirectory data/tordata/ +CookieAuthentication 1 +ControlPort 9051 +HashedControlPassword ''' + str(password) + ''' ''' torrc = open(self.torConfigLocation, 'w') torrc.write(torrcData) @@ -74,20 +97,20 @@ DataDirectory data/tordata/ self.generateTorrc() if os.path.exists('./tor'): - torBinary = './tor' + self.torBinary = './tor' elif os.path.exists('/usr/bin/tor'): - torBinary = '/usr/bin/tor' + self.torBinary = '/usr/bin/tor' else: - torBinary = 'tor' + self.torBinary = 'tor' try: - tor = subprocess.Popen([torBinary, '-f', self.torConfigLocation], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + tor = subprocess.Popen([self.torBinary, '-f', self.torConfigLocation], stdout=subprocess.PIPE, stderr=subprocess.PIPE) except FileNotFoundError: logger.fatal("Tor was not found in your path or the Onionr directory. Please install Tor and try again.") sys.exit(1) else: # Test Tor Version - torVersion = subprocess.Popen([torBinary, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + torVersion = subprocess.Popen([self.torBinary, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) for line in iter(torVersion.stdout.readline, b''): if 'Tor 0.2.' in line.decode(): logger.warn("Running 0.2.x Tor series, no support for v3 onion peers") diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py index 4095f814..e56f39df 100644 --- a/onionr/static-data/default-plugins/cliui/main.py +++ b/onionr/static-data/default-plugins/cliui/main.py @@ -46,6 +46,7 @@ class OnionrCLIUI: showMenu = True isOnline = "No" firstRun = True + choice = '' if self.myCore._utils.localCommand('ping') == 'pong': firstRun = False diff --git a/requirements.txt b/requirements.txt index 97f8969e..69322e22 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ defusedxml==0.5.0 simple_crypt==4.1.7 Flask==1.0.2 PySocks==1.6.8 +stem==1.6.0 From a44d511e1da06cc81ea0dafb93d16e0ef4d7b6ba Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 7 Sep 2018 13:57:20 -0500 Subject: [PATCH 08/70] bind to random control port --- onionr/netcontroller.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py index 28214b95..74430bee 100644 --- a/onionr/netcontroller.py +++ b/onionr/netcontroller.py @@ -21,7 +21,7 @@ import subprocess, os, random, sys, logger, time, signal, config, base64 from stem.control import Controller from onionrblockapi import Block - +from dependencies import secrets class NetController: ''' This class handles hidden service setup on Tor and I2P @@ -69,6 +69,8 @@ class NetController: plaintext = base64.b64encode(os.urandom(50)).decode() config.set('tor.controlpassword', plaintext, savefile=True) + controlPort = random.randint(1025, 65535) + hashedPassword = subprocess.Popen([self.torBinary, '--hash-password', plaintext], stdout=subprocess.PIPE, stderr=subprocess.PIPE) for line in iter(hashedPassword.stdout.readline, b''): password = line.decode() @@ -80,7 +82,7 @@ HiddenServiceDir data/hs/ HiddenServicePort 80 127.0.0.1:''' + str(self.hsPort) + ''' DataDirectory data/tordata/ CookieAuthentication 1 -ControlPort 9051 +ControlPort ''' + str(controlPort) + ''' HashedControlPassword ''' + str(password) + ''' ''' torrc = open(self.torConfigLocation, 'w') From 151b12424cd33fb6788c5fe1dcc479414bf8e501 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 9 Sep 2018 00:12:41 -0500 Subject: [PATCH 09/70] work on block processing module --- onionr/core.py | 2 ++ onionr/netcontroller.py | 3 ++- onionr/onionrutils.py | 16 +++------------- onionr/static-data/default-plugins/cliui/main.py | 2 +- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index ab8b640b..97b822af 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -102,6 +102,8 @@ class Core: logger.warn("POW token for pubkey base64 representation exceeded 120 bytes, is " + str(sys.getsizeof(powID))) return False + events.event('pubkey_add', data = {'key': peerID}, onionr = None) + conn = sqlite3.connect(self.peerDB) hashID = self._crypto.pubKeyHashID(peerID) c = conn.cursor() diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py index 74430bee..b3691956 100644 --- a/onionr/netcontroller.py +++ b/onionr/netcontroller.py @@ -74,7 +74,8 @@ class NetController: hashedPassword = subprocess.Popen([self.torBinary, '--hash-password', plaintext], stdout=subprocess.PIPE, stderr=subprocess.PIPE) for line in iter(hashedPassword.stdout.readline, b''): password = line.decode() - break + if 'warn' not in password: + break torrcData = '''SocksPort ''' + str(self.socksPort) + ''' HiddenServiceDir data/hs/ diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index a383684c..470dc125 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -23,6 +23,7 @@ import nacl.signing, nacl.encoding from onionrblockapi import Block import onionrexceptions from defusedxml import minidom +import onionrevents import pgpwords, onionrusers, storagecounter if sys.version_info < (3, 6): try: @@ -276,19 +277,8 @@ class OnionrUtils: if len(blockType) <= 10: self._core.updateBlockInfo(blockHash, 'dataType', blockType) - if blockType == 'userInfo': - if myBlock.verifySig(): - peerName = myBlock.getMetadata('name') - try: - if len(peerName) > 20: - raise onionrexceptions.InvalidMetdata('Peer name specified is too large') - except TypeError: - pass - except onionrexceptions.InvalidMetadata: - pass - else: - self._core.setPeerInfo(signer, 'name', peerName) - logger.info('%s is now using the name %s.' % (signer, self.escapeAnsi(peerName))) + onionrevents.event('processBlocks', data = {'block': myBlock, 'type': blockType}, onionr = None) + except TypeError: pass diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py index e56f39df..5fc24385 100644 --- a/onionr/static-data/default-plugins/cliui/main.py +++ b/onionr/static-data/default-plugins/cliui/main.py @@ -54,7 +54,7 @@ class OnionrCLIUI: while showMenu: if firstRun: print("please wait while Onionr starts...") - daemon = subprocess.Popen(["./onionr.py", "start"], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + daemon = subprocess.Popen(["./onionr.py", "start"], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL) time.sleep(30) firstRun = False From ce2423e6d9ac70e04d99e9902ccd45fde933fd2d Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 10 Sep 2018 00:02:28 -0500 Subject: [PATCH 10/70] * moved metadata processor to its own module * improved some comments * removed defunct utils functions --- onionr/blockprocessor.py | 0 onionr/onionrsockets.py | 19 +++++++ onionr/onionrutils.py | 34 +++++------- .../metadataprocessor/info.json | 5 ++ .../default-plugins/metadataprocessor/main.py | 52 +++++++++++++++++++ 5 files changed, 89 insertions(+), 21 deletions(-) create mode 100644 onionr/blockprocessor.py create mode 100644 onionr/onionrsockets.py create mode 100644 onionr/static-data/default-plugins/metadataprocessor/info.json create mode 100644 onionr/static-data/default-plugins/metadataprocessor/main.py diff --git a/onionr/blockprocessor.py b/onionr/blockprocessor.py new file mode 100644 index 00000000..e69de29b diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py new file mode 100644 index 00000000..df4a1ad7 --- /dev/null +++ b/onionr/onionrsockets.py @@ -0,0 +1,19 @@ +''' + Onionr - P2P Anonymous Storage Network + + Onionr Socket interface +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 470dc125..6449e152 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -37,17 +37,20 @@ class OnionrUtils: Various useful functions for validating things, etc functions, connectivity ''' def __init__(self, coreInstance): - self.fingerprintFile = 'data/own-fingerprint.txt' - self._core = coreInstance + #self.fingerprintFile = 'data/own-fingerprint.txt' #TODO Remove since probably not needed + self._core = coreInstance # onionr core instance - self.timingToken = '' + self.timingToken = '' # for when we make local connections to our http api, to bypass timing attack defense mechanism self.avoidDupe = [] # list used to prevent duplicate requests per peer for certain actions self.peerProcessing = {} # dict of current peer actions: peer, actionList - self.storageCounter = storagecounter.StorageCounter(self._core) - config.reload() + self.storageCounter = storagecounter.StorageCounter(self._core) # used to keep track of how much data onionr is using on disk + config.reload() # onionr config return def getTimeBypassToken(self): + ''' + Load our timingToken from disk for faster local HTTP API + ''' try: if os.path.exists('data/time-bypass.txt'): with open('data/time-bypass.txt', 'r') as bypass: @@ -64,22 +67,6 @@ class OnionrUtils: epoch = self.getEpoch() return epoch - (epoch % roundS) - def incrementAddressSuccess(self, address): - ''' - Increase the recorded sucesses for an address - ''' - increment = self._core.getAddressInfo(address, 'success') + 1 - self._core.setAddressInfo(address, 'success', increment) - return - - def decrementAddressSuccess(self, address): - ''' - Decrease the recorded sucesses for an address - ''' - increment = self._core.getAddressInfo(address, 'success') - 1 - self._core.setAddressInfo(address, 'success', increment) - return - def mergeKeys(self, newKeyList): ''' Merge ed25519 key list to our database, comma seperated string @@ -89,6 +76,7 @@ class OnionrUtils: if newKeyList != False: for key in newKeyList.split(','): key = key.split('-') + # Test if key is valid try: if len(key[0]) > 60 or len(key[1]) > 1000: logger.warn('%s or its pow value is too large.' % key[0]) @@ -100,15 +88,19 @@ class OnionrUtils: value = base64.b64decode(key[1]) except binascii.Error: continue + # Load the pow token hashedKey = self._core._crypto.blake2bHash(key[0]) powHash = self._core._crypto.blake2bHash(value + hashedKey) try: powHash = powHash.encode() except AttributeError: pass + # if POW meets required difficulty, TODO make configurable/dynamic if powHash.startswith(b'0000'): + # if we don't already have the key and its not our key, add it. if not key[0] in self._core.listPeers(randomOrder=False) and type(key) != None and key[0] != self._core._crypto.pubKey: if self._core.addPeer(key[0], key[1]): + # Check if the peer has a set username already onionrusers.OnionrUser(self._core, key[0]).findAndSetID() retVal = True else: diff --git a/onionr/static-data/default-plugins/metadataprocessor/info.json b/onionr/static-data/default-plugins/metadataprocessor/info.json new file mode 100644 index 00000000..355d98f1 --- /dev/null +++ b/onionr/static-data/default-plugins/metadataprocessor/info.json @@ -0,0 +1,5 @@ +{ + "name" : "metadataprocessor", + "version" : "1.0", + "author" : "onionr" +} diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py new file mode 100644 index 00000000..842eaf88 --- /dev/null +++ b/onionr/static-data/default-plugins/metadataprocessor/main.py @@ -0,0 +1,52 @@ +''' + Onionr - P2P Anonymous Storage Network + + This processes metadata for Onionr blocks +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' + +# useful libraries +import logger, config +import os, sys, json, time, random, shutil, base64, getpass, datetime, re +from onionrblockapi import Block + +plugin_name = 'metadataprocessor' + +# event listeners + +def on_processBlocks(api): + myBlock = api.data['block'] + blockType = api.data['type'] + print('blockType is ' + blockType) + if blockType == 'userInfo': + if myBlock.verifySig(): + peerName = myBlock.getMetadata('name') + try: + if len(peerName) > 20: + raise onionrexceptions.InvalidMetdata('Peer name specified is too large') + except TypeError: + pass + except onionrexceptions.InvalidMetadata: + pass + else: + api.get_core().setPeerInfo(signer, 'name', peerName) + logger.info('%s is now using the name %s.' % (signer, api.get_utils().escapeAnsi(peerName))) + +def on_init(api, data = None): + + pluginapi = api + + return From d151e0d3023bafe676c8d639110e2be1b002c6b5 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 11 Sep 2018 14:45:06 -0500 Subject: [PATCH 11/70] work on forward secrecy --- .gitignore | 1 + onionr/core.py | 15 +++---- onionr/dbcreator.py | 5 ++- onionr/onionrusers.py | 16 ++++++- .../default-plugins/metadataprocessor/main.py | 44 ++++++++++++++----- 5 files changed, 60 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 6edc23ff..26e43b0e 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ onionr/data-encrypted.dat onionr/.onionr-lock core .vscode/* +venv/* diff --git a/onionr/core.py b/onionr/core.py index 97b822af..c1edf921 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -472,18 +472,17 @@ class Core: id text 0 name text, 1 adders text, 2 - forwardKey text, 3 - dateSeen not null, 4 - bytesStored int, 5 - trust int 6 - pubkeyExchanged int 7 - hashID text 8 - pow text 9 + dateSeen not null, 3 + bytesStored int, 4 + trust int 5 + pubkeyExchanged int 6 + hashID text 7 + pow text 8 ''' conn = sqlite3.connect(self.peerDB) c = conn.cursor() command = (peer,) - infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'forwardKey': 3, 'dateSeen': 4, 'bytesStored': 5, 'trust': 6, 'pubkeyExchanged': 7, 'hashID': 8} + infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'bytesStored': 4, 'trust': 5, 'pubkeyExchanged': 6, 'hashID': 7} info = infoNumbers[info] iterCount = 0 retVal = '' diff --git a/onionr/dbcreator.py b/onionr/dbcreator.py index 5f3d2c79..05ea796e 100644 --- a/onionr/dbcreator.py +++ b/onionr/dbcreator.py @@ -61,7 +61,6 @@ class DBCreator: ID text not null, name text, adders text, - forwardKey text, dateSeen not null, bytesStored int, trust int, @@ -69,6 +68,10 @@ class DBCreator: hashID text, pow text not null); ''') + c.execute('''CREATE TABLE forwardKeys( + peerKey text not null, + forwardKey text not null, + date int not null);''') conn.commit() conn.close() return diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index 7340fed3..10e2be33 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import onionrblockapi, logger, onionrexceptions, json +import onionrblockapi, logger, onionrexceptions, json, sqlite3 class OnionrUser: def __init__(self, coreInst, publicKey): self.trust = 0 @@ -60,6 +60,20 @@ class OnionrUser: def forwardDecrypt(self, encrypted): return + def addForwardKey(self, newKey): + # Add a forward secrecy key for the peer + conn = sqlite3.connect(self._core.peerDB) + c = conn.cursor() + # Prepare the insert + time = self._core._utils.getEpoch() + command = (self.publicKey, newKey, time) + + c.execute("INSERT INTO forwardKeys VALUES(?, ?, ?);", command) + + conn.commit() + conn.close() + return + def findAndSetID(self): '''Find any info about the user from existing blocks and cache it to their DB entry''' infoBlocks = [] diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py index 842eaf88..994bf818 100644 --- a/onionr/static-data/default-plugins/metadataprocessor/main.py +++ b/onionr/static-data/default-plugins/metadataprocessor/main.py @@ -22,28 +22,50 @@ import logger, config import os, sys, json, time, random, shutil, base64, getpass, datetime, re from onionrblockapi import Block +import onionrusers plugin_name = 'metadataprocessor' # event listeners +def _processUserInfo(api, newBlock): + ''' + Set the username for a particular user, from a signed block by them + ''' + myBlock = newBlock + peerName = myBlock.getMetadata('name') + try: + if len(peerName) > 20: + raise onionrexceptions.InvalidMetdata('Peer name specified is too large') + except TypeError: + pass + except onionrexceptions.InvalidMetadata: + pass + else: + api.get_core().setPeerInfo(signer, 'name', peerName) + logger.info('%s is now using the name %s.' % (signer, api.get_utils().escapeAnsi(peerName))) + +def _processForwardKey(api, myBlock): + ''' + Get the forward secrecy key specified by the user for us to use + ''' + peer = onionrusers.OnionrUser(self.api.get_core(), myBlock.signer) + def on_processBlocks(api): myBlock = api.data['block'] blockType = api.data['type'] print('blockType is ' + blockType) + + # Process specific block types + + # userInfo blocks, such as for setting username if blockType == 'userInfo': if myBlock.verifySig(): - peerName = myBlock.getMetadata('name') - try: - if len(peerName) > 20: - raise onionrexceptions.InvalidMetdata('Peer name specified is too large') - except TypeError: - pass - except onionrexceptions.InvalidMetadata: - pass - else: - api.get_core().setPeerInfo(signer, 'name', peerName) - logger.info('%s is now using the name %s.' % (signer, api.get_utils().escapeAnsi(peerName))) + _processUserInfo(api, myBlock) + # forwardKey blocks + elif blockType == 'forwardKey': + if myBlock.verifySig(): + _processForwardKey(api, myBlock) def on_init(api, data = None): From c4dcd89dfe8622fff43dd14973d86cba10c6a6ae Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 11 Sep 2018 21:58:51 -0500 Subject: [PATCH 12/70] + added methods to import and select new forward secrecy keys --- onionr/core.py | 2 +- onionr/onionrusers.py | 22 +++++++++++++++++-- .../default-plugins/metadataprocessor/main.py | 2 +- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index c1edf921..15c862c5 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -693,7 +693,7 @@ class Core: data = data.encode() except AttributeError: pass - # sign before encrypt, as unauthenticated crypto should not be a problem here + if sign: signature = self._crypto.edSign(jsonMeta.encode() + data, key=self._crypto.privKey, encodeResult=True) signer = self._crypto.pubKey diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index 10e2be33..4b74258e 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -50,7 +50,7 @@ class OnionrUser: encrypted = coreInst._crypto.pubKeyEncrypt(data, self.publicKey, encodedData=True) return encrypted - def decrypt(self, data): + def decrypt(self, data, anonymous=True): decrypted = coreInst._crypto.pubKeyDecrypt(data, self.publicKey, encodedData=True) return decrypted @@ -59,8 +59,26 @@ class OnionrUser: def forwardDecrypt(self, encrypted): return - + + def _getLatestForwardKey(self): + # Get the latest forward secrecy key for a peer + conn = sqlite3.connect(self._core.peerDB) + c = conn.cursor() + # Prepare the insert + time = self._core._utils.getEpoch() + key = '' + + for row in c.execute("SELECT forwardKey FROM forwardKeys WHERE DATE=(SELECT max(date) FROM forwardKeys);"): + key = row[0] + break + + conn.commit() + conn.close() + return key + def addForwardKey(self, newKey): + if not self._core._utils.validatePubKey(newKey): + raise onionrexceptions.InvalidPubkey # Add a forward secrecy key for the peer conn = sqlite3.connect(self._core.peerDB) c = conn.cursor() diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py index 994bf818..74397c12 100644 --- a/onionr/static-data/default-plugins/metadataprocessor/main.py +++ b/onionr/static-data/default-plugins/metadataprocessor/main.py @@ -54,7 +54,7 @@ def _processForwardKey(api, myBlock): def on_processBlocks(api): myBlock = api.data['block'] blockType = api.data['type'] - print('blockType is ' + blockType) + logger.info('blockType is ' + blockType) # Process specific block types From 1c2a8a2f400fc7f6f83ed19071c932239c352584 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Wed, 12 Sep 2018 20:23:50 -0500 Subject: [PATCH 13/70] work on forward secrecy --- onionr/onionrusers.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index 4b74258e..898f9ae4 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -55,26 +55,43 @@ class OnionrUser: return decrypted def forwardEncrypt(self, data): + retData = '' + forwardKey = self._getLatestForwardKey() + if self._core._utils.validatePubKey(forwardKey): + encrypted = self._core._crypto.pubKeyEncrypt(data, forwardKey, encodedData=True) + else: + raise Exception("No valid forward key available for this user") return def forwardDecrypt(self, encrypted): + retData = '' return def _getLatestForwardKey(self): # Get the latest forward secrecy key for a peer conn = sqlite3.connect(self._core.peerDB) c = conn.cursor() - # Prepare the insert - time = self._core._utils.getEpoch() - key = '' - for row in c.execute("SELECT forwardKey FROM forwardKeys WHERE DATE=(SELECT max(date) FROM forwardKeys);"): + for row in c.execute("SELECT forwardKey FROM forwardKeys WHERE peerKey = ? AND date=(SELECT max(date) FROM forwardKeys)", (self.publicKey,)): key = row[0] break conn.commit() conn.close() return key + + def _getForwardKeys(self): + conn = sqlite3.connect(self._core.peerDB) + c = conn.cursor() + keyList = [] + for row in c.execute("SELECT forwardKey FROM forwardKeys WHERE peerKey = ?", (self.publicKey,)): + key = row[0] + keyList.append(key) + + conn.commit() + conn.close() + + return list(keyList) def addForwardKey(self, newKey): if not self._core._utils.validatePubKey(newKey): From ee2a74380b02f23c91ed77dc21939dca3976706e Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 13 Sep 2018 12:26:22 -0500 Subject: [PATCH 14/70] work on metadata, forward secrecy, and starting on sockets --- onionr/core.py | 1 + onionr/dbcreator.py | 24 ++++++++++++++++++- onionr/onionrusers.py | 20 ++++++++++++++++ onionr/onionrutils.py | 3 ++- .../default-plugins/metadataprocessor/main.py | 19 +++++++++++---- 5 files changed, 61 insertions(+), 6 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index 15c862c5..3ce4bbfd 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -48,6 +48,7 @@ class Core: self.torPort = torPort self.dataNonceFile = 'data/block-nonces.dat' self.dbCreate = dbcreator.DBCreator(self) + self.forwardKeysFile = 'data/forward-keys.db' self.usageFile = 'data/disk-usage.txt' self.config = config diff --git a/onionr/dbcreator.py b/onionr/dbcreator.py index 05ea796e..f9838342 100644 --- a/onionr/dbcreator.py +++ b/onionr/dbcreator.py @@ -71,7 +71,9 @@ class DBCreator: c.execute('''CREATE TABLE forwardKeys( peerKey text not null, forwardKey text not null, - date int not null);''') + date int not null, + expire int not null + );''') conn.commit() conn.close() return @@ -108,4 +110,24 @@ class DBCreator: ''') conn.commit() conn.close() + return + + def createForwardKeyDB(self): + ''' + Create the forward secrecy key db (*for *OUR* keys*) + ''' + if os.path.exists(self.core.forwardKeysFile): + raise Exception("Block database already exists") + conn = sqlite3.connect(self.core.forwardKeysFile) + c = conn.cursor() + c.execute('''CREATE TABLE myForwardKeys( + peer text not null, + public key text not null, + private key text not null, + date int not null, + expire int not null + ); + ''') + conn.commit() + conn.close() return \ No newline at end of file diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index 898f9ae4..1e8cdf29 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -93,6 +93,26 @@ class OnionrUser: return list(keyList) + def generateForwardKey(self, expire=432000): + + # Generate a forward secrecy key for the peer + conn = sqlite3.connect(self._core.forwardKeysFile) + c = conn.cursor() + # Prepare the insert + time = self._core._utils.getEpoch() + newKeys = self._core._crypto.generatePubKey() + newPub = newKeys[0] + newPriv = newKeys[1] + + time = self._core._utils.getEpoch() + command = (self.publicKey, newPub, newPriv, time, expire) + + c.execute("INSERT INTO myForwardKeys VALUES(?, ?, ?, ?);", command) + + conn.commit() + conn.close() + + def addForwardKey(self, newKey): if not self._core._utils.validatePubKey(newKey): raise onionrexceptions.InvalidPubkey diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 6449e152..b5fc10c1 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -265,11 +265,12 @@ class OnionrUtils: myBlock.decrypt() blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks signer = self.bytesToStr(myBlock.signer) + valid = myBlock.verifySig() try: if len(blockType) <= 10: self._core.updateBlockInfo(blockHash, 'dataType', blockType) - onionrevents.event('processBlocks', data = {'block': myBlock, 'type': blockType}, onionr = None) + onionrevents.event('processBlocks', data = {'block': myBlock, 'type': blockType, 'signer': signer, 'validSig': valid}, onionr = None) except TypeError: pass diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py index 74397c12..5145d911 100644 --- a/onionr/static-data/default-plugins/metadataprocessor/main.py +++ b/onionr/static-data/default-plugins/metadataprocessor/main.py @@ -22,7 +22,7 @@ import logger, config import os, sys, json, time, random, shutil, base64, getpass, datetime, re from onionrblockapi import Block -import onionrusers +import onionrusers, onionrexceptions plugin_name = 'metadataprocessor' @@ -50,6 +50,13 @@ def _processForwardKey(api, myBlock): Get the forward secrecy key specified by the user for us to use ''' peer = onionrusers.OnionrUser(self.api.get_core(), myBlock.signer) + key = myBlock.getMetadata('newFSKey') + + # We don't need to validate here probably, but it helps + if api.get_utils().validatePubKey(key): + peer.addForwardKey(key) + else: + raise onionrexceptions.InvalidPubkey("%s is nota valid pubkey key" % (key,)) def on_processBlocks(api): myBlock = api.data['block'] @@ -60,12 +67,16 @@ def on_processBlocks(api): # userInfo blocks, such as for setting username if blockType == 'userInfo': - if myBlock.verifySig(): + if api.data['validSig']: _processUserInfo(api, myBlock) - # forwardKey blocks + # forwardKey blocks, add a new forward secrecy key for a peer elif blockType == 'forwardKey': - if myBlock.verifySig(): + if api.data['validSig']: _processForwardKey(api, myBlock) + # socket blocks + elif blockType == 'openSocket': + if api.data['validSig']: + pass def on_init(api, data = None): From e0fbe2033e2df596c197effba21cca0e8af6b6e8 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 14 Sep 2018 20:05:25 -0500 Subject: [PATCH 15/70] work on sockets --- onionr/communicator2.py | 8 +++++- onionr/onionrexceptions.py | 5 ++++ onionr/onionrsockets.py | 25 +++++++++++++++++++ .../default-plugins/metadataprocessor/main.py | 8 +++++- 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index f203dd70..e6d49d28 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -21,7 +21,7 @@ ''' import sys, os, core, config, json, requests, time, logger, threading, base64, onionr import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block -import onionrdaemontools +import onionrdaemontools, onionrsockets from defusedxml import minidom class OnionrCommunicatorDaemon: @@ -79,6 +79,9 @@ class OnionrCommunicatorDaemon: #self.daemonTools = onionrdaemontools.DaemonTools(self) self.daemonTools = onionrdaemontools.DaemonTools(self) + # Active sockets for direct connections + self.sockets = [] + if debug or developmentMode: OnionrCommunicatorTimers(self, self.heartbeat, 10) @@ -462,6 +465,9 @@ class OnionrCommunicatorDaemon: elif cmd[0] == 'uploadBlock': self.blockToUpload = cmd[1] threading.Thread(target=self.uploadBlock).start() + elif cmd[0] == 'createSocket': + # Create a socket + self.onionrsockets.append(onionrsockets.OnionrSockets(self._core, startData)) else: logger.info('Recieved daemonQueue command:' + cmd[0]) diff --git a/onionr/onionrexceptions.py b/onionr/onionrexceptions.py index c32fbbb4..4954550e 100644 --- a/onionr/onionrexceptions.py +++ b/onionr/onionrexceptions.py @@ -65,4 +65,9 @@ class InvalidAddress(Exception): # file exceptions class DiskAllocationReached(Exception): + pass + +# onionrsocket exceptions + +class MissingAddress(Exception): pass \ No newline at end of file diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index df4a1ad7..0f00519f 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -17,3 +17,28 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' +import stem +import onionrexceptions +from dependencies import secrets + +class OnionrSockets: + def __init__(self, coreInst, socketInfo): + '''Create a new Socket object. This interface is named a bit misleadingly + and does not actually forward network requests. + + Accepts coreInst, an instance of Onionr core library, and socketInfo, a dict with these values: + 'peer': peer master public key + 'address': string, if we're connecting to a socket, this is the address we connect to. Not applicable if we're creating our own + create: bool + ''' + self.socketID = secrets.token_hex(32) # Generate an ID for this socket + self._core = coreInst + + # Make sure socketInfo provides all necessary values + for i in ('peer', 'address', 'create'): + try: + socketInfo[i] + except KeyError: + raise ValueError('Must provide peer, address, and create in socketInfo dict argument') + + \ No newline at end of file diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py index 5145d911..4d23fa9a 100644 --- a/onionr/static-data/default-plugins/metadataprocessor/main.py +++ b/onionr/static-data/default-plugins/metadataprocessor/main.py @@ -59,6 +59,7 @@ def _processForwardKey(api, myBlock): raise onionrexceptions.InvalidPubkey("%s is nota valid pubkey key" % (key,)) def on_processBlocks(api): + # Generally fired by utils. myBlock = api.data['block'] blockType = api.data['type'] logger.info('blockType is ' + blockType) @@ -76,7 +77,12 @@ def on_processBlocks(api): # socket blocks elif blockType == 'openSocket': if api.data['validSig']: - pass + try: + address = api.data['address'] + except KeyError: + raise onionrexceptions.MissingAddress("Missing address for new socket") + socketInfo = json.dumps({'peer': api.data['signer'], 'address': address, create = False}) + api.get_core().daemonQueueAdd('createSocket', socketInfo) def on_init(api, data = None): From d80e72d18cf097663d2285ee858c683ec5b726de Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 14 Sep 2018 23:48:48 -0500 Subject: [PATCH 16/70] work on sockets --- onionr/communicator2.py | 4 ++-- onionr/onionrsockets.py | 11 +++++++++-- .../default-plugins/metadataprocessor/main.py | 8 ++++---- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index e6d49d28..c0fbb698 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -465,8 +465,8 @@ class OnionrCommunicatorDaemon: elif cmd[0] == 'uploadBlock': self.blockToUpload = cmd[1] threading.Thread(target=self.uploadBlock).start() - elif cmd[0] == 'createSocket': - # Create a socket + elif cmd[0] == 'startSocket': + # Create a socket or connect to one self.onionrsockets.append(onionrsockets.OnionrSockets(self._core, startData)) else: logger.info('Recieved daemonQueue command:' + cmd[0]) diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index 0f00519f..f80b1637 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -33,6 +33,7 @@ class OnionrSockets: ''' self.socketID = secrets.token_hex(32) # Generate an ID for this socket self._core = coreInst + self.socketInfo = socketInfo # Make sure socketInfo provides all necessary values for i in ('peer', 'address', 'create'): @@ -40,5 +41,11 @@ class OnionrSockets: socketInfo[i] except KeyError: raise ValueError('Must provide peer, address, and create in socketInfo dict argument') - - \ No newline at end of file + + self.isServer = socketInfo['create'] + + self.serverKey = socketInfo['peer'] + self.serverAddress = socketInfo['address'] + + def createServer(self): + return \ No newline at end of file diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py index 4d23fa9a..199c8821 100644 --- a/onionr/static-data/default-plugins/metadataprocessor/main.py +++ b/onionr/static-data/default-plugins/metadataprocessor/main.py @@ -68,21 +68,21 @@ def on_processBlocks(api): # userInfo blocks, such as for setting username if blockType == 'userInfo': - if api.data['validSig']: + if api.data['validSig'] == True: # we use == True for type safety _processUserInfo(api, myBlock) # forwardKey blocks, add a new forward secrecy key for a peer elif blockType == 'forwardKey': - if api.data['validSig']: + if api.data['validSig'] == True: _processForwardKey(api, myBlock) # socket blocks elif blockType == 'openSocket': - if api.data['validSig']: + if api.data['validSig'] == True: try: address = api.data['address'] except KeyError: raise onionrexceptions.MissingAddress("Missing address for new socket") socketInfo = json.dumps({'peer': api.data['signer'], 'address': address, create = False}) - api.get_core().daemonQueueAdd('createSocket', socketInfo) + api.get_core().daemonQueueAdd('startSocket', socketInfo) def on_init(api, data = None): From 620897a2ebcf33f93eede43c029975b6af5fdd2e Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 15 Sep 2018 11:13:03 -0500 Subject: [PATCH 17/70] work on sockets --- onionr/onionrsockets.py | 27 ++++++++++++++++--- .../default-plugins/metadataprocessor/main.py | 9 +++++-- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index f80b1637..972ef328 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -17,7 +17,8 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import stem +import stem.control +import socket, selectors import onionrexceptions from dependencies import secrets @@ -36,16 +37,34 @@ class OnionrSockets: self.socketInfo = socketInfo # Make sure socketInfo provides all necessary values - for i in ('peer', 'address', 'create'): + for i in ('peer', 'address', 'create', 'port'): try: socketInfo[i] except KeyError: raise ValueError('Must provide peer, address, and create in socketInfo dict argument') - self.isServer = socketInfo['create'] + self.isServer = socketInfo['create'] # if we are the one creating the service - self.serverKey = socketInfo['peer'] + self.remotePeer = socketInfo['peer'] + self.socketPort = socketInfo['port'] self.serverAddress = socketInfo['address'] + + if self.isServer: + self.createServer() def createServer(self): + # Create our HS and advertise it via a block + dataID = uuid.uuid4().hex + ourAddress = '' + ourPort = 1337 + ourInternalPort = 1338 + + # Setup the empheral HS + with stem.control.Controller.from_port() as controller: + controller.authenticate() + socketHS = controller.create_ephemeral_hidden_service({ourPort: ourInternalPort}, await_publication = True) + ourAddress = socketHS.service_id + + meta = {'address': ourAddress, 'port': ourPort} + self._core.insertBlock(dataID, header='openSocket', encryptType='asym', asymPeer=self.remotePeer, sign=True, meta=meta) return \ No newline at end of file diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py index 199c8821..3b0191ae 100644 --- a/onionr/static-data/default-plugins/metadataprocessor/main.py +++ b/onionr/static-data/default-plugins/metadataprocessor/main.py @@ -76,12 +76,17 @@ def on_processBlocks(api): _processForwardKey(api, myBlock) # socket blocks elif blockType == 'openSocket': - if api.data['validSig'] == True: + if api.data['validSig'] == True and myBlock.decrypted: # we check if it is decrypted as a way of seeing if it was for us try: address = api.data['address'] except KeyError: raise onionrexceptions.MissingAddress("Missing address for new socket") - socketInfo = json.dumps({'peer': api.data['signer'], 'address': address, create = False}) + try: + port = api.data['port'] + except KeyError: + raise ValueError("Missing port for new socket") + + socketInfo = json.dumps({'peer': api.data['signer'], 'address': address, 'port': port, create = False}) api.get_core().daemonQueueAdd('startSocket', socketInfo) def on_init(api, data = None): From 1d7fd65f381f3a4c71bc5ddb661a05b1fbe4dc69 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 17 Sep 2018 00:02:16 -0500 Subject: [PATCH 18/70] work on sockets --- onionr/onionrsockets.py | 58 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index 972ef328..8111f039 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -19,8 +19,9 @@ ''' import stem.control import socket, selectors -import onionrexceptions +import onionrexceptions, time from dependencies import secrets +sel = selectors.DefaultSelector() class OnionrSockets: def __init__(self, coreInst, socketInfo): @@ -48,6 +49,9 @@ class OnionrSockets: self.remotePeer = socketInfo['peer'] self.socketPort = socketInfo['port'] self.serverAddress = socketInfo['address'] + self.connected = False + self.segment = 0 + self.connData = {} if self.isServer: self.createServer() @@ -65,6 +69,52 @@ class OnionrSockets: socketHS = controller.create_ephemeral_hidden_service({ourPort: ourInternalPort}, await_publication = True) ourAddress = socketHS.service_id - meta = {'address': ourAddress, 'port': ourPort} - self._core.insertBlock(dataID, header='openSocket', encryptType='asym', asymPeer=self.remotePeer, sign=True, meta=meta) - return \ No newline at end of file + # Advertise the server + meta = {'address': ourAddress, 'port': ourPort} + self._core.insertBlock(dataID, header='openSocket', encryptType='asym', asymPeer=self.remotePeer, sign=True, meta=meta) + + # Build the socket server + sock = socket.socket() + sock.bind(('127.0.0.1', ourInternalPort)) + sock.listen(100) + sock.setblocking(False) + sel.register(sock, selectors.EVENT_READ, self._accept) + + while True: + events = sel.select() + for key, mask in events: + callback = key.data + callback(key.fileobj, mask) + + return + + def connectServer(self): + return + + def _accept(self, sock, mask): + # Just accept the connection and pass it to our handler + conn, addr = sock.accept() + conn.setblocking(False) + sel.register(conn, selectors.EVENT_READ, self._read) + + def _read(self, conn, mask): + data = conn.recv(1000).decode() + if data: + self.segment += 1 + self.connData[self.segment] = data + conn.send(data) + else: + sel.unregister(conn) + conn.close() + + def readConnection(self): + if not self.connected: + raise Exception("Connection closed") + count = 0 + while self.connected: + try: + yield self.connData[count] + count += 1 + except KeyError: + pass + time.sleep(0.01) From f8b10cfe124ef67b0becebe3c7a27dc3266fa32a Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Wed, 19 Sep 2018 23:35:26 -0500 Subject: [PATCH 19/70] a lot of work on sockets, and added chat module --- onionr/communicator2.py | 8 ++- onionr/core.py | 3 +- onionr/netcontroller.py | 1 + onionr/onionrchat.py | 30 +++++++++ onionr/onionrsockets.py | 63 ++++++++++++------- .../default-plugins/metadataprocessor/main.py | 6 +- 6 files changed, 83 insertions(+), 28 deletions(-) create mode 100644 onionr/onionrchat.py diff --git a/onionr/communicator2.py b/onionr/communicator2.py index c0fbb698..01af7270 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -466,8 +466,12 @@ class OnionrCommunicatorDaemon: self.blockToUpload = cmd[1] threading.Thread(target=self.uploadBlock).start() elif cmd[0] == 'startSocket': - # Create a socket or connect to one - self.onionrsockets.append(onionrsockets.OnionrSockets(self._core, startData)) + # Create a socket or connect to one. + # The socket handler (such as the plugin or app using it) is specified in startData['reason] + startData = json.loads(cmd[1]) + rCallback = onionrsockets.getSocketCallbackRecieveHandler(self._core, startData['reason'], startData['create']) + sCallback = onionrsockets.getSocketCallbackSendHandler(self._core, startData['reason'], startData['create']) + self.onionrsockets.append(onionrsockets.OnionrSockets(self._core, startData, recieveCallback=rCallback, sendCallback=sCallback)) else: logger.info('Recieved daemonQueue command:' + cmd[0]) diff --git a/onionr/core.py b/onionr/core.py index 3ce4bbfd..54219390 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -21,7 +21,7 @@ import sqlite3, os, sys, time, math, base64, tarfile, getpass, simplecrypt, hash from onionrblockapi import Block import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues -import onionrblacklist +import onionrblacklist, onionrchat import dbcreator if sys.version_info < (3, 6): try: @@ -79,6 +79,7 @@ class Core: # Initialize the crypto object self._crypto = onionrcrypto.OnionrCrypto(self) self._blacklist = onionrblacklist.OnionrBlackList(self) + self.chatInst = onionrchat.OnionrChat(self) except Exception as error: logger.error('Failed to initialize core Onionr library.', error=error) diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py index b3691956..56b12573 100644 --- a/onionr/netcontroller.py +++ b/onionr/netcontroller.py @@ -68,6 +68,7 @@ class NetController: # Set the Tor control password. Meant to make it harder to manipulate our Tor instance plaintext = base64.b64encode(os.urandom(50)).decode() config.set('tor.controlpassword', plaintext, savefile=True) + config.set('tor.socksport', self.socksPort, savefile=True) controlPort = random.randint(1025, 65535) diff --git a/onionr/onionrchat.py b/onionr/onionrchat.py new file mode 100644 index 00000000..a4051828 --- /dev/null +++ b/onionr/onionrchat.py @@ -0,0 +1,30 @@ +''' + Onionr - P2P Anonymous Storage Network + + Onionr Chat Messages +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import logger +class OnionrChat: + def __init__(self, coreInst): + return + + def recieveMessage(self, socketInst, data): + logger.info('Got %s' % (data,)) + return '' + + def sendMessage(self, socketInst, data): + return "Hello" \ No newline at end of file diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index 8111f039..2a63613c 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -18,13 +18,25 @@ along with this program. If not, see . ''' import stem.control -import socket, selectors -import onionrexceptions, time +import socket, selectors, socks, config +import onionrexceptions, time, onionrchat from dependencies import secrets sel = selectors.DefaultSelector() +def getSocketCallbackRecieveHandler(coreInst, reason, create): + '''Return the recieve handler function for a given socket reason''' + retData = '' + if startData == 'chat': + retData = coreInst.chatInst.recieveMessage + +def getSocketCallbackSendHandler(coreInst, reason, create): + '''Return the send handler function for a given socket reason''' + retData = '' + if startData == 'chat': + retData = coreInst.chatInst.sendMessage + class OnionrSockets: - def __init__(self, coreInst, socketInfo): + def __init__(self, coreInst, socketInfo, recieveCallback=None, sendCallback=None): '''Create a new Socket object. This interface is named a bit misleadingly and does not actually forward network requests. @@ -36,6 +48,12 @@ class OnionrSockets: self.socketID = secrets.token_hex(32) # Generate an ID for this socket self._core = coreInst self.socketInfo = socketInfo + + if not callable(sendCallback) or not callable(recieveCallback) + raise ValueError("callback must be a function") + + self.sendCallback = sendCallback + self.recieveCallback = recieveCallback # Make sure socketInfo provides all necessary values for i in ('peer', 'address', 'create', 'port'): @@ -50,11 +68,11 @@ class OnionrSockets: self.socketPort = socketInfo['port'] self.serverAddress = socketInfo['address'] self.connected = False - self.segment = 0 - self.connData = {} if self.isServer: self.createServer() + else: + self.connectServer() def createServer(self): # Create our HS and advertise it via a block @@ -87,34 +105,31 @@ class OnionrSockets: callback(key.fileobj, mask) return - - def connectServer(self): - return def _accept(self, sock, mask): # Just accept the connection and pass it to our handler conn, addr = sock.accept() conn.setblocking(False) sel.register(conn, selectors.EVENT_READ, self._read) + self.connected = True def _read(self, conn, mask): - data = conn.recv(1000).decode() + data = conn.recv(1024) if data: - self.segment += 1 - self.connData[self.segment] = data - conn.send(data) + data = data.decode() + self.callback(self, data) else: sel.unregister(conn) conn.close() - - def readConnection(self): - if not self.connected: - raise Exception("Connection closed") - count = 0 - while self.connected: - try: - yield self.connData[count] - count += 1 - except KeyError: - pass - time.sleep(0.01) + + def connectServer(self): + # Set the Tor proxy + socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, '127.0.0.1', config.get('tor.socksport'), rdns=True) + socket.socket = socks.socksocket + remoteSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + with remoteSocket as s: + s.connect((self.serverAddress, self.port)) + data = s.recv(1024) + data.send(self.sendCallback(self, data.decode())) + return \ No newline at end of file diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py index 3b0191ae..940dc424 100644 --- a/onionr/static-data/default-plugins/metadataprocessor/main.py +++ b/onionr/static-data/default-plugins/metadataprocessor/main.py @@ -85,8 +85,12 @@ def on_processBlocks(api): port = api.data['port'] except KeyError: raise ValueError("Missing port for new socket") + try: + reason = api.data['reason'] + except KeyError: + raise ValueError("Missing socket reason") - socketInfo = json.dumps({'peer': api.data['signer'], 'address': address, 'port': port, create = False}) + socketInfo = json.dumps({'peer': api.data['signer'], 'address': address, 'port': port, 'create' = False, 'reason': reason}) api.get_core().daemonQueueAdd('startSocket', socketInfo) def on_init(api, data = None): From 557afb8d9a7869c976a6a87fc5fbfadd45299949 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Wed, 19 Sep 2018 23:36:59 -0500 Subject: [PATCH 20/70] a lot of work on sockets, and added chat module --- onionr/static-data/default-plugins/metadataprocessor/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py index 940dc424..b4065f11 100644 --- a/onionr/static-data/default-plugins/metadataprocessor/main.py +++ b/onionr/static-data/default-plugins/metadataprocessor/main.py @@ -90,7 +90,7 @@ def on_processBlocks(api): except KeyError: raise ValueError("Missing socket reason") - socketInfo = json.dumps({'peer': api.data['signer'], 'address': address, 'port': port, 'create' = False, 'reason': reason}) + socketInfo = json.dumps({'peer': api.data['signer'], 'address': address, 'port': port, 'create': False, 'reason': reason}) api.get_core().daemonQueueAdd('startSocket', socketInfo) def on_init(api, data = None): From 7baa7d5d5f7d9c5535419130087bb26ee0cb2d71 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 20 Sep 2018 00:13:26 -0500 Subject: [PATCH 21/70] work on sockets --- onionr/communicator2.py | 4 +--- onionr/onionr.py | 4 +++- onionr/onionrsockets.py | 31 ++++++++++++++++++++++--------- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 01af7270..e8f5e7f3 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -469,9 +469,7 @@ class OnionrCommunicatorDaemon: # Create a socket or connect to one. # The socket handler (such as the plugin or app using it) is specified in startData['reason] startData = json.loads(cmd[1]) - rCallback = onionrsockets.getSocketCallbackRecieveHandler(self._core, startData['reason'], startData['create']) - sCallback = onionrsockets.getSocketCallbackSendHandler(self._core, startData['reason'], startData['create']) - self.onionrsockets.append(onionrsockets.OnionrSockets(self._core, startData, recieveCallback=rCallback, sendCallback=sCallback)) + self.onionrsockets.append(onionrsockets.OnionrSockets(self._core, startData)) else: logger.info('Recieved daemonQueue command:' + cmd[0]) diff --git a/onionr/onionr.py b/onionr/onionr.py index c8bb2873..1a50783b 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -26,6 +26,7 @@ if sys.version_info[0] == 2 or sys.version_info[1] < 5: print('Error, Onionr requires Python 3.4+') sys.exit(1) import os, base64, random, getpass, shutil, subprocess, requests, time, platform, datetime, re, json, getpass, sqlite3 +import webbrowser from threading import Thread import api, core, config, logger, onionrplugins as plugins, onionrevents as events import onionrutils @@ -217,6 +218,8 @@ class Onionr: 'getpasswd': self.printWebPassword, 'get-passwd': self.printWebPassword, + 'chat': self.onionrCore.chatInst.connect() + 'friend': self.friendCmd } @@ -826,7 +829,6 @@ class Onionr: logger.error('%s add-file ' % sys.argv[0], timestamp = False) def openUI(self): - import webbrowser url = 'http://127.0.0.1:%s/ui/index.html?timingToken=%s' % (config.get('client.port', 59496), self.onionrUtils.getTimeBypassToken()) print('Opening %s ...' % url) diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index 2a63613c..610074c6 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -36,7 +36,7 @@ def getSocketCallbackSendHandler(coreInst, reason, create): retData = coreInst.chatInst.sendMessage class OnionrSockets: - def __init__(self, coreInst, socketInfo, recieveCallback=None, sendCallback=None): + def __init__(self, coreInst, socketInfo): '''Create a new Socket object. This interface is named a bit misleadingly and does not actually forward network requests. @@ -48,12 +48,6 @@ class OnionrSockets: self.socketID = secrets.token_hex(32) # Generate an ID for this socket self._core = coreInst self.socketInfo = socketInfo - - if not callable(sendCallback) or not callable(recieveCallback) - raise ValueError("callback must be a function") - - self.sendCallback = sendCallback - self.recieveCallback = recieveCallback # Make sure socketInfo provides all necessary values for i in ('peer', 'address', 'create', 'port'): @@ -69,6 +63,9 @@ class OnionrSockets: self.serverAddress = socketInfo['address'] self.connected = False + self.readData = [] + self.sendData = 0 + if self.isServer: self.createServer() else: @@ -117,11 +114,25 @@ class OnionrSockets: data = conn.recv(1024) if data: data = data.decode() - self.callback(self, data) + self.readData.append(data) else: sel.unregister(conn) conn.close() + def sendData(self, data): + try: + data = data.encode() + except AttributeError: + pass + self.sendData = data + + def readData(self): + try: + data = self.readData.pop(0) + except IndexError: + data = '' + return data + def connectServer(self): # Set the Tor proxy socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, '127.0.0.1', config.get('tor.socksport'), rdns=True) @@ -131,5 +142,7 @@ class OnionrSockets: with remoteSocket as s: s.connect((self.serverAddress, self.port)) data = s.recv(1024) - data.send(self.sendCallback(self, data.decode())) + if self.sendData != 0: + s.send(self.sendData) + self.sendData = 0 return \ No newline at end of file From e826bca19ef974db931e345173f971b451acaf5b Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 20 Sep 2018 12:04:58 -0500 Subject: [PATCH 22/70] work on sockets --- onionr/communicator2.py | 23 ++++++++++++++++++++--- onionr/core.py | 1 - onionr/onionr.py | 7 ++++++- onionr/onionrchat.py | 20 ++++++++++---------- onionr/onionrsockets.py | 15 ++------------- 5 files changed, 38 insertions(+), 28 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index e8f5e7f3..36b5794d 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -21,7 +21,8 @@ ''' import sys, os, core, config, json, requests, time, logger, threading, base64, onionr import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block -import onionrdaemontools, onionrsockets +import onionrdaemontools, onionrsockets, onionrchat +from dependencies import secrets from defusedxml import minidom class OnionrCommunicatorDaemon: @@ -80,7 +81,8 @@ class OnionrCommunicatorDaemon: self.daemonTools = onionrdaemontools.DaemonTools(self) # Active sockets for direct connections - self.sockets = [] + self.sockets = {} + self.socketExchange = {} # Socket ID exchange if debug or developmentMode: OnionrCommunicatorTimers(self, self.heartbeat, 10) @@ -469,12 +471,27 @@ class OnionrCommunicatorDaemon: # Create a socket or connect to one. # The socket handler (such as the plugin or app using it) is specified in startData['reason] startData = json.loads(cmd[1]) - self.onionrsockets.append(onionrsockets.OnionrSockets(self._core, startData)) + threading.Thread(target=self.startSocket, args=(startData,)).start() else: logger.info('Recieved daemonQueue command:' + cmd[0]) self.decrementThreadCount('daemonCommands') + def startSocket(self, startData): + # Start a socket client + mySocket = onionrsockets.OnionrSockets(self._core, startData)) + self.sockets[mySocket.socketID] = mySocket + + sockProgram = '' # Function for socket handler (application) + + if startData['reason'] == 'chat': + sockProgram = onionrchat.OnionrChat + else: + del self.sockets[mySocket.socketID] # Delete socket if we have no handler for it + + threading.Thread(target=sockProgram, args=(self, mySocket)).start() + mySocket.startConn() + def uploadBlock(self): '''Upload our block to a few peers''' # when inserting a block, we try to upload it to a few peers to add some deniability diff --git a/onionr/core.py b/onionr/core.py index 54219390..f6cb88b6 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -79,7 +79,6 @@ class Core: # Initialize the crypto object self._crypto = onionrcrypto.OnionrCrypto(self) self._blacklist = onionrblacklist.OnionrBlackList(self) - self.chatInst = onionrchat.OnionrChat(self) except Exception as error: logger.error('Failed to initialize core Onionr library.', error=error) diff --git a/onionr/onionr.py b/onionr/onionr.py index 1a50783b..26edf57d 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -218,7 +218,7 @@ class Onionr: 'getpasswd': self.printWebPassword, 'get-passwd': self.printWebPassword, - 'chat': self.onionrCore.chatInst.connect() + 'chat': self.startChat(), 'friend': self.friendCmd } @@ -270,6 +270,11 @@ class Onionr: THIS SECTION HANDLES THE COMMANDS ''' + def startChat(self): + self.onionrCore.daemonQueueAdd() + socketInfo = json.dumps({'peer': api.data['signer'], 'address': address, 'port': port, 'create': True, 'reason': reason}) + self.onionrCore.daemonQueueAdd('startSocket', socketInfo) + def getCommands(self): return self.cmds diff --git a/onionr/onionrchat.py b/onionr/onionrchat.py index a4051828..f37f270e 100644 --- a/onionr/onionrchat.py +++ b/onionr/onionrchat.py @@ -17,14 +17,14 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import logger +import logger, time class OnionrChat: - def __init__(self, coreInst): - return - - def recieveMessage(self, socketInst, data): - logger.info('Got %s' % (data,)) - return '' - - def sendMessage(self, socketInst, data): - return "Hello" \ No newline at end of file + def __init__(self, communicatorInst, socketInst): + self.communicator = communicatorInst + self.socket = socketInst + + while True: + time.sleep(2) + logger.info(self.socket.readData()) + self.socket.sendData('rekt') + return \ No newline at end of file diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index 610074c6..740c9054 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -23,18 +23,6 @@ import onionrexceptions, time, onionrchat from dependencies import secrets sel = selectors.DefaultSelector() -def getSocketCallbackRecieveHandler(coreInst, reason, create): - '''Return the recieve handler function for a given socket reason''' - retData = '' - if startData == 'chat': - retData = coreInst.chatInst.recieveMessage - -def getSocketCallbackSendHandler(coreInst, reason, create): - '''Return the send handler function for a given socket reason''' - retData = '' - if startData == 'chat': - retData = coreInst.chatInst.sendMessage - class OnionrSockets: def __init__(self, coreInst, socketInfo): '''Create a new Socket object. This interface is named a bit misleadingly @@ -64,8 +52,9 @@ class OnionrSockets: self.connected = False self.readData = [] - self.sendData = 0 + self.sendData = 0 + def startConn(): if self.isServer: self.createServer() else: From 55879b71a5dbde1439f58548921dfecadd83771d Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 20 Sep 2018 12:05:44 -0500 Subject: [PATCH 23/70] work on sockets --- onionr/onionr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/onionr/onionr.py b/onionr/onionr.py index 26edf57d..f7224ddc 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -271,7 +271,6 @@ class Onionr: ''' def startChat(self): - self.onionrCore.daemonQueueAdd() socketInfo = json.dumps({'peer': api.data['signer'], 'address': address, 'port': port, 'create': True, 'reason': reason}) self.onionrCore.daemonQueueAdd('startSocket', socketInfo) From c2b0277612e0d247d3050fe05a527cc5c7b22164 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 20 Sep 2018 12:07:50 -0500 Subject: [PATCH 24/70] work on sockets --- onionr/communicator2.py | 2 +- onionr/onionr.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 36b5794d..6716dcbe 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -479,7 +479,7 @@ class OnionrCommunicatorDaemon: def startSocket(self, startData): # Start a socket client - mySocket = onionrsockets.OnionrSockets(self._core, startData)) + mySocket = onionrsockets.OnionrSockets(self._core, startData) self.sockets[mySocket.socketID] = mySocket sockProgram = '' # Function for socket handler (application) diff --git a/onionr/onionr.py b/onionr/onionr.py index f7224ddc..8843aab9 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -218,7 +218,7 @@ class Onionr: 'getpasswd': self.printWebPassword, 'get-passwd': self.printWebPassword, - 'chat': self.startChat(), + 'chat': self.startChat, 'friend': self.friendCmd } From 2164ded6793b6ba3e3281bbc760eb8c65acbd8d3 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 20 Sep 2018 12:15:08 -0500 Subject: [PATCH 25/70] work on sockets --- onionr/onionr.py | 2 +- onionr/static-data/default-plugins/metadataprocessor/main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/onionr/onionr.py b/onionr/onionr.py index 8843aab9..adf247b3 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -271,7 +271,7 @@ class Onionr: ''' def startChat(self): - socketInfo = json.dumps({'peer': api.data['signer'], 'address': address, 'port': port, 'create': True, 'reason': reason}) + socketInfo = json.dumps({'peer': '', 'address': '', 'port': port, 'create': True, 'reason': 'chat'}) self.onionrCore.daemonQueueAdd('startSocket', socketInfo) def getCommands(self): diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py index b4065f11..759db071 100644 --- a/onionr/static-data/default-plugins/metadataprocessor/main.py +++ b/onionr/static-data/default-plugins/metadataprocessor/main.py @@ -49,7 +49,7 @@ def _processForwardKey(api, myBlock): ''' Get the forward secrecy key specified by the user for us to use ''' - peer = onionrusers.OnionrUser(self.api.get_core(), myBlock.signer) + peer = onionrusers.OnionrUser(api.get_core(), myBlock.signer) key = myBlock.getMetadata('newFSKey') # We don't need to validate here probably, but it helps From 7fa41f31e726d240611468b39e86701ac62b1749 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 20 Sep 2018 12:16:37 -0500 Subject: [PATCH 26/70] work on sockets --- onionr/onionr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/onionr.py b/onionr/onionr.py index adf247b3..bb2114ca 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -271,7 +271,7 @@ class Onionr: ''' def startChat(self): - socketInfo = json.dumps({'peer': '', 'address': '', 'port': port, 'create': True, 'reason': 'chat'}) + socketInfo = json.dumps({'peer': '', 'address': '', 'port': 1337, 'create': True, 'reason': 'chat'}) self.onionrCore.daemonQueueAdd('startSocket', socketInfo) def getCommands(self): From d3f4e912f97c7c79d454a0ab22d5c403d085d096 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 20 Sep 2018 12:41:34 -0500 Subject: [PATCH 27/70] work on sockets --- onionr/communicator2.py | 2 +- onionr/netcontroller.py | 2 ++ onionr/onionr.py | 3 ++- onionr/onionrchat.py | 9 +++++---- onionr/onionrsockets.py | 13 +++++++------ 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 6716dcbe..7c8b6c0b 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -489,7 +489,7 @@ class OnionrCommunicatorDaemon: else: del self.sockets[mySocket.socketID] # Delete socket if we have no handler for it - threading.Thread(target=sockProgram, args=(self, mySocket)).start() + threading.Thread(target=sockProgram, args=(self, mySocket.socketID)).start() mySocket.startConn() def uploadBlock(self): diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py index 56b12573..69dd8aeb 100644 --- a/onionr/netcontroller.py +++ b/onionr/netcontroller.py @@ -72,6 +72,8 @@ class NetController: controlPort = random.randint(1025, 65535) + config.set('tor.controlPort', controlPort, savefile=True) + hashedPassword = subprocess.Popen([self.torBinary, '--hash-password', plaintext], stdout=subprocess.PIPE, stderr=subprocess.PIPE) for line in iter(hashedPassword.stdout.readline, b''): password = line.decode() diff --git a/onionr/onionr.py b/onionr/onionr.py index bb2114ca..12e9b2fd 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -271,7 +271,8 @@ class Onionr: ''' def startChat(self): - socketInfo = json.dumps({'peer': '', 'address': '', 'port': 1337, 'create': True, 'reason': 'chat'}) + peer = sys.argv[2] + socketInfo = json.dumps({'peer': '', 'address': peer, 'port': 1337, 'create': True, 'reason': 'chat'}) self.onionrCore.daemonQueueAdd('startSocket', socketInfo) def getCommands(self): diff --git a/onionr/onionrchat.py b/onionr/onionrchat.py index f37f270e..33981d41 100644 --- a/onionr/onionrchat.py +++ b/onionr/onionrchat.py @@ -19,12 +19,13 @@ ''' import logger, time class OnionrChat: - def __init__(self, communicatorInst, socketInst): + def __init__(self, communicatorInst, socketID): self.communicator = communicatorInst - self.socket = socketInst + self.socket = self.communicator.sockets[socketID] while True: time.sleep(2) - logger.info(self.socket.readData()) - self.socket.sendData('rekt') + logger.info('Chat: got %s' % (self.socket.getReadData(),)) + time.sleep(1) + self.socket.addSendData('rekt') return \ No newline at end of file diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index 740c9054..dc9702c1 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -18,7 +18,7 @@ along with this program. If not, see . ''' import stem.control -import socket, selectors, socks, config +import socket, selectors, socks, config, uuid import onionrexceptions, time, onionrchat from dependencies import secrets sel = selectors.DefaultSelector() @@ -53,8 +53,9 @@ class OnionrSockets: self.readData = [] self.sendData = 0 + config.reload() - def startConn(): + def startConn(self): if self.isServer: self.createServer() else: @@ -68,8 +69,8 @@ class OnionrSockets: ourInternalPort = 1338 # Setup the empheral HS - with stem.control.Controller.from_port() as controller: - controller.authenticate() + with stem.control.Controller.from_port(port=config.get('tor.controlPort')) as controller: + controller.authenticate(config.get('tor.controlpassword')) socketHS = controller.create_ephemeral_hidden_service({ourPort: ourInternalPort}, await_publication = True) ourAddress = socketHS.service_id @@ -108,14 +109,14 @@ class OnionrSockets: sel.unregister(conn) conn.close() - def sendData(self, data): + def addSendData(self, data): try: data = data.encode() except AttributeError: pass self.sendData = data - def readData(self): + def getReadData(self): try: data = self.readData.pop(0) except IndexError: From 4e8f7e2761b6dd1adbfa0f7538d0077f7925b9f3 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 20 Sep 2018 23:47:40 -0500 Subject: [PATCH 28/70] work on sockets --- onionr/api.py | 2 +- onionr/communicator2.py | 26 +-- onionr/onionrsockets.py | 149 ++++++------------ .../default-plugins/metadataprocessor/main.py | 2 +- 4 files changed, 56 insertions(+), 123 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index adfabf88..21ba7a66 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -140,7 +140,7 @@ class API: self.overrideCSP = False return resp - + @app.route('/www/private/') def www_private(path): startTime = math.floor(time.time()) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 7c8b6c0b..56a8e8e6 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -108,6 +108,8 @@ class OnionrCommunicatorDaemon: cleanupTimer.count = (cleanupTimer.frequency - 60) announceTimer.count = (cleanupTimer.frequency - 60) + self.socketServer = onionrsockets.OnionrSocketServer(self._core) + # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking try: while not self.shutdown: @@ -467,31 +469,15 @@ class OnionrCommunicatorDaemon: elif cmd[0] == 'uploadBlock': self.blockToUpload = cmd[1] threading.Thread(target=self.uploadBlock).start() - elif cmd[0] == 'startSocket': - # Create a socket or connect to one. - # The socket handler (such as the plugin or app using it) is specified in startData['reason] - startData = json.loads(cmd[1]) - threading.Thread(target=self.startSocket, args=(startData,)).start() + elif cmd[0] == 'addSocket': + socketInfo = json.loads(cmd[1]) + if socketInfo['reason'] in ('chat'): + onionrsockets.OnionrSocketClient(self._core, socketInfo['peer']) else: logger.info('Recieved daemonQueue command:' + cmd[0]) self.decrementThreadCount('daemonCommands') - def startSocket(self, startData): - # Start a socket client - mySocket = onionrsockets.OnionrSockets(self._core, startData) - self.sockets[mySocket.socketID] = mySocket - - sockProgram = '' # Function for socket handler (application) - - if startData['reason'] == 'chat': - sockProgram = onionrchat.OnionrChat - else: - del self.sockets[mySocket.socketID] # Delete socket if we have no handler for it - - threading.Thread(target=sockProgram, args=(self, mySocket.socketID)).start() - mySocket.startConn() - def uploadBlock(self): '''Upload our block to a few peers''' # when inserting a block, we try to upload it to a few peers to add some deniability diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index dc9702c1..bb505c8c 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -18,121 +18,68 @@ along with this program. If not, see . ''' import stem.control -import socket, selectors, socks, config, uuid -import onionrexceptions, time, onionrchat +import socks, config, uuid +import onionrexceptions, time, requests from dependencies import secrets -sel = selectors.DefaultSelector() +from flask import request, Response, abort -class OnionrSockets: - def __init__(self, coreInst, socketInfo): - '''Create a new Socket object. This interface is named a bit misleadingly - and does not actually forward network requests. - - Accepts coreInst, an instance of Onionr core library, and socketInfo, a dict with these values: - 'peer': peer master public key - 'address': string, if we're connecting to a socket, this is the address we connect to. Not applicable if we're creating our own - create: bool - ''' - self.socketID = secrets.token_hex(32) # Generate an ID for this socket +class OnionrSocketServer: + def __init__(self, coreInst): + self.sockets = {} # pubkey: tor address + self.connPool = {} + self.bindPort = 1337 self._core = coreInst - self.socketInfo = socketInfo + self.responseData = {} + self.killSocket = False + app = flask.Flask(__name__) - # Make sure socketInfo provides all necessary values - for i in ('peer', 'address', 'create', 'port'): - try: - socketInfo[i] - except KeyError: - raise ValueError('Must provide peer, address, and create in socketInfo dict argument') + http_server = WSGIServer((socket.service_id, bindPort), app) + http_server.serve_forever() - self.isServer = socketInfo['create'] # if we are the one creating the service + @app.route('/dc/', methods=['POST']) + def acceptConn(self): + data = request.form['data'] + data = self._core._utils.bytesTorStr(data) - self.remotePeer = socketInfo['peer'] - self.socketPort = socketInfo['port'] - self.serverAddress = socketInfo['address'] - self.connected = False - - self.readData = [] - self.sendData = 0 - config.reload() - - def startConn(self): - if self.isServer: - self.createServer() + if request.host in self.connPool: + self.connPool[request.host].append(data) else: - self.connectServer() - - def createServer(self): - # Create our HS and advertise it via a block - dataID = uuid.uuid4().hex - ourAddress = '' - ourPort = 1337 - ourInternalPort = 1338 + self.connPool[request.host] = [data] - # Setup the empheral HS + retData = self.responseData[request.host] + + self.responseData[request.host] = '' + + return retData + + def setResponseData(self, host, data): + self.responseData[host] = data + + def addSocket(self, peer): + bindPort = 1337 with stem.control.Controller.from_port(port=config.get('tor.controlPort')) as controller: controller.authenticate(config.get('tor.controlpassword')) - socketHS = controller.create_ephemeral_hidden_service({ourPort: ourInternalPort}, await_publication = True) - ourAddress = socketHS.service_id - # Advertise the server - meta = {'address': ourAddress, 'port': ourPort} - self._core.insertBlock(dataID, header='openSocket', encryptType='asym', asymPeer=self.remotePeer, sign=True, meta=meta) + socket = controller.create_ephemeral_hidden_service({80: bindPort}, await_publication = True) + self.sockets[peer] = socket.service_id - # Build the socket server - sock = socket.socket() - sock.bind(('127.0.0.1', ourInternalPort)) - sock.listen(100) - sock.setblocking(False) - sel.register(sock, selectors.EVENT_READ, self._accept) + self.responseData[socket.service_id] = '' - while True: - events = sel.select() - for key, mask in events: - callback = key.data - callback(key.fileobj, mask) + self._core.insertBlock(uuid.uuid4(), header='startSocket', sign=True, encryptType='asym', asymPeer=peer, meta={}) + while not self.killSocket: + time.sleep(3) return - - def _accept(self, sock, mask): - # Just accept the connection and pass it to our handler - conn, addr = sock.accept() - conn.setblocking(False) - sel.register(conn, selectors.EVENT_READ, self._read) - self.connected = True - - def _read(self, conn, mask): - data = conn.recv(1024) - if data: - data = data.decode() - self.readData.append(data) - else: - sel.unregister(conn) - conn.close() - - def addSendData(self, data): - try: - data = data.encode() - except AttributeError: - pass - self.sendData = data - def getReadData(self): - try: - data = self.readData.pop(0) - except IndexError: - data = '' - return data +class OnionrSocketClient: + def __init__(self, coreInst): + self.sockets = {} # pubkey: tor address + self.connPool = {} + self.bindPort = 1337 + self._core = coreInst + self.response = '' + self.request = '' + self.connected = False - def connectServer(self): - # Set the Tor proxy - socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, '127.0.0.1', config.get('tor.socksport'), rdns=True) - socket.socket = socks.socksocket - remoteSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - with remoteSocket as s: - s.connect((self.serverAddress, self.port)) - data = s.recv(1024) - if self.sendData != 0: - s.send(self.sendData) - self.sendData = 0 - return \ No newline at end of file + def getResponse(self, peer): + self._core._utils.doPostRequest(self.) \ No newline at end of file diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py index 759db071..85b7d9de 100644 --- a/onionr/static-data/default-plugins/metadataprocessor/main.py +++ b/onionr/static-data/default-plugins/metadataprocessor/main.py @@ -91,7 +91,7 @@ def on_processBlocks(api): raise ValueError("Missing socket reason") socketInfo = json.dumps({'peer': api.data['signer'], 'address': address, 'port': port, 'create': False, 'reason': reason}) - api.get_core().daemonQueueAdd('startSocket', socketInfo) + api.get_core().daemonQueueAdd('addSocket', socketInfo) def on_init(api, data = None): From 759da5509427d60e4dfa0b1b9cba80f5b79b0c33 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 22 Sep 2018 00:01:17 -0500 Subject: [PATCH 29/70] work on sockets --- onionr/communicator2.py | 10 ++++++--- onionr/onionr.py | 5 ++--- onionr/onionrsockets.py | 46 +++++++++++++++++++++++++++++++++++++---- 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 56a8e8e6..108ecf0d 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -109,6 +109,7 @@ class OnionrCommunicatorDaemon: announceTimer.count = (cleanupTimer.frequency - 60) self.socketServer = onionrsockets.OnionrSocketServer(self._core) + self.socketClient = onionrsockets.OnionrSocketClient(self._core) # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking try: @@ -469,10 +470,13 @@ class OnionrCommunicatorDaemon: elif cmd[0] == 'uploadBlock': self.blockToUpload = cmd[1] threading.Thread(target=self.uploadBlock).start() - elif cmd[0] == 'addSocket': + elif cmd[0] == 'startSocket': socketInfo = json.loads(cmd[1]) - if socketInfo['reason'] in ('chat'): - onionrsockets.OnionrSocketClient(self._core, socketInfo['peer']) + peer = socketInfo['peer'] + reason = socketInfo['reason'] + self.socketServer.addSocket(peer, reason) + elif cmd[0] == 'connectSocket': + pass else: logger.info('Recieved daemonQueue command:' + cmd[0]) diff --git a/onionr/onionr.py b/onionr/onionr.py index 12e9b2fd..c1c07628 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -271,9 +271,8 @@ class Onionr: ''' def startChat(self): - peer = sys.argv[2] - socketInfo = json.dumps({'peer': '', 'address': peer, 'port': 1337, 'create': True, 'reason': 'chat'}) - self.onionrCore.daemonQueueAdd('startSocket', socketInfo) + data = json.dumps({'peer': sys.argv[2], 'reason': 'chat'}) + self.onionrCore.daemonQueueAdd('startSocket', data) def getCommands(self): return self.cmds diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index bb505c8c..73920f27 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -19,7 +19,7 @@ ''' import stem.control import socks, config, uuid -import onionrexceptions, time, requests +import onionrexceptions, time, requests, onionrblockapi from dependencies import secrets from flask import request, Response, abort @@ -27,10 +27,12 @@ class OnionrSocketServer: def __init__(self, coreInst): self.sockets = {} # pubkey: tor address self.connPool = {} + self.bindPort = 1337 self._core = coreInst self.responseData = {} self.killSocket = False + app = flask.Flask(__name__) http_server = WSGIServer((socket.service_id, bindPort), app) @@ -55,7 +57,7 @@ class OnionrSocketServer: def setResponseData(self, host, data): self.responseData[host] = data - def addSocket(self, peer): + def addSocket(self, peer, reason=''): bindPort = 1337 with stem.control.Controller.from_port(port=config.get('tor.controlPort')) as controller: controller.authenticate(config.get('tor.controlpassword')) @@ -65,7 +67,7 @@ class OnionrSocketServer: self.responseData[socket.service_id] = '' - self._core.insertBlock(uuid.uuid4(), header='startSocket', sign=True, encryptType='asym', asymPeer=peer, meta={}) + self._core.insertBlock(uuid.uuid4(), header='startSocket', sign=True, encryptType='asym', asymPeer=peer, meta={'reason': reason}) while not self.killSocket: time.sleep(3) @@ -75,11 +77,47 @@ class OnionrSocketClient: def __init__(self, coreInst): self.sockets = {} # pubkey: tor address self.connPool = {} + self.sendData = {} self.bindPort = 1337 self._core = coreInst self.response = '' self.request = '' self.connected = False + self.killSocket = False + def startSocket(self, peer): + address = '' + # Find the newest open socket for a given peer + for block in self._core.getBlocksByType('openSocket'): + block = onionrblockapi.Block(block, core=self._myCore) + if block.decrypt(): + if block.verifySig() and block.signer == peer: + address = block.getMetadata('address') + if self._core._utils.validateID(address): + # If we got their address, it is valid, and verified, we can break out + break + else: + address = '' + if address != '': + self.sockets[peer] = address + data = '' + while not self.killSocket: + try: + data = self.sendData[peer] + except KeyError: + pass + else: + self.sendData[peer] = '' + postData = {'data': data} + self.connPool[peer] = self._core._utils.doPostRequest('http://' + address + '/dc/', data=postData) + def getResponse(self, peer): - self._core._utils.doPostRequest(self.) \ No newline at end of file + retData = '' + try: + retData = self.connPool[peer] + except KeyError: + pass + return + + def sendData(self, peer, data): + self.sendData[peer] = data \ No newline at end of file From 70e2ccbc0a6c27d94cdfc8ca6bc73d4be3c82c40 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 22 Sep 2018 20:21:39 -0500 Subject: [PATCH 30/70] work on sockets --- onionr/communicator2.py | 13 +++++++++---- onionr/onionrsockets.py | 7 ++++--- .../default-plugins/metadataprocessor/main.py | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 108ecf0d..00e80e13 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -19,7 +19,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import sys, os, core, config, json, requests, time, logger, threading, base64, onionr +import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block import onionrdaemontools, onionrsockets, onionrchat from dependencies import secrets @@ -471,12 +471,17 @@ class OnionrCommunicatorDaemon: self.blockToUpload = cmd[1] threading.Thread(target=self.uploadBlock).start() elif cmd[0] == 'startSocket': + # Create our own socket server socketInfo = json.loads(cmd[1]) peer = socketInfo['peer'] reason = socketInfo['reason'] - self.socketServer.addSocket(peer, reason) - elif cmd[0] == 'connectSocket': - pass + threading.Thread(target=self.socketServer.addSocket, args=(peer, reason)).start() + elif cmd[0] == 'addSocket': + # Socket server was created for us + socketInfo = json.loads(cmd[1]) + peer = socketInfo['peer'] + reason = socketInfo['reason'] + threading.Thread(target=self.socketClient.startSocket, args=(peer, reason)).start() else: logger.info('Recieved daemonQueue command:' + cmd[0]) diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index 73920f27..735b22c6 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -18,6 +18,7 @@ along with this program. If not, see . ''' import stem.control +import threading import socks, config, uuid import onionrexceptions, time, requests, onionrblockapi from dependencies import secrets @@ -36,7 +37,7 @@ class OnionrSocketServer: app = flask.Flask(__name__) http_server = WSGIServer((socket.service_id, bindPort), app) - http_server.serve_forever() + threading.Thread(target=http_server.serve_forever).start() @app.route('/dc/', methods=['POST']) def acceptConn(self): @@ -67,7 +68,7 @@ class OnionrSocketServer: self.responseData[socket.service_id] = '' - self._core.insertBlock(uuid.uuid4(), header='startSocket', sign=True, encryptType='asym', asymPeer=peer, meta={'reason': reason}) + self._core.insertBlock(uuid.uuid4(), header='socket', sign=True, encryptType='asym', asymPeer=peer, meta={'reason': reason}) while not self.killSocket: time.sleep(3) @@ -85,7 +86,7 @@ class OnionrSocketClient: self.connected = False self.killSocket = False - def startSocket(self, peer): + def startSocket(self, peer, reason): address = '' # Find the newest open socket for a given peer for block in self._core.getBlocksByType('openSocket'): diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py index 85b7d9de..2b4deb86 100644 --- a/onionr/static-data/default-plugins/metadataprocessor/main.py +++ b/onionr/static-data/default-plugins/metadataprocessor/main.py @@ -75,7 +75,7 @@ def on_processBlocks(api): if api.data['validSig'] == True: _processForwardKey(api, myBlock) # socket blocks - elif blockType == 'openSocket': + elif blockType == 'socket': if api.data['validSig'] == True and myBlock.decrypted: # we check if it is decrypted as a way of seeing if it was for us try: address = api.data['address'] From ad3d7940f5009a77bddcea58304fa76eeb7d3556 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 22 Sep 2018 23:53:09 -0500 Subject: [PATCH 31/70] work on sockets --- onionr/communicator2.py | 9 ++++--- onionr/core.py | 3 +++ onionr/onionr.py | 8 ++++-- onionr/onionrsockets.py | 56 ++++++++++++++++++++++++++--------------- onionr/onionrutils.py | 3 ++- 5 files changed, 52 insertions(+), 27 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 00e80e13..3cf69513 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -108,7 +108,8 @@ class OnionrCommunicatorDaemon: cleanupTimer.count = (cleanupTimer.frequency - 60) announceTimer.count = (cleanupTimer.frequency - 60) - self.socketServer = onionrsockets.OnionrSocketServer(self._core) + self.socketServer = threading.Thread(target=onionrsockets.OnionrSocketServer, args=(self._core,)) + self.socketServer.start() self.socketClient = onionrsockets.OnionrSocketClient(self._core) # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking @@ -124,6 +125,7 @@ class OnionrCommunicatorDaemon: pass logger.info('Goodbye.') + self._core.killSockets = True self._core._utils.localCommand('shutdown') # shutdown the api time.sleep(0.5) @@ -473,9 +475,8 @@ class OnionrCommunicatorDaemon: elif cmd[0] == 'startSocket': # Create our own socket server socketInfo = json.loads(cmd[1]) - peer = socketInfo['peer'] - reason = socketInfo['reason'] - threading.Thread(target=self.socketServer.addSocket, args=(peer, reason)).start() + socketInfo['id'] = uuid.uuid4() + self._core.startSocket = socketInfo elif cmd[0] == 'addSocket': # Socket server was created for us socketInfo = json.loads(cmd[1]) diff --git a/onionr/core.py b/onionr/core.py index f6cb88b6..17036e30 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -50,6 +50,9 @@ class Core: self.dbCreate = dbcreator.DBCreator(self) self.forwardKeysFile = 'data/forward-keys.db' + self.killSockets = False + self.startSocket = {} + self.usageFile = 'data/disk-usage.txt' self.config = config diff --git a/onionr/onionr.py b/onionr/onionr.py index c1c07628..593055ef 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -271,8 +271,12 @@ class Onionr: ''' def startChat(self): - data = json.dumps({'peer': sys.argv[2], 'reason': 'chat'}) - self.onionrCore.daemonQueueAdd('startSocket', data) + try: + data = json.dumps({'peer': sys.argv[2], 'reason': 'chat'}) + except IndexError: + logger.error('Must specify peer to chat with.') + else: + self.onionrCore.daemonQueueAdd('startSocket', data) def getCommands(self): return self.cmds diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index 735b22c6..4d16ea7b 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -20,41 +20,59 @@ import stem.control import threading import socks, config, uuid -import onionrexceptions, time, requests, onionrblockapi +import onionrexceptions, time, requests, onionrblockapi, logger from dependencies import secrets +from gevent.pywsgi import WSGIServer from flask import request, Response, abort - +import flask class OnionrSocketServer: def __init__(self, coreInst): + app = flask.Flask(__name__) self.sockets = {} # pubkey: tor address self.connPool = {} self.bindPort = 1337 self._core = coreInst self.responseData = {} - self.killSocket = False + threading.Thread(target=self.detectShutdown).start() + threading.Thread(target=self.socketStarter).start() app = flask.Flask(__name__) - - http_server = WSGIServer((socket.service_id, bindPort), app) - threading.Thread(target=http_server.serve_forever).start() + self.http_server = WSGIServer(('127.0.0.1', self.bindPort), app) + self.http_server.serve_forever() - @app.route('/dc/', methods=['POST']) - def acceptConn(self): - data = request.form['data'] - data = self._core._utils.bytesTorStr(data) + @app.route('/dc/', methods=['POST']) + def acceptConn(self): + data = request.form['data'] + data = self._core._utils.bytesTorStr(data) - if request.host in self.connPool: - self.connPool[request.host].append(data) - else: - self.connPool[request.host] = [data] + if request.host in self.connPool: + self.connPool[request.host].append(data) + else: + self.connPool[request.host] = [data] - retData = self.responseData[request.host] + retData = self.responseData[request.host] - self.responseData[request.host] = '' + self.responseData[request.host] = '' - return retData + return retData + def socketStarter(self): + while not self._core.killSockets: + try: + self.addSocket(self._core.startSocket['peer'], reason=self._core.startSocket['reason']) + except KeyError: + pass + else: + logger.info('%s socket started with %s' % (self._core.startSocket['reason'], self._core.startSocket['peer'])) + self._core.startSocket = {} + + def detectShutdown(self): + while not self._core.killSockets: + time.sleep(5) + logger.info('Killing socket server') + self.http_server.stop() + def setResponseData(self, host, data): self.responseData[host] = data @@ -68,10 +86,8 @@ class OnionrSocketServer: self.responseData[socket.service_id] = '' - self._core.insertBlock(uuid.uuid4(), header='socket', sign=True, encryptType='asym', asymPeer=peer, meta={'reason': reason}) + self._core.insertBlock(str(uuid.uuid4()), header='socket', sign=True, encryptType='asym', asymPeer=peer, meta={'reason': reason}) - while not self.killSocket: - time.sleep(3) return class OnionrSocketClient: diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index b5fc10c1..15bbc866 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -593,7 +593,8 @@ class OnionrUtils: except ValueError as e: logger.debug('Failed to make request', error = e) except requests.exceptions.RequestException as e: - logger.debug('Error: %s' % str(e)) + if not 'ConnectTimeoutError' in str(e): + logger.debug('Error: %s' % str(e)) retData = False return retData From 711cf3f2d3b01eabea6f3c6af2d5967b6687112f Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 23 Sep 2018 20:47:27 -0500 Subject: [PATCH 32/70] work on sockets --- onionr/communicator2.py | 7 +++--- onionr/core.py | 4 +++ onionr/onionrchat.py | 29 ++++++++++++++-------- onionr/onionrsockets.py | 54 ++++++++++++++++++++++++++--------------- requirements.txt | 1 + 5 files changed, 63 insertions(+), 32 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 3cf69513..24f64c09 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -80,9 +80,7 @@ class OnionrCommunicatorDaemon: #self.daemonTools = onionrdaemontools.DaemonTools(self) self.daemonTools = onionrdaemontools.DaemonTools(self) - # Active sockets for direct connections - self.sockets = {} - self.socketExchange = {} # Socket ID exchange + self._chat = onionrchat.OnionrChat(self) if debug or developmentMode: OnionrCommunicatorTimers(self, self.heartbeat, 10) @@ -112,6 +110,9 @@ class OnionrCommunicatorDaemon: self.socketServer.start() self.socketClient = onionrsockets.OnionrSocketClient(self._core) + # Loads chat messages into memory + threading.Thread(target=self._chat.chatHandler).start() + # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking try: while not self.shutdown: diff --git a/onionr/core.py b/onionr/core.py index 17036e30..ac1c4902 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -50,8 +50,12 @@ class Core: self.dbCreate = dbcreator.DBCreator(self) self.forwardKeysFile = 'data/forward-keys.db' + # Socket data, defined here because of multithreading constraints with gevent self.killSockets = False self.startSocket = {} + self.socketServerConnData = {} + self.socketReasons = {} + self.socketServerResponseData = {} self.usageFile = 'data/disk-usage.txt' self.config = config diff --git a/onionr/onionrchat.py b/onionr/onionrchat.py index 33981d41..6d8fafa6 100644 --- a/onionr/onionrchat.py +++ b/onionr/onionrchat.py @@ -18,14 +18,23 @@ along with this program. If not, see . ''' import logger, time -class OnionrChat: - def __init__(self, communicatorInst, socketID): - self.communicator = communicatorInst - self.socket = self.communicator.sockets[socketID] - while True: - time.sleep(2) - logger.info('Chat: got %s' % (self.socket.getReadData(),)) - time.sleep(1) - self.socket.addSendData('rekt') - return \ No newline at end of file +class OnionrChat: + def __init__(self, communicatorInst): + '''OnionrChat uses onionrsockets (handled by the communicator) to exchange direct chat messages''' + self.communicator = communicatorInst + self._core = self.communicator._core + self._utils = self._core._utils + + self.chats = {} # {'peer': {'date': date, message': message}} + + def chatHandler(self): + while not self.communicator.shutdown: + for peer in self._core.socketServerConnData: + try: + assert self._core.socketReasons[peer] == "chat" + except (AssertionError, KeyError) as e: + continue + else: + self.chats[peer] = {'date': self._core.socketServerConnData[peer]['date'], 'data': self._core.socketServerConnData[peer]['data']} + logger.info("CHAT MESSAGE RECIEVED: %s" % self.chats[peer]['data']) \ No newline at end of file diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index 4d16ea7b..72bdbe38 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -27,12 +27,16 @@ from flask import request, Response, abort import flask class OnionrSocketServer: def __init__(self, coreInst): - app = flask.Flask(__name__) - self.sockets = {} # pubkey: tor address - self.connPool = {} - - self.bindPort = 1337 self._core = coreInst + app = flask.Flask(__name__) + self._core.socketServerConnData = {} + self.bindPort = 0 + + self.sockets = {} + + while self.bindPort < 1024: + self.bindPort = secrets.randbelow(65535) + self.responseData = {} threading.Thread(target=self.detectShutdown).start() @@ -45,15 +49,27 @@ class OnionrSocketServer: def acceptConn(self): data = request.form['data'] data = self._core._utils.bytesTorStr(data) - - if request.host in self.connPool: - self.connPool[request.host].append(data) + data = {'date': self._core._utils.getEpoch(), 'data': data} + myPeer = '' + retData = '' + for peer in self.sockets: + if self.sockets[peer] == request.host: + myPeer = peer + break else: - self.connPool[request.host] = [data] + return "" - retData = self.responseData[request.host] + if request.host in self.sockets: + self._core.socketServerConnData[myPeer].append(data) + else: + self._core.socketServerConnData[myPeer] = [data] - self.responseData[request.host] = '' + try: + retData = self._core.socketServerResponseData[myPeer] + except KeyError: + pass + + self._core.socketServerConnData[myPeer] = '' return retData @@ -66,6 +82,7 @@ class OnionrSocketServer: else: logger.info('%s socket started with %s' % (self._core.startSocket['reason'], self._core.startSocket['peer'])) self._core.startSocket = {} + time.sleep(1) def detectShutdown(self): while not self._core.killSockets: @@ -73,11 +90,11 @@ class OnionrSocketServer: logger.info('Killing socket server') self.http_server.stop() - def setResponseData(self, host, data): - self.responseData[host] = data - def addSocket(self, peer, reason=''): bindPort = 1337 + + assert len(reason) <= 12 + with stem.control.Controller.from_port(port=config.get('tor.controlPort')) as controller: controller.authenticate(config.get('tor.controlpassword')) @@ -86,8 +103,8 @@ class OnionrSocketServer: self.responseData[socket.service_id] = '' - self._core.insertBlock(str(uuid.uuid4()), header='socket', sign=True, encryptType='asym', asymPeer=peer, meta={'reason': reason}) - + self._core.insertBlock(str(uuid.uuid4()), header='socket', sign=True, encryptType='asym', asymPee=peer, meta={'reason': reason}) + self._core.socketReasons[peer] = reason return class OnionrSocketClient: @@ -95,7 +112,6 @@ class OnionrSocketClient: self.sockets = {} # pubkey: tor address self.connPool = {} self.sendData = {} - self.bindPort = 1337 self._core = coreInst self.response = '' self.request = '' @@ -117,7 +133,7 @@ class OnionrSocketClient: address = '' if address != '': self.sockets[peer] = address - data = '' + data = 'hey' while not self.killSocket: try: data = self.sendData[peer] @@ -126,7 +142,7 @@ class OnionrSocketClient: else: self.sendData[peer] = '' postData = {'data': data} - self.connPool[peer] = self._core._utils.doPostRequest('http://' + address + '/dc/', data=postData) + self.connPool[peer] = {'date': self._core._utils.getEpoch(), 'data': self._core._utils.doPostRequest('http://' + address + '/dc/', data=postData)} def getResponse(self, peer): retData = '' diff --git a/requirements.txt b/requirements.txt index 69322e22..05797aed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ simple_crypt==4.1.7 Flask==1.0.2 PySocks==1.6.8 stem==1.6.0 +ntfy==2.6.0 From 49aae74e721a261545e4cf60434c93728c5ab5d8 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 23 Sep 2018 21:02:39 -0500 Subject: [PATCH 33/70] work on sockets --- onionr/onionrsockets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index 72bdbe38..27296a58 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -103,7 +103,7 @@ class OnionrSocketServer: self.responseData[socket.service_id] = '' - self._core.insertBlock(str(uuid.uuid4()), header='socket', sign=True, encryptType='asym', asymPee=peer, meta={'reason': reason}) + self._core.insertBlock(str(uuid.uuid4()), header='socket', sign=True, encryptType='asym', asymPeer=peer, meta={'reason': reason}) self._core.socketReasons[peer] = reason return From 67b9f6e51fbae49829792b0bbfa5c7b09b58e83c Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 24 Sep 2018 16:13:40 -0500 Subject: [PATCH 34/70] work on sockets --- onionr/onionrsockets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index 27296a58..d58a6d21 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -132,6 +132,7 @@ class OnionrSocketClient: else: address = '' if address != '': + logger.info('%s socket client started with %s' % (reason, peer)) self.sockets[peer] = address data = 'hey' while not self.killSocket: From fa701f37dc21969c1d1a3b80bf7930aa22b213bc Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 24 Sep 2018 16:21:59 -0500 Subject: [PATCH 35/70] work on sockets --- onionr/onionrsockets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index d58a6d21..fb96281c 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -121,7 +121,7 @@ class OnionrSocketClient: def startSocket(self, peer, reason): address = '' # Find the newest open socket for a given peer - for block in self._core.getBlocksByType('openSocket'): + for block in self._core.getBlocksByType('socket'): block = onionrblockapi.Block(block, core=self._myCore) if block.decrypt(): if block.verifySig() and block.signer == peer: From 8b4105fac4df74c7b512137b9d7ee34aaaf7139d Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 24 Sep 2018 17:04:17 -0500 Subject: [PATCH 36/70] work on sockets --- onionr/onionrsockets.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index fb96281c..726cb169 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -120,6 +120,7 @@ class OnionrSocketClient: def startSocket(self, peer, reason): address = '' + logger.info('Trying to find socket server for %s' % (peer,)) # Find the newest open socket for a given peer for block in self._core.getBlocksByType('socket'): block = onionrblockapi.Block(block, core=self._myCore) @@ -128,7 +129,8 @@ class OnionrSocketClient: address = block.getMetadata('address') if self._core._utils.validateID(address): # If we got their address, it is valid, and verified, we can break out - break + if block.getMetadata('reason') == 'chat': + break else: address = '' if address != '': From 1a856c365f6a9b94ebd736f91d74b85644ef1f58 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 25 Sep 2018 23:58:11 -0500 Subject: [PATCH 37/70] work on sockets, added profile setter script --- onionr/api.py | 6 +-- onionr/blockprocessor.py | 0 onionr/communicator2.py | 2 +- onionr/config.py | 9 +++- onionr/core.py | 44 +++++++++++-------- onionr/netcontroller.py | 21 ++++++--- onionr/onionr.py | 35 +++++++++------ onionr/onionrblacklist.py | 2 +- onionr/onionrblockapi.py | 4 +- onionr/onionrchat.py | 11 ++++- onionr/onionrcrypto.py | 6 +-- onionr/onionrplugins.py | 11 ++++- onionr/onionrsockets.py | 28 ++++++++---- onionr/onionrutils.py | 15 ++++--- .../default-plugins/metadataprocessor/main.py | 11 +++-- setprofile.sh | 7 +++ 16 files changed, 140 insertions(+), 72 deletions(-) delete mode 100644 onionr/blockprocessor.py create mode 100755 setprofile.sh diff --git a/onionr/api.py b/onionr/api.py index 21ba7a66..79df9451 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -102,7 +102,7 @@ class API: self.mimeType = 'text/plain' self.overrideCSP = False - with open('data/time-bypass.txt', 'w') as bypass: + with open(self._core.dataDir + 'time-bypass.txt', 'w') as bypass: bypass.write(self.timeBypassToken) if not debug and not self._developmentMode: @@ -111,7 +111,7 @@ class API: else: self.host = '127.0.0.1' - with open('data/host.txt', 'w') as file: + with open(self._core.dataDir + 'host.txt', 'w') as file: file.write(self.host) @app.before_request @@ -466,7 +466,7 @@ class API: elif action == 'getData': resp = '' if self._utils.validateHash(data): - if os.path.exists('data/blocks/' + data + '.dat'): + if os.path.exists(self._core.dataDir + 'blocks/' + data + '.dat'): block = Block(hash=data.encode(), core=self._core) resp = base64.b64encode(block.getRaw().encode()).decode() if len(resp) == 0: diff --git a/onionr/blockprocessor.py b/onionr/blockprocessor.py deleted file mode 100644 index e69de29b..00000000 diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 24f64c09..fc62fe44 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -459,7 +459,7 @@ class OnionrCommunicatorDaemon: self.announce(cmd[1]) elif cmd[0] == 'runCheck': logger.debug('Status check; looks good.') - open('data/.runcheck', 'w+').close() + open(self._core.dataDir + '.runcheck', 'w+').close() elif cmd[0] == 'connectedPeers': self.printOnlinePeers() elif cmd[0] == 'kex': diff --git a/onionr/config.py b/onionr/config.py index 880b4dba..1e782dbe 100644 --- a/onionr/config.py +++ b/onionr/config.py @@ -20,7 +20,14 @@ import os, json, logger -_configfile = os.path.abspath('data/config.json') +try: + dataDir = os.environ['ONIONR_HOME'] + if not dataDir.endswith('/'): + dataDir += '/' +except KeyError: + dataDir = 'data/' + +_configfile = os.path.abspath(dataDir + 'config.json') _config = {} def get(key, default = None): diff --git a/onionr/core.py b/onionr/core.py index ac1c4902..6bd88eaa 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -35,20 +35,28 @@ class Core: ''' Initialize Core Onionr library ''' + try: - self.queueDB = 'data/queue.db' - self.peerDB = 'data/peers.db' - self.blockDB = 'data/blocks.db' - self.blockDataLocation = 'data/blocks/' - self.addressDB = 'data/address.db' + self.dataDir = os.environ['ONIONR_HOME'] + if not self.dataDir.endswith('/'): + self.dataDir += '/' + except KeyError: + self.dataDir = 'data/' + + try: + self.queueDB = self.dataDir + 'queue.db' + self.peerDB = self.dataDir + 'peers.db' + self.blockDB = self.dataDir + 'blocks.db' + self.blockDataLocation = self.dataDir + 'blocks/' + self.addressDB = self.dataDir + 'address.db' self.hsAddress = '' self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt' self.bootstrapList = [] self.requirements = onionrvalues.OnionrValues() self.torPort = torPort - self.dataNonceFile = 'data/block-nonces.dat' + self.dataNonceFile = self.dataDir + 'block-nonces.dat' self.dbCreate = dbcreator.DBCreator(self) - self.forwardKeysFile = 'data/forward-keys.db' + self.forwardKeysFile = self.dataDir + 'forward-keys.db' # Socket data, defined here because of multithreading constraints with gevent self.killSockets = False @@ -57,20 +65,20 @@ class Core: self.socketReasons = {} self.socketServerResponseData = {} - self.usageFile = 'data/disk-usage.txt' + self.usageFile = self.dataDir + 'disk-usage.txt' self.config = config self.maxBlockSize = 10000000 # max block size in bytes - if not os.path.exists('data/'): - os.mkdir('data/') - if not os.path.exists('data/blocks/'): - os.mkdir('data/blocks/') + if not os.path.exists(self.dataDir): + os.mkdir(self.dataDir) + if not os.path.exists(self.dataDir + 'blocks/'): + os.mkdir(self.dataDir + 'blocks/') if not os.path.exists(self.blockDB): self.createBlockDB() - if os.path.exists('data/hs/hostname'): - with open('data/hs/hostname', 'r') as hs: + if os.path.exists(self.dataDir + '/hs/hostname'): + with open(self.dataDir + '/hs/hostname', 'r') as hs: self.hsAddress = hs.read().strip() # Load bootstrap address list @@ -95,8 +103,8 @@ class Core: def refreshFirstStartVars(self): '''Hack to refresh some vars which may not be set on first start''' - if os.path.exists('data/hs/hostname'): - with open('data/hs/hostname', 'r') as hs: + if os.path.exists(self.dataDir + '/hs/hostname'): + with open(self.dataDir + '/hs/hostname', 'r') as hs: self.hsAddress = hs.read().strip() def addPeer(self, peerID, powID, name=''): @@ -136,7 +144,7 @@ class Core: ''' Add an address to the address database (only tor currently) ''' - if address == config.get('i2p.ownAddr', None): + if address == config.get('i2p.ownAddr', None) or address == self.hsAddress: return False if self._utils.validateID(address): @@ -197,7 +205,7 @@ class Core: c.execute('Delete from hashes where hash=?;', t) conn.commit() conn.close() - blockFile = 'data/blocks/' + block + '.dat' + blockFile = self.dataDir + '/blocks/' + block + '.dat' dataSize = 0 try: ''' Get size of data when loaded as an object/var, rather than on disk, diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py index 69dd8aeb..97e73296 100644 --- a/onionr/netcontroller.py +++ b/onionr/netcontroller.py @@ -28,7 +28,14 @@ class NetController: ''' def __init__(self, hsPort): - self.torConfigLocation = 'data/torrc' + try: + self.dataDir = os.environ['ONIONR_HOME'] + if not self.dataDir.endswith('/'): + self.dataDir += '/' + except KeyError: + self.dataDir = 'data/' + + self.torConfigLocation = self.dataDir + 'torrc' self.readyState = False self.socksPort = random.randint(1024, 65535) self.hsPort = hsPort @@ -81,10 +88,10 @@ class NetController: break torrcData = '''SocksPort ''' + str(self.socksPort) + ''' -HiddenServiceDir data/hs/ +HiddenServiceDir ''' + self.dataDir + '''hs/ \n''' + hsVer + '''\n HiddenServicePort 80 127.0.0.1:''' + str(self.hsPort) + ''' -DataDirectory data/tordata/ +DataDirectory ''' + self.dataDir + '''tordata/ CookieAuthentication 1 ControlPort ''' + str(controlPort) + ''' HashedControlPassword ''' + str(password) + ''' @@ -140,11 +147,11 @@ HashedControlPassword ''' + str(password) + ''' logger.debug('Finished starting Tor.', timestamp=True) self.readyState = True - myID = open('data/hs/hostname', 'r') + myID = open(self.dataDir + 'hs/hostname', 'r') self.myID = myID.read().replace('\n', '') myID.close() - torPidFile = open('data/torPid.txt', 'w') + torPidFile = open(self.dataDir + 'torPid.txt', 'w') torPidFile.write(str(tor.pid)) torPidFile.close() @@ -156,7 +163,7 @@ HashedControlPassword ''' + str(password) + ''' ''' try: - pid = open('data/torPid.txt', 'r') + pid = open(self.dataDir + 'torPid.txt', 'r') pidN = pid.read() pid.close() except FileNotFoundError: @@ -169,7 +176,7 @@ HashedControlPassword ''' + str(password) + ''' try: os.kill(int(pidN), signal.SIGTERM) - os.remove('data/torPid.txt') + os.remove(self.dataDir + 'torPid.txt') except ProcessLookupError: pass except FileNotFoundError: diff --git a/onionr/onionr.py b/onionr/onionr.py index 593055ef..c0c39d23 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -57,18 +57,25 @@ class Onionr: except FileNotFoundError: pass + try: + self.dataDir = os.environ['ONIONR_HOME'] + if not self.dataDir.endswith('/'): + self.dataDir += '/' + except KeyError: + self.dataDir = 'data/' + # Load global configuration data - data_exists = os.path.exists('data/') + data_exists = os.path.exists(self.dataDir) if not data_exists: - os.mkdir('data/') + os.mkdir(self.dataDir) if os.path.exists('static-data/default_config.json'): config.set_config(json.loads(open('static-data/default_config.json').read())) # this is the default config, it will be overwritten if a config file already exists. Else, it saves it else: # the default config file doesn't exist, try hardcoded config - config.set_config({'dev_mode': True, 'log': {'file': {'output': True, 'path': 'data/output.log'}, 'console': {'output': True, 'color': True}}}) + config.set_config({'dev_mode': True, 'log': {'file': {'output': True, 'path': self.dataDir + 'output.log'}, 'console': {'output': True, 'color': True}}}) if not data_exists: config.save() config.reload() # this will read the configuration file into memory @@ -80,7 +87,7 @@ class Onionr: settings = settings | logger.OUTPUT_TO_CONSOLE if config.get('log.file.output', True): settings = settings | logger.OUTPUT_TO_FILE - logger.set_file(config.get('log.file.path', '/tmp/onionr.log')) + logger.set_file(config.get('log.file.path', '/tmp/onionr.log').replace('data/', self.dataDir)) logger.set_settings(settings) if str(config.get('general.dev_mode', True)).lower() == 'true': @@ -102,15 +109,15 @@ class Onionr: print('Enter password to decrypt:') password = getpass.getpass() result = self.onionrCore.dataDirDecrypt(password) - if os.path.exists('data/'): + if os.path.exists(self.dataDir): break else: logger.error('Failed to decrypt: ' + result[1], timestamp = False) else: # If data folder does not exist if not data_exists: - if not os.path.exists('data/blocks/'): - os.mkdir('data/blocks/') + if not os.path.exists(self.dataDir + 'blocks/'): + os.mkdir(self.dataDir + 'blocks/') # Copy default plugins into plugins folder if not os.path.exists(plugins.get_plugins_folder()): @@ -262,7 +269,7 @@ class Onionr: if not self._developmentMode: encryptionPassword = self.onionrUtils.getPassword('Enter password to encrypt directory: ') self.onionrCore.dataDirEncrypt(encryptionPassword) - shutil.rmtree('data/') + shutil.rmtree(self.dataDir) return @@ -695,7 +702,7 @@ class Onionr: powToken = self.onionrCore._crypto.pubKeyPowToken messages = { # info about local client - 'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if self.onionrUtils.isCommunicatorRunning(timeout = 2) else logger.colors.fg.red + 'Offline'), + 'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if self.onionrUtils.isCommunicatorRunning(timeout = 9) else logger.colors.fg.red + 'Offline'), 'Public Key' : self.onionrCore._crypto.pubKey, 'POW Token' : powToken, 'Combined' : self.onionrCore._crypto.pubKey + '-' + powToken, @@ -704,14 +711,14 @@ class Onionr: # file and folder size stats 'div1' : True, # this creates a solid line across the screen, a div - 'Total Block Size' : onionrutils.humanSize(onionrutils.size('data/blocks/')), - 'Total Plugin Size' : onionrutils.humanSize(onionrutils.size('data/plugins/')), - 'Log File Size' : onionrutils.humanSize(onionrutils.size('data/output.log')), + 'Total Block Size' : onionrutils.humanSize(onionrutils.size(self.dataDir + 'blocks/')), + 'Total Plugin Size' : onionrutils.humanSize(onionrutils.size(self.dataDir + 'plugins/')), + 'Log File Size' : onionrutils.humanSize(onionrutils.size(self.dataDir + 'output.log')), # count stats 'div2' : True, 'Known Peers Count' : str(len(self.onionrCore.listPeers()) - 1), - 'Enabled Plugins Count' : str(len(config.get('plugins.enabled', list()))) + ' / ' + str(len(os.listdir('data/plugins/'))), + 'Enabled Plugins Count' : str(len(config.get('plugins.enabled', list()))) + ' / ' + str(len(os.listdir(self.dataDir + 'plugins/'))), 'Known Blocks Count' : str(totalBlocks), 'Percent Blocks Signed' : str(round(100 * signedBlocks / max(totalBlocks, 1), 2)) + '%' } @@ -777,7 +784,7 @@ class Onionr: def get_hostname(self): try: - with open('./data/hs/hostname', 'r') as hostname: + with open('./' + self.dataDir + 'hs/hostname', 'r') as hostname: return hostname.read().strip() except Exception: return None diff --git a/onionr/onionrblacklist.py b/onionr/onionrblacklist.py index 863ddc37..f87ccd65 100644 --- a/onionr/onionrblacklist.py +++ b/onionr/onionrblacklist.py @@ -20,7 +20,7 @@ import sqlite3, os, logger class OnionrBlackList: def __init__(self, coreInst): - self.blacklistDB = 'data/blacklist.db' + self.blacklistDB = coreInst.dataDir + 'blacklist.db' self._core = coreInst if not os.path.exists(self.blacklistDB): diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index 78d3e71e..78b4b5bc 100644 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -149,7 +149,7 @@ class Block: # read from file if it's still None if blockdata is None: - filelocation = 'data/blocks/%s.dat' % self.getHash() + filelocation = self.core.dataDir + 'blocks/%s.dat' % self.getHash() if readfile: with open(filelocation, 'rb') as f: @@ -727,7 +727,7 @@ class Block: if type(hash) == Block: blockfile = hash.getBlockFile() else: - blockfile = 'data/blocks/%s.dat' % hash + blockfile = onionrcore.Core().dataDir + 'blocks/%s.dat' % hash return os.path.exists(blockfile) and os.path.isfile(blockfile) diff --git a/onionr/onionrchat.py b/onionr/onionrchat.py index 6d8fafa6..80569e6e 100644 --- a/onionr/onionrchat.py +++ b/onionr/onionrchat.py @@ -27,6 +27,7 @@ class OnionrChat: self._utils = self._core._utils self.chats = {} # {'peer': {'date': date, message': message}} + self.chatSend = {} def chatHandler(self): while not self.communicator.shutdown: @@ -34,7 +35,15 @@ class OnionrChat: try: assert self._core.socketReasons[peer] == "chat" except (AssertionError, KeyError) as e: + logger.warn('Peer is not for chat') continue else: self.chats[peer] = {'date': self._core.socketServerConnData[peer]['date'], 'data': self._core.socketServerConnData[peer]['data']} - logger.info("CHAT MESSAGE RECIEVED: %s" % self.chats[peer]['data']) \ No newline at end of file + logger.info("CHAT MESSAGE RECIEVED: %s" % self.chats[peer]['data']) + for peer in self.communicator.socketClient.sockets: + try: + logger.info(self.communicator.socketClient.connPool[peer]['data']) + self.communicator.socketClient.sendData(peer, "lol") + except: + pass + time.sleep(2) \ No newline at end of file diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index 9e055716..340f8530 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -28,8 +28,8 @@ elif sys.version_info[0] == 3 and sys.version_info[1] >= 6: class OnionrCrypto: def __init__(self, coreInstance): self._core = coreInstance - self._keyFile = 'data/keys.txt' - self.keyPowFile = 'data/keyPow.txt' + self._keyFile = self._core.dataDir + 'keys.txt' + self.keyPowFile = self._core.dataDir + 'keyPow.txt' self.pubKey = None self.privKey = None @@ -42,7 +42,7 @@ class OnionrCrypto: # Load our own pub/priv Ed25519 keys, gen & save them if they don't exist if os.path.exists(self._keyFile): - with open('data/keys.txt', 'r') as keys: + with open(self._core.dataDir + 'keys.txt', 'r') as keys: keys = keys.read().split(',') self.pubKey = keys[0] self.privKey = keys[1] diff --git a/onionr/onionrplugins.py b/onionr/onionrplugins.py index a699433c..ce038856 100644 --- a/onionr/onionrplugins.py +++ b/onionr/onionrplugins.py @@ -21,7 +21,14 @@ import os, re, importlib, config, logger import onionrevents as events -_pluginsfolder = 'data/plugins/' +try: + dataDir = os.environ['ONIONR_HOME'] + if not dataDir.endswith('/'): + dataDir += '/' +except KeyError: + dataDir = 'data/' + +_pluginsfolder = dataDir + 'plugins/' _instances = dict() def reload(onionr = None, stop_event = True): @@ -217,7 +224,7 @@ def get_plugin_data_folder(name, absolute = True): Returns the location of a plugin's data folder ''' - return get_plugins_folder(name, absolute) + 'data/' + return get_plugins_folder(name, absolute) + dataDir def check(): ''' diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py index 726cb169..f20ad4f8 100644 --- a/onionr/onionrsockets.py +++ b/onionr/onionrsockets.py @@ -68,8 +68,8 @@ class OnionrSocketServer: retData = self._core.socketServerResponseData[myPeer] except KeyError: pass - - self._core.socketServerConnData[myPeer] = '' + else: + self._core.socketServerResponseData[myPeer] = '' return retData @@ -99,11 +99,11 @@ class OnionrSocketServer: controller.authenticate(config.get('tor.controlpassword')) socket = controller.create_ephemeral_hidden_service({80: bindPort}, await_publication = True) - self.sockets[peer] = socket.service_id + self.sockets[peer] = socket.service_id + '.onion' - self.responseData[socket.service_id] = '' + self.responseData[socket.service_id + '.onion'] = '' - self._core.insertBlock(str(uuid.uuid4()), header='socket', sign=True, encryptType='asym', asymPeer=peer, meta={'reason': reason}) + self._core.insertBlock(str(uuid.uuid4()), header='socket', sign=True, encryptType='asym', asymPeer=peer, meta={'reason': reason, 'address': socket.service_id + '.onion'}) self._core.socketReasons[peer] = reason return @@ -123,16 +123,26 @@ class OnionrSocketClient: logger.info('Trying to find socket server for %s' % (peer,)) # Find the newest open socket for a given peer for block in self._core.getBlocksByType('socket'): - block = onionrblockapi.Block(block, core=self._myCore) + block = onionrblockapi.Block(block, core=self._core) if block.decrypt(): - if block.verifySig() and block.signer == peer: + theSigner = block.signer + try: + theSigner = theSigner.decode() + except AttributeError: + pass + if block.verifySig() and theSigner == peer: address = block.getMetadata('address') if self._core._utils.validateID(address): # If we got their address, it is valid, and verified, we can break out - if block.getMetadata('reason') == 'chat': + if block.getMetadata('reason') == reason: break + else: + logger.error('The socket the peer opened is not for %s' % (reason,)) else: + logger.error('Peer transport id is invalid for socket: %s' % (address,)) address = '' + else: + logger.warn('Block has invalid sig or id, was for %s' % (theSigner,)) if address != '': logger.info('%s socket client started with %s' % (reason, peer)) self.sockets[peer] = address @@ -140,12 +150,14 @@ class OnionrSocketClient: while not self.killSocket: try: data = self.sendData[peer] + logger.info('Sending %s to %s' % (data, peer)) except KeyError: pass else: self.sendData[peer] = '' postData = {'data': data} self.connPool[peer] = {'date': self._core._utils.getEpoch(), 'data': self._core._utils.doPostRequest('http://' + address + '/dc/', data=postData)} + time.sleep(2) def getResponse(self, peer): retData = '' diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 15bbc866..ba8d6b42 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -52,8 +52,8 @@ class OnionrUtils: Load our timingToken from disk for faster local HTTP API ''' try: - if os.path.exists('data/time-bypass.txt'): - with open('data/time-bypass.txt', 'r') as bypass: + if os.path.exists(self._core.dataDir + 'time-bypass.txt'): + with open(self._core.dataDir + 'time-bypass.txt', 'r') as bypass: self.timingToken = bypass.read() except Exception as error: logger.error('Failed to fetch time bypass token.', error = error) @@ -143,7 +143,7 @@ class OnionrUtils: def getMyAddress(self): try: - with open('./data/hs/hostname', 'r') as hostname: + with open('./' + self._core.dataDir + 'hs/hostname', 'r') as hostname: return hostname.read().strip() except Exception as error: logger.error('Failed to read my address.', error = error) @@ -158,7 +158,7 @@ class OnionrUtils: self.getTimeBypassToken() # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless. try: - with open('data/host.txt', 'r') as host: + with open(self._core.dataDir + 'host.txt', 'r') as host: hostname = host.read() except FileNotFoundError: return False @@ -270,9 +270,10 @@ class OnionrUtils: if len(blockType) <= 10: self._core.updateBlockInfo(blockHash, 'dataType', blockType) - onionrevents.event('processBlocks', data = {'block': myBlock, 'type': blockType, 'signer': signer, 'validSig': valid}, onionr = None) + onionrevents.event('processblocks', data = {'block': myBlock, 'type': blockType, 'signer': signer, 'validSig': valid}, onionr = None) except TypeError: + logger.warn("Missing block information") pass def escapeAnsi(self, line): @@ -479,7 +480,7 @@ class OnionrUtils: def isCommunicatorRunning(self, timeout = 5, interval = 0.1): try: - runcheck_file = 'data/.runcheck' + runcheck_file = self._core.dataDir + '.runcheck' if os.path.isfile(runcheck_file): os.remove(runcheck_file) @@ -593,7 +594,7 @@ class OnionrUtils: except ValueError as e: logger.debug('Failed to make request', error = e) except requests.exceptions.RequestException as e: - if not 'ConnectTimeoutError' in str(e): + if not 'ConnectTimeoutError' in str(e) and not 'Request rejected or failed' in str(e): logger.debug('Error: %s' % str(e)) retData = False return retData diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py index 2b4deb86..6b0b5a28 100644 --- a/onionr/static-data/default-plugins/metadataprocessor/main.py +++ b/onionr/static-data/default-plugins/metadataprocessor/main.py @@ -58,7 +58,7 @@ def _processForwardKey(api, myBlock): else: raise onionrexceptions.InvalidPubkey("%s is nota valid pubkey key" % (key,)) -def on_processBlocks(api): +def on_processblocks(api): # Generally fired by utils. myBlock = api.data['block'] blockType = api.data['type'] @@ -77,21 +77,24 @@ def on_processBlocks(api): # socket blocks elif blockType == 'socket': if api.data['validSig'] == True and myBlock.decrypted: # we check if it is decrypted as a way of seeing if it was for us + logger.info('Detected socket advertised to us...') try: - address = api.data['address'] + address = myBlock.getMetadata('address') except KeyError: raise onionrexceptions.MissingAddress("Missing address for new socket") try: - port = api.data['port'] + port = myBlock.getMetadata('port') except KeyError: raise ValueError("Missing port for new socket") try: - reason = api.data['reason'] + reason = myBlock.getMetadata('reason') except KeyError: raise ValueError("Missing socket reason") socketInfo = json.dumps({'peer': api.data['signer'], 'address': address, 'port': port, 'create': False, 'reason': reason}) api.get_core().daemonQueueAdd('addSocket', socketInfo) + else: + logger.warn("socket is not for us or is invalid") def on_init(api, data = None): diff --git a/setprofile.sh b/setprofile.sh new file mode 100755 index 00000000..3541a652 --- /dev/null +++ b/setprofile.sh @@ -0,0 +1,7 @@ +#!/bin/bash +ONIONR_HOME=. +if [ $# -gt 0 ]; then + ONIONR_HOME=$1 +export ONIONR_HOME +echo "set ONIONR_HOME to $ONIONR_HOME" +fi From 8dbaac21980e73075fbb2e7663e10901bd4be99a Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Wed, 26 Sep 2018 18:40:33 -0500 Subject: [PATCH 38/70] catch signature failure better --- onionr/onionrcrypto.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index 340f8530..8953bf8f 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -78,6 +78,9 @@ class OnionrCrypto: except nacl.exceptions.ValueError: logger.warn('Signature by unknown key (cannot reverse hash)') return False + except binascii.Error: + logger.warn('Could not load key for verification, invalid padding') + return False retData = False sig = base64.b64decode(sig) try: From b5fecdf1e839def7b5f7a6314e085e004c3721e5 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Wed, 26 Sep 2018 20:37:52 -0500 Subject: [PATCH 39/70] logging is less spammy, file disabled by default, and we dont try to scan encrypted blocks that aren't for us --- onionr/api.py | 2 +- onionr/onionrutils.py | 23 +++++++++++++---------- onionr/static-data/default_config.json | 2 +- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index 79df9451..146d43c5 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -515,7 +515,7 @@ class API: while len(self._core.hsAddress) == 0: self._core.refreshFirstStartVars() time.sleep(0.5) - self.http_server = WSGIServer((self.host, bindPort), app) + self.http_server = WSGIServer((self.host, bindPort), app, log=None) self.http_server.serve_forever() except KeyboardInterrupt: pass diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index ba8d6b42..f5820059 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -263,18 +263,21 @@ class OnionrUtils: myBlock = Block(blockHash, self._core) if myBlock.isEncrypted: myBlock.decrypt() - blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks - signer = self.bytesToStr(myBlock.signer) - valid = myBlock.verifySig() - try: - if len(blockType) <= 10: - self._core.updateBlockInfo(blockHash, 'dataType', blockType) + if myBlock.decrypted: + blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks + signer = self.bytesToStr(myBlock.signer) + valid = myBlock.verifySig() + try: + if len(blockType) <= 10: + self._core.updateBlockInfo(blockHash, 'dataType', blockType) - onionrevents.event('processblocks', data = {'block': myBlock, 'type': blockType, 'signer': signer, 'validSig': valid}, onionr = None) + onionrevents.event('processblocks', data = {'block': myBlock, 'type': blockType, 'signer': signer, 'validSig': valid}, onionr = None) - except TypeError: - logger.warn("Missing block information") - pass + except TypeError: + logger.warn("Missing block information") + pass + else: + logger.debug('Not processing metadata on encrypted block we cannot decrypt.') def escapeAnsi(self, line): ''' diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 5ebcfa4a..de27f4c1 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -30,7 +30,7 @@ "log": { "file": { - "output": true, + "output": false, "path": "data/output.log" }, From 761dc9eb95afa89aae294c6a26bbaf43ae7b048c Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 28 Sep 2018 12:29:07 -0500 Subject: [PATCH 40/70] use timeouts in sqlite3 --- onionr/core.py | 42 +++++++++++++++++++++--------------------- onionr/onionrusers.py | 8 ++++---- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index 6bd88eaa..c3f721c6 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -120,7 +120,7 @@ class Core: events.event('pubkey_add', data = {'key': peerID}, onionr = None) - conn = sqlite3.connect(self.peerDB) + conn = sqlite3.connect(self.peerDB, timeout=10) hashID = self._crypto.pubKeyHashID(peerID) c = conn.cursor() t = (peerID, name, 'unknown', hashID, powID, 0) @@ -148,7 +148,7 @@ class Core: return False if self._utils.validateID(address): - conn = sqlite3.connect(self.addressDB) + conn = sqlite3.connect(self.addressDB, timeout=10) c = conn.cursor() # check if address is in database # this is safe to do because the address is validated above, but we strip some chars here too just in case @@ -180,7 +180,7 @@ class Core: Remove an address from the address database ''' if self._utils.validateID(address): - conn = sqlite3.connect(self.addressDB) + conn = sqlite3.connect(self.addressDB, timeout=10) c = conn.cursor() t = (address,) c.execute('Delete from adders where address=?;', t) @@ -199,7 +199,7 @@ class Core: **You may want blacklist.addToDB(blockHash) ''' if self._utils.validateHash(block): - conn = sqlite3.connect(self.blockDB) + conn = sqlite3.connect(self.blockDB, timeout=10) c = conn.cursor() t = (block,) c.execute('Delete from hashes where hash=?;', t) @@ -246,7 +246,7 @@ class Core: raise Exception('Block db does not exist') if self._utils.hasBlock(newHash): return - conn = sqlite3.connect(self.blockDB) + conn = sqlite3.connect(self.blockDB, timeout=10) c = conn.cursor() currentTime = self._utils.getEpoch() if selfInsert or dataSaved: @@ -305,7 +305,7 @@ class Core: blockFile = open(blockFileName, 'wb') blockFile.write(data) blockFile.close() - conn = sqlite3.connect(self.blockDB) + conn = sqlite3.connect(self.blockDB, timeout=10) c = conn.cursor() c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = '" + dataHash + "';") conn.commit() @@ -363,7 +363,7 @@ class Core: if not os.path.exists(self.queueDB): self.makeDaemonDB() else: - conn = sqlite3.connect(self.queueDB) + conn = sqlite3.connect(self.queueDB, timeout=10) c = conn.cursor() try: for row in c.execute('SELECT command, data, date, min(ID) FROM commands group by id'): @@ -383,7 +383,7 @@ class Core: def makeDaemonDB(self): '''generate the daemon queue db''' - conn = sqlite3.connect(self.queueDB) + conn = sqlite3.connect(self.queueDB, timeout=10) c = conn.cursor() # Create table c.execute('''CREATE TABLE commands @@ -398,7 +398,7 @@ class Core: retData = True # Intended to be used by the web server date = self._utils.getEpoch() - conn = sqlite3.connect(self.queueDB) + conn = sqlite3.connect(self.queueDB, timeout=10) c = conn.cursor() t = (command, data, date) try: @@ -416,7 +416,7 @@ class Core: ''' Clear the daemon queue (somewhat dangerous) ''' - conn = sqlite3.connect(self.queueDB) + conn = sqlite3.connect(self.queueDB, timeout=10) c = conn.cursor() try: c.execute('DELETE FROM commands;') @@ -432,7 +432,7 @@ class Core: ''' Return a list of addresses ''' - conn = sqlite3.connect(self.addressDB) + conn = sqlite3.connect(self.addressDB, timeout=10) c = conn.cursor() if randomOrder: addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();') @@ -451,7 +451,7 @@ class Core: randomOrder determines if the list should be in a random order trust sets the minimum trust to list ''' - conn = sqlite3.connect(self.peerDB) + conn = sqlite3.connect(self.peerDB, timeout=10) c = conn.cursor() payload = "" if trust not in (0, 1, 2): @@ -495,7 +495,7 @@ class Core: hashID text 7 pow text 8 ''' - conn = sqlite3.connect(self.peerDB) + conn = sqlite3.connect(self.peerDB, timeout=10) c = conn.cursor() command = (peer,) infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'bytesStored': 4, 'trust': 5, 'pubkeyExchanged': 6, 'hashID': 7} @@ -517,7 +517,7 @@ class Core: ''' Update a peer for a key ''' - conn = sqlite3.connect(self.peerDB) + conn = sqlite3.connect(self.peerDB, timeout=10) c = conn.cursor() command = (data, peer) # TODO: validate key on whitelist @@ -541,7 +541,7 @@ class Core: failure int 6 lastConnect 7 ''' - conn = sqlite3.connect(self.addressDB) + conn = sqlite3.connect(self.addressDB, timeout=10) c = conn.cursor() command = (address,) infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'DBHash': 5, 'failure': 6, 'lastConnect': 7} @@ -562,7 +562,7 @@ class Core: ''' Update an address for a key ''' - conn = sqlite3.connect(self.addressDB) + conn = sqlite3.connect(self.addressDB, timeout=10) c = conn.cursor() command = (data, address) # TODO: validate key on whitelist @@ -578,7 +578,7 @@ class Core: ''' Get list of our blocks ''' - conn = sqlite3.connect(self.blockDB) + conn = sqlite3.connect(self.blockDB, timeout=10) c = conn.cursor() if unsaved: execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();' @@ -595,7 +595,7 @@ class Core: ''' Returns the date a block was received ''' - conn = sqlite3.connect(self.blockDB) + conn = sqlite3.connect(self.blockDB, timeout=10) c = conn.cursor() execute = 'SELECT dateReceived FROM hashes WHERE hash=?;' args = (blockHash,) @@ -609,7 +609,7 @@ class Core: ''' Returns a list of blocks by the type ''' - conn = sqlite3.connect(self.blockDB) + conn = sqlite3.connect(self.blockDB, timeout=10) c = conn.cursor() if orderDate: execute = 'SELECT hash FROM hashes WHERE dataType=? ORDER BY dateReceived;' @@ -628,7 +628,7 @@ class Core: Sets the type of block ''' - conn = sqlite3.connect(self.blockDB) + conn = sqlite3.connect(self.blockDB, timeout=10) c = conn.cursor() c.execute("UPDATE hashes SET dataType='" + blockType + "' WHERE hash = '" + hash + "';") conn.commit() @@ -653,7 +653,7 @@ class Core: if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed'): return False - conn = sqlite3.connect(self.blockDB) + conn = sqlite3.connect(self.blockDB, timeout=10) c = conn.cursor() args = (data, hash) c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args) diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index 1e8cdf29..d950ec33 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -69,7 +69,7 @@ class OnionrUser: def _getLatestForwardKey(self): # Get the latest forward secrecy key for a peer - conn = sqlite3.connect(self._core.peerDB) + conn = sqlite3.connect(self._core.peerDB, timeout=10) c = conn.cursor() for row in c.execute("SELECT forwardKey FROM forwardKeys WHERE peerKey = ? AND date=(SELECT max(date) FROM forwardKeys)", (self.publicKey,)): @@ -81,7 +81,7 @@ class OnionrUser: return key def _getForwardKeys(self): - conn = sqlite3.connect(self._core.peerDB) + conn = sqlite3.connect(self._core.peerDB, timeout=10) c = conn.cursor() keyList = [] for row in c.execute("SELECT forwardKey FROM forwardKeys WHERE peerKey = ?", (self.publicKey,)): @@ -96,7 +96,7 @@ class OnionrUser: def generateForwardKey(self, expire=432000): # Generate a forward secrecy key for the peer - conn = sqlite3.connect(self._core.forwardKeysFile) + conn = sqlite3.connect(self._core.forwardKeysFile, timeout=10) c = conn.cursor() # Prepare the insert time = self._core._utils.getEpoch() @@ -117,7 +117,7 @@ class OnionrUser: if not self._core._utils.validatePubKey(newKey): raise onionrexceptions.InvalidPubkey # Add a forward secrecy key for the peer - conn = sqlite3.connect(self._core.peerDB) + conn = sqlite3.connect(self._core.peerDB, timeout=10) c = conn.cursor() # Prepare the insert time = self._core._utils.getEpoch() From b344c535630af26cd92c2bbcc777efe8d49c7c00 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 29 Sep 2018 23:42:31 -0500 Subject: [PATCH 41/70] delete expired blocks --- onionr/core.py | 18 ++++++++++++++++-- onionr/dbcreator.py | 4 +++- onionr/onionrdaemontools.py | 9 +++++++-- onionr/onionrutils.py | 10 ++++++++-- 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index c3f721c6..bb889813 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -620,9 +620,22 @@ class Core: for row in c.execute(execute, args): for i in row: rows.append(i) - return rows + def getExpiredBlocks(self): + '''Returns a list of expired blocks''' + conn = sqlite3.connect(self.blockDB, timeout=10) + c = conn.cursor() + date = int(self._utils.getEpoch()) + + execute = 'SELECT hash FROM hashes WHERE expire >= %s ORDER BY dateReceived;' % (date,) + + rows = list() + for row in c.execute(execute): + for i in row: + rows.append(i) + return rows + def setBlockType(self, hash, blockType): ''' Sets the type of block @@ -648,9 +661,10 @@ class Core: sig - optional signature by the author (not optional if author is specified) author - multi-round partial sha3-256 hash of authors public key dateClaimed - timestamp claimed inside the block, only as trustworthy as the block author is + expire - expire date for a block ''' - if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed'): + if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'): return False conn = sqlite3.connect(self.blockDB, timeout=10) diff --git a/onionr/dbcreator.py b/onionr/dbcreator.py index f9838342..7f970310 100644 --- a/onionr/dbcreator.py +++ b/onionr/dbcreator.py @@ -91,6 +91,7 @@ class DBCreator: sig - optional signature by the author (not optional if author is specified) author - multi-round partial sha3-256 hash of authors public key dateClaimed - timestamp claimed inside the block, only as trustworthy as the block author is + expire int - block expire date in epoch ''' if os.path.exists(self.core.blockDB): raise Exception("Block database already exists") @@ -105,7 +106,8 @@ class DBCreator: dataSaved int, sig text, author text, - dateClaimed int + dateClaimed int, + expire int ); ''') conn.commit() diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index bbd7af64..8192e0de 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -65,12 +65,17 @@ class DaemonTools: self.daemon.decrementThreadCount('netCheck') def cleanOldBlocks(self): - '''Delete old blocks if our disk allocation is full/near full''' + '''Delete old blocks if our disk allocation is full/near full, and also expired blocks''' + while self.daemon._core._utils.storageCounter.isFull(): oldest = self.daemon._core.getBlockList()[0] self.daemon._core._blacklist.addToDB(oldest) self.daemon._core.removeBlock(oldest) - logger.info('Deleted block: %s' % (oldest,)) + logger.info('Deleted block: %s' % (oldest,)) + # Delete expired blocks + for bHash in self.daemon._core.getExpiredBlocks(): + self.daemon._core._blacklist.addToDB(bHash) + self.daemon._core.removeBlock(bHash) self.daemon.decrementThreadCount('cleanOldBlocks') def cooldownPeer(self): diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index f5820059..1c9cf637 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -270,12 +270,18 @@ class OnionrUtils: try: if len(blockType) <= 10: self._core.updateBlockInfo(blockHash, 'dataType', blockType) - onionrevents.event('processblocks', data = {'block': myBlock, 'type': blockType, 'signer': signer, 'validSig': valid}, onionr = None) - except TypeError: logger.warn("Missing block information") pass + # Set block expire time if specified + try: + expireTime = myBlock.getMetadata('expire') + assert len(int(expireTime)) < 20 # test that expire time is an integer of sane length (for epoch) + except (AssertionError, ValueError) as e: + pass + else: + self._core.updateBlockInfo(blockHash, 'expire', expireTime) else: logger.debug('Not processing metadata on encrypted block we cannot decrypt.') From 5f67cc388fadd78c2f547f577eb9a52a9ab775bc Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 30 Sep 2018 11:53:39 -0500 Subject: [PATCH 42/70] delete expired blocks --- onionr/core.py | 12 +++++++++--- onionr/onionr.py | 4 ++-- onionr/onionrblockapi.py | 3 ++- onionr/onionrutils.py | 17 ++++++++++++----- onionr/onionrvalues.py | 2 +- 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index bb889813..8a6f5b15 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -628,7 +628,7 @@ class Core: c = conn.cursor() date = int(self._utils.getEpoch()) - execute = 'SELECT hash FROM hashes WHERE expire >= %s ORDER BY dateReceived;' % (date,) + execute = 'SELECT hash FROM hashes WHERE expire <= %s ORDER BY dateReceived;' % (date,) rows = list() for row in c.execute(execute): @@ -675,7 +675,7 @@ class Core: conn.close() return True - def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = None): + def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = None, expire=None): ''' Inserts a block into the network encryptType must be specified to encrypt a block @@ -753,6 +753,11 @@ class Core: metadata['sig'] = signature metadata['signer'] = signer metadata['time'] = str(self._utils.getEpoch()) + + # ensure expire is integer and of sane length + if type(expire) is not type(None): + assert len(str(int(expire))) < 14 + metadata['expire'] = expire # send block data (and metadata) to POW module to get tokenized block data proof = onionrproofs.POW(metadata, data) @@ -760,7 +765,8 @@ class Core: if payload != False: retData = self.setData(payload) self.addToBlockDB(retData, selfInsert=True, dataSaved=True) - self.setBlockType(retData, meta['type']) + #self.setBlockType(retData, meta['type']) + self._utils.processBlockMetadata(retData) self.daemonQueueAdd('uploadBlock', retData) if retData != False: diff --git a/onionr/onionr.py b/onionr/onionr.py index c0c39d23..80737bde 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -41,9 +41,9 @@ except ImportError: raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)") ONIONR_TAGLINE = 'Anonymous P2P Platform - GPLv3 - https://Onionr.VoidNet.Tech' -ONIONR_VERSION = '0.2.0' # for debugging and stuff +ONIONR_VERSION = '0.3.0' # for debugging and stuff ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION) -API_VERSION = '4' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes know how to communicate without learning too much information about you. +API_VERSION = '5' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes know how to communicate without learning too much information about you. class Onionr: def __init__(self): diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index 78b4b5bc..fea9df1e 100644 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -25,7 +25,7 @@ class Block: blockCacheOrder = list() # NEVER write your own code that writes to this! blockCache = dict() # should never be accessed directly, look at Block.getCache() - def __init__(self, hash = None, core = None, type = None, content = None): + def __init__(self, hash = None, core = None, type = None, content = None, expire=None): # take from arguments # sometimes people input a bytes object instead of str in `hash` if (not hash is None) and isinstance(hash, bytes): @@ -35,6 +35,7 @@ class Block: self.core = core self.btype = type self.bcontent = content + self.expire = expire # initialize variables self.valid = True diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 1c9cf637..59b8829b 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -263,7 +263,7 @@ class OnionrUtils: myBlock = Block(blockHash, self._core) if myBlock.isEncrypted: myBlock.decrypt() - if myBlock.decrypted: + if (myBlock.isEncrypted and myBlock.decrypted) or (not myBlock.isEncrypted): blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks signer = self.bytesToStr(myBlock.signer) valid = myBlock.verifySig() @@ -276,9 +276,9 @@ class OnionrUtils: pass # Set block expire time if specified try: - expireTime = myBlock.getMetadata('expire') - assert len(int(expireTime)) < 20 # test that expire time is an integer of sane length (for epoch) - except (AssertionError, ValueError) as e: + expireTime = myBlock.getHeader('expire') + assert len(str(int(expireTime))) < 20 # test that expire time is an integer of sane length (for epoch) + except (AssertionError, ValueError, TypeError) as e: pass else: self._core.updateBlockInfo(blockHash, 'expire', expireTime) @@ -379,8 +379,14 @@ class OnionrUtils: if not self.isIntegerString(metadata[i]): logger.warn('Block metadata time stamp is not integer string') break + elif i == 'expire': + try: + assert int(metadata[i]) > self.getEpoch() + except AssertionError: + logger.warn('Block is expired') + break else: - # if metadata loop gets no errors, it does not break, therefore metadata is valid + # if metadata loop gets no errors, it does not break, therefore metadata is valid # make sure we do not have another block with the same data content (prevent data duplication and replay attacks) nonce = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(blockData)) try: @@ -532,6 +538,7 @@ class OnionrUtils: if self._core._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''): self._core.addToBlockDB(block.replace('.dat', ''), dataSaved=True) logger.info('Imported block %s.' % block) + self._core._utils.processBlockMetadata(block) else: logger.warn('Failed to verify hash for %s' % block) diff --git a/onionr/onionrvalues.py b/onionr/onionrvalues.py index 3f806702..84203d88 100644 --- a/onionr/onionrvalues.py +++ b/onionr/onionrvalues.py @@ -21,4 +21,4 @@ class OnionrValues: def __init__(self): self.passwordLength = 20 - self.blockMetadataLengths = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'powRandomToken': 1000, 'encryptType': 4} #TODO properly refine values to minimum needed \ No newline at end of file + self.blockMetadataLengths = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'powRandomToken': 1000, 'encryptType': 4, 'expire': 14} #TODO properly refine values to minimum needed \ No newline at end of file From 0b9bb42927fbf31d4929365dc80d33128d312d14 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 2 Oct 2018 00:02:05 -0500 Subject: [PATCH 43/70] bug fixes and work on fs --- onionr/communicator2.py | 2 +- onionr/onionrblockapi.py | 11 ++++++++++- onionr/onionrutils.py | 7 ++++++- onionr/static-data/default-plugins/flow/main.py | 4 ++-- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index fc62fe44..5a293c0f 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -94,7 +94,7 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.getBlocks'), requiresPeer=True) OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58) OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 65) - OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True) + #OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True) OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True) OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True) netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index fea9df1e..c14afc46 100644 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -228,7 +228,7 @@ class Block: blockFile.write(self.getRaw().encode()) self.update() else: - self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign) + self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign, expire=self.getExpire()) self.update() return self.getHash() @@ -241,6 +241,15 @@ class Block: # getters + def getExpire(self): + ''' + Returns the expire time for a block + + Outputs: + - (int): the expire time for a block, or None + ''' + return self.expire + def getHash(self): ''' Returns the hash of the block if saved to file diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 59b8829b..c9eea940 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -372,7 +372,12 @@ class OnionrUtils: logger.warn('Block has invalid metadata key ' + i) break else: - if self._core.requirements.blockMetadataLengths[i] < len(metadata[i]): + testData = metadata[i] + try: + testData = len(testData) + except (TypeError, AttributeError) as e: + testData = len(str(testData)) + if self._core.requirements.blockMetadataLengths[i] < testData: logger.warn('Block metadata key ' + i + ' exceeded maximum size') break if i == 'time': diff --git a/onionr/static-data/default-plugins/flow/main.py b/onionr/static-data/default-plugins/flow/main.py index b2fb1dfa..a7f8842b 100644 --- a/onionr/static-data/default-plugins/flow/main.py +++ b/onionr/static-data/default-plugins/flow/main.py @@ -45,9 +45,9 @@ class OnionrFlow: self.flowRunning = False if message == "q": self.flowRunning = False - + expireTime = self.myCore._utils.getEpoch() + 43200 if len(message) > 0: - Block(content = message, type = 'txt', core = self.myCore).save() + Block(content = message, type = 'txt', expire=expireTime, core = self.myCore).save() logger.info("Flow is exiting, goodbye") return From 15877449f8115ef43d324620ad0c66f0017022dc Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 2 Oct 2018 11:45:56 -0500 Subject: [PATCH 44/70] bug fixes --- onionr/communicator2.py | 7 +++++-- onionr/onionrcrypto.py | 2 +- onionr/static-data/default-plugins/pms/main.py | 7 +++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 5a293c0f..c3a104f2 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -220,7 +220,7 @@ class OnionrCommunicatorDaemon: logger.info("Attempting to download %s..." % blockHash) peerUsed = self.pickOnlinePeer() content = self.peerAction(peerUsed, 'getData', data=blockHash) # block content from random peer (includes metadata) - if content != False: + if content != False and len(content) > 0: try: content = content.encode() except AttributeError: @@ -266,7 +266,10 @@ class OnionrCommunicatorDaemon: onionrpeers.PeerProfiles(peerUsed, self._core).addScore(-50) logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash) if removeFromQueue: - self.blockQueue.remove(blockHash) # remove from block queue both if success or false + try: + self.blockQueue.remove(blockHash) # remove from block queue both if success or false + except ValueError: + pass self.currentDownloading.remove(blockHash) self.decrementThreadCount('getBlocks') return diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index 8953bf8f..ab07edd2 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -76,7 +76,7 @@ class OnionrCrypto: try: key = nacl.signing.VerifyKey(key=key, encoder=nacl.encoding.Base32Encoder) except nacl.exceptions.ValueError: - logger.warn('Signature by unknown key (cannot reverse hash)') + #logger.debug('Signature by unknown key (cannot reverse hash)') return False except binascii.Error: logger.warn('Could not load key for verification, invalid padding') diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index f569ec9b..a47983b0 100644 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -68,6 +68,7 @@ class OnionrMail: pmBlocks = {} logger.info('Decrypting messages...') choice = '' + displayList = [] # this could use a lot of memory if someone has recieved a lot of messages for blockHash in self.myCore.getBlocksByType('pm'): @@ -93,8 +94,10 @@ class OnionrMail: senderDisplay = senderKey blockDate = pmBlocks[blockHash].getDate().strftime("%m/%d %H:%M") - print('%s. %s - %s: %s' % (blockCount, blockDate, senderDisplay[:12], blockHash)) - + displayList.append('%s. %s - %s: %s' % (blockCount, blockDate, senderDisplay[:12], blockHash)) + #displayList.reverse() + for i in displayList: + print(i) try: choice = logger.readline('Enter a block number, -r to refresh, or -q to stop: ').strip().lower() except (EOFError, KeyboardInterrupt): From 8de7bd16c6170a373496783831b1444a141d2119 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 6 Oct 2018 13:06:46 -0500 Subject: [PATCH 45/70] work on foward secrecy --- onionr/core.py | 10 +++++++++- onionr/onionrusers.py | 17 +++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index 8a6f5b15..691f1739 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -21,7 +21,7 @@ import sqlite3, os, sys, time, math, base64, tarfile, getpass, simplecrypt, hash from onionrblockapi import Block import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues -import onionrblacklist, onionrchat +import onionrblacklist, onionrchat, onionrusers import dbcreator if sys.version_info < (3, 6): try: @@ -731,8 +731,16 @@ class Core: if len(jsonMeta) > 1000: raise onionrexceptions.InvalidMetadata('meta in json encoded form must not exceed 1000 bytes') + user = onionrusers.OnionrUser(self, symKey) + # encrypt block metadata/sig/content if encryptType == 'sym': + + # Encrypt block data with forward secrecy key first, but not meta + forwardEncrypted = onionrusers.OnionrUser(self, key=symKey).forwardEncrypt(data) + data = forwardEncrypted[0] + jsonMeta['newFSKey'] = forwardEncrypted[1] + if len(symKey) < self.requirements.passwordLength: raise onionrexceptions.SecurityError('Weak encryption key') jsonMeta = self._crypto.symmetricEncrypt(jsonMeta, key=symKey, returnEncoded=True).decode() diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index d950ec33..ea664839 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -55,20 +55,23 @@ class OnionrUser: return decrypted def forwardEncrypt(self, data): + self.generateForwardKey() retData = '' forwardKey = self._getLatestForwardKey() if self._core._utils.validatePubKey(forwardKey): encrypted = self._core._crypto.pubKeyEncrypt(data, forwardKey, encodedData=True) else: - raise Exception("No valid forward key available for this user") - return + raise onionrexceptions.InvalidPubkey("No valid forward key available for this user") + return (data, forwardKey) def forwardDecrypt(self, encrypted): retData = '' + for key in self return def _getLatestForwardKey(self): # Get the latest forward secrecy key for a peer + key = "" conn = sqlite3.connect(self._core.peerDB, timeout=10) c = conn.cursor() @@ -111,7 +114,17 @@ class OnionrUser: conn.commit() conn.close() + return newPub + def getGeneratedForwardKeys(self, peer): + # Fetch the keys we generated for the peer, that are still around + conn = sqlite3.connect(self._core.peerDB, timeout=10) + c = conn.cursor() + command = (peer,) + keyList = [] # list of tuples containing pub, private for peer + for result in c.execute("SELECT * FROM myForwardKeys where peer=?", command): + keyList.append((result[1], result[2])) + return keyList def addForwardKey(self, newKey): if not self._core._utils.validatePubKey(newKey): From 5606a07757ae0ad61cab36ec984e2226d1d65bae Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 7 Oct 2018 00:06:44 -0500 Subject: [PATCH 46/70] work on foward secrecy --- onionr/core.py | 2 ++ onionr/onionrexceptions.py | 3 +++ onionr/onionrusers.py | 19 ++++++++++++------- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index 691f1739..442ad901 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -76,6 +76,8 @@ class Core: os.mkdir(self.dataDir + 'blocks/') if not os.path.exists(self.blockDB): self.createBlockDB() + if not os.path.exists(self.forwardKeysFile): + self.dbCreate.createForwardKeyDB() if os.path.exists(self.dataDir + '/hs/hostname'): with open(self.dataDir + '/hs/hostname', 'r') as hs: diff --git a/onionr/onionrexceptions.py b/onionr/onionrexceptions.py index 4954550e..f3cefe36 100644 --- a/onionr/onionrexceptions.py +++ b/onionr/onionrexceptions.py @@ -37,6 +37,9 @@ class InvalidPubkey(Exception): class KeyNotKnown(Exception): pass +class DecryptionError(Exception): + pass + # block exceptions class InvalidMetadata(Exception): pass diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index ea664839..5594a66c 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -59,15 +59,20 @@ class OnionrUser: retData = '' forwardKey = self._getLatestForwardKey() if self._core._utils.validatePubKey(forwardKey): - encrypted = self._core._crypto.pubKeyEncrypt(data, forwardKey, encodedData=True) + retData = self._core._crypto.pubKeyEncrypt(data, forwardKey, encodedData=True) else: raise onionrexceptions.InvalidPubkey("No valid forward key available for this user") - return (data, forwardKey) + return (retData, forwardKey) def forwardDecrypt(self, encrypted): - retData = '' - for key in self - return + retData = "" + for key in self.getGeneratedForwardKeys(): + retData = self._core._crypto.pubKeyDecrypt(encrypted, pubkey=key[1]) + if retData != False: + break + else: + raise onionrexceptions.DecryptionError("Could not decrypt forward secrecy content") + return retData def _getLatestForwardKey(self): # Get the latest forward secrecy key for a peer @@ -116,11 +121,11 @@ class OnionrUser: conn.close() return newPub - def getGeneratedForwardKeys(self, peer): + def getGeneratedForwardKeys(self): # Fetch the keys we generated for the peer, that are still around conn = sqlite3.connect(self._core.peerDB, timeout=10) c = conn.cursor() - command = (peer,) + command = (self.publicKey,) keyList = [] # list of tuples containing pub, private for peer for result in c.execute("SELECT * FROM myForwardKeys where peer=?", command): keyList.append((result[1], result[2])) From 980406b699f888c5dda716321953de1dba6be9e2 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 7 Oct 2018 15:39:22 -0500 Subject: [PATCH 47/70] work on foward secrecy --- onionr/core.py | 13 ++++++++----- onionr/onionrusers.py | 10 +++++----- onionr/onionrutils.py | 4 ++++ onionr/static-data/index.html | 2 +- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index 442ad901..d9ce201d 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -738,11 +738,6 @@ class Core: # encrypt block metadata/sig/content if encryptType == 'sym': - # Encrypt block data with forward secrecy key first, but not meta - forwardEncrypted = onionrusers.OnionrUser(self, key=symKey).forwardEncrypt(data) - data = forwardEncrypted[0] - jsonMeta['newFSKey'] = forwardEncrypted[1] - if len(symKey) < self.requirements.passwordLength: raise onionrexceptions.SecurityError('Weak encryption key') jsonMeta = self._crypto.symmetricEncrypt(jsonMeta, key=symKey, returnEncoded=True).decode() @@ -751,6 +746,14 @@ class Core: signer = self._crypto.symmetricEncrypt(signer, key=symKey, returnEncoded=True).decode() elif encryptType == 'asym': if self._utils.validatePubKey(asymPeer): + # Encrypt block data with forward secrecy key first, but not meta + try: + forwardEncrypted = onionrusers.OnionrUser(self, asymPeer).forwardEncrypt(data) + data = forwardEncrypted[0] + meta['newFSKey'] = forwardEncrypted[1][0] + except onionrexceptions.InvalidPubkey: + meta['newFSKey'] = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys()[0][0] + jsonMeta = json.dumps(meta) jsonMeta = self._crypto.pubKeyEncrypt(jsonMeta, asymPeer, encodedData=True, anonymous=True).decode() data = self._crypto.pubKeyEncrypt(data, asymPeer, encodedData=True, anonymous=True).decode() signature = self._crypto.pubKeyEncrypt(signature, asymPeer, encodedData=True, anonymous=True).decode() diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index 5594a66c..57b1808d 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -115,7 +115,7 @@ class OnionrUser: time = self._core._utils.getEpoch() command = (self.publicKey, newPub, newPriv, time, expire) - c.execute("INSERT INTO myForwardKeys VALUES(?, ?, ?, ?);", command) + c.execute("INSERT INTO myForwardKeys VALUES(?, ?, ?, ?, ?);", command) conn.commit() conn.close() @@ -123,7 +123,7 @@ class OnionrUser: def getGeneratedForwardKeys(self): # Fetch the keys we generated for the peer, that are still around - conn = sqlite3.connect(self._core.peerDB, timeout=10) + conn = sqlite3.connect(self._core.forwardKeysFile, timeout=10) c = conn.cursor() command = (self.publicKey,) keyList = [] # list of tuples containing pub, private for peer @@ -131,7 +131,7 @@ class OnionrUser: keyList.append((result[1], result[2])) return keyList - def addForwardKey(self, newKey): + def addForwardKey(self, newKey, expire=432000): if not self._core._utils.validatePubKey(newKey): raise onionrexceptions.InvalidPubkey # Add a forward secrecy key for the peer @@ -139,9 +139,9 @@ class OnionrUser: c = conn.cursor() # Prepare the insert time = self._core._utils.getEpoch() - command = (self.publicKey, newKey, time) + command = (self.publicKey, newKey, time, expire) - c.execute("INSERT INTO forwardKeys VALUES(?, ?, ?);", command) + c.execute("INSERT INTO forwardKeys VALUES(?, ?, ?, ?);", command) conn.commit() conn.close() diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index c9eea940..f4a06776 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -267,6 +267,10 @@ class OnionrUtils: blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks signer = self.bytesToStr(myBlock.signer) valid = myBlock.verifySig() + + if myBlock.getMetadata('newFSKey') is not None: + onionrusers.OnionrUser(self._core, signer).addForwardKey(myBlock.getMetadata('newFSKey')) + try: if len(blockType) <= 10: self._core.updateBlockInfo(blockHash, 'dataType', blockType) diff --git a/onionr/static-data/index.html b/onionr/static-data/index.html index a48dad37..157f1586 100644 --- a/onionr/static-data/index.html +++ b/onionr/static-data/index.html @@ -2,6 +2,6 @@

The content on this server is not necessarily created by the server owner, and was not necessarily stored specifically with the owner's knowledge of its contents.

-

Onionr is a decentralized, distributed data storage system, that anyone can insert data into.

+

Onionr is a decentralized data storage system that anyone can insert data into.

To learn more about Onionr, see the website at https://Onionr.VoidNet.tech/

From 38913b62ce5a53967b09d1f4c3985ef73079494a Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 7 Oct 2018 21:25:59 -0500 Subject: [PATCH 48/70] work on foward secrecy --- .gitignore | 1 + onionr/core.py | 13 +++++++------ onionr/onionrblockapi.py | 8 +++++++- onionr/onionrusers.py | 2 +- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 26e43b0e..6fcdd586 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ onionr/.onionr-lock core .vscode/* venv/* +onionr/fs* diff --git a/onionr/core.py b/onionr/core.py index d9ce201d..c2cfc85d 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -726,6 +726,13 @@ class Core: except AttributeError: pass + try: + forwardEncrypted = onionrusers.OnionrUser(self, asymPeer).forwardEncrypt(data) + data = forwardEncrypted[0] + meta['newFSKey'] = forwardEncrypted[1][0] + except onionrexceptions.InvalidPubkey: + meta['newFSKey'] = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys()[0][0] + if sign: signature = self._crypto.edSign(jsonMeta.encode() + data, key=self._crypto.privKey, encodeResult=True) signer = self._crypto.pubKey @@ -747,12 +754,6 @@ class Core: elif encryptType == 'asym': if self._utils.validatePubKey(asymPeer): # Encrypt block data with forward secrecy key first, but not meta - try: - forwardEncrypted = onionrusers.OnionrUser(self, asymPeer).forwardEncrypt(data) - data = forwardEncrypted[0] - meta['newFSKey'] = forwardEncrypted[1][0] - except onionrexceptions.InvalidPubkey: - meta['newFSKey'] = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys()[0][0] jsonMeta = json.dumps(meta) jsonMeta = self._crypto.pubKeyEncrypt(jsonMeta, asymPeer, encodedData=True, anonymous=True).decode() data = self._crypto.pubKeyEncrypt(data, asymPeer, encodedData=True, anonymous=True).decode() diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index c14afc46..b2d41cdf 100644 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -18,7 +18,7 @@ along with this program. If not, see . ''' -import core as onionrcore, logger, config, onionrexceptions, nacl.exceptions +import core as onionrcore, logger, config, onionrexceptions, nacl.exceptions, onionrusers import json, os, sys, datetime, base64 class Block: @@ -91,6 +91,12 @@ class Block: self.signature = core._crypto.pubKeyDecrypt(self.signature, anonymous=anonymous, encodedData=encodedData) self.signer = core._crypto.pubKeyDecrypt(self.signer, anonymous=anonymous, encodedData=encodedData) self.signedData = json.dumps(self.bmetadata) + self.bcontent.decode() + try: + assert self.bmetadata['forwardEnc'] is True + except (AssertionError, KeyError) as e: + pass + else: + self.bcontent = onionrusers.OnionrUser(self.core, self.signer).forwardDecrypt() except nacl.exceptions.CryptoError: pass #logger.debug('Could not decrypt block. Either invalid key or corrupted data') diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index 57b1808d..20dced6d 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -55,13 +55,13 @@ class OnionrUser: return decrypted def forwardEncrypt(self, data): - self.generateForwardKey() retData = '' forwardKey = self._getLatestForwardKey() if self._core._utils.validatePubKey(forwardKey): retData = self._core._crypto.pubKeyEncrypt(data, forwardKey, encodedData=True) else: raise onionrexceptions.InvalidPubkey("No valid forward key available for this user") + self.generateForwardKey() return (retData, forwardKey) def forwardDecrypt(self, encrypted): From c823eecfe3200083c20c8186ca2a0da33289520b Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 8 Oct 2018 00:11:46 -0500 Subject: [PATCH 49/70] work on foward secrecy --- onionr/core.py | 11 ++++++----- onionr/onionrblockapi.py | 5 ++++- onionr/onionrusers.py | 6 ++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index c2cfc85d..7a1325cc 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -714,8 +714,6 @@ class Core: meta['type'] = header meta['type'] = str(meta['type']) - jsonMeta = json.dumps(meta) - if encryptType in ('asym', 'sym', ''): metadata['encryptType'] = encryptType else: @@ -729,10 +727,13 @@ class Core: try: forwardEncrypted = onionrusers.OnionrUser(self, asymPeer).forwardEncrypt(data) data = forwardEncrypted[0] - meta['newFSKey'] = forwardEncrypted[1][0] + meta['newFSKey'] = forwardEncrypted[1] + meta['forwardEnc'] = True except onionrexceptions.InvalidPubkey: - meta['newFSKey'] = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys()[0][0] - + onionrusers.OnionrUser(self, asymPeer).generateForwardKey() + fsKey = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys()[0] + meta['newFSKey'] = fsKey[0] + jsonMeta = json.dumps(meta) if sign: signature = self._crypto.edSign(jsonMeta.encode() + data, key=self._crypto.privKey, encodeResult=True) signer = self._crypto.pubKey diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index b2d41cdf..f7d66891 100644 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -96,7 +96,10 @@ class Block: except (AssertionError, KeyError) as e: pass else: - self.bcontent = onionrusers.OnionrUser(self.core, self.signer).forwardDecrypt() + try: + self.bcontent = onionrusers.OnionrUser(self.core, self.signer).forwardDecrypt(self.bcontent) + except onionrexceptions.DecryptionError: + pass except nacl.exceptions.CryptoError: pass #logger.debug('Could not decrypt block. Either invalid key or corrupted data') diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index 20dced6d..e1a3c88b 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -58,7 +58,7 @@ class OnionrUser: retData = '' forwardKey = self._getLatestForwardKey() if self._core._utils.validatePubKey(forwardKey): - retData = self._core._crypto.pubKeyEncrypt(data, forwardKey, encodedData=True) + retData = self._core._crypto.pubKeyEncrypt(data, forwardKey, encodedData=True, anonymous=True) else: raise onionrexceptions.InvalidPubkey("No valid forward key available for this user") self.generateForwardKey() @@ -67,7 +67,8 @@ class OnionrUser: def forwardDecrypt(self, encrypted): retData = "" for key in self.getGeneratedForwardKeys(): - retData = self._core._crypto.pubKeyDecrypt(encrypted, pubkey=key[1]) + retData = self._core._crypto.pubKeyDecrypt(encrypted, pubkey=key[1], anonymous=True) + logger('decrypting ' + key + ' got ' + retData) if retData != False: break else: @@ -132,6 +133,7 @@ class OnionrUser: return keyList def addForwardKey(self, newKey, expire=432000): + logger.info(newKey) if not self._core._utils.validatePubKey(newKey): raise onionrexceptions.InvalidPubkey # Add a forward secrecy key for the peer From fbd82d38fe0df0498cfa8b85b771f9dc074c289f Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 9 Oct 2018 18:36:52 -0500 Subject: [PATCH 50/70] work on foward secrecy --- onionr/core.py | 7 ++++--- onionr/onionrblockapi.py | 8 ++++---- onionr/onionrcrypto.py | 8 ++++++-- onionr/onionrusers.py | 28 +++++++++++++++++++--------- onionr/onionrutils.py | 3 ++- 5 files changed, 35 insertions(+), 19 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index 7a1325cc..df0430c2 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -727,12 +727,13 @@ class Core: try: forwardEncrypted = onionrusers.OnionrUser(self, asymPeer).forwardEncrypt(data) data = forwardEncrypted[0] - meta['newFSKey'] = forwardEncrypted[1] meta['forwardEnc'] = True except onionrexceptions.InvalidPubkey: onionrusers.OnionrUser(self, asymPeer).generateForwardKey() - fsKey = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys()[0] - meta['newFSKey'] = fsKey[0] + else: + logger.info(forwardEncrypted) + fsKey = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys()[0] + meta['newFSKey'] = fsKey[0] jsonMeta = json.dumps(meta) if sign: signature = self._crypto.edSign(jsonMeta.encode() + data, key=self._crypto.privKey, encodeResult=True) diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index f7d66891..dbc05a0a 100644 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -97,12 +97,12 @@ class Block: pass else: try: - self.bcontent = onionrusers.OnionrUser(self.core, self.signer).forwardDecrypt(self.bcontent) - except onionrexceptions.DecryptionError: + self.bcontent = onionrusers.OnionrUser(self.getCore(), self.signer).forwardDecrypt(self.bcontent) + except (onionrexceptions.DecryptionError, nacl.exceptions.CryptoError) as e: + logger.error(str(e)) pass except nacl.exceptions.CryptoError: - pass - #logger.debug('Could not decrypt block. Either invalid key or corrupted data') + logger.debug('Could not decrypt block. Either invalid key or corrupted data') else: retData = True self.decrypted = True diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index ab07edd2..8285f74d 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -142,7 +142,7 @@ class OnionrCrypto: retVal = anonBox.encrypt(data, encoder=encoding) return retVal - def pubKeyDecrypt(self, data, pubkey='', anonymous=False, encodedData=False): + def pubKeyDecrypt(self, data, pubkey='', privkey='', anonymous=False, encodedData=False): '''pubkey decrypt (Curve25519, taken from Ed25519 pubkey)''' retVal = False if encodedData: @@ -154,7 +154,11 @@ class OnionrCrypto: ourBox = nacl.public.Box(ownKey, pubkey) decrypted = ourBox.decrypt(data, encoder=encoding) elif anonymous: - anonBox = nacl.public.SealedBox(ownKey) + if self._core._utils.validatePubKey(privkey): + privkey = nacl.signing.SigningKey(seed=privkey, encoder=nacl.encoding.Base32Encoder()).to_curve25519_private_key() + anonBox = nacl.public.SealedBox(privkey) + else: + anonBox = nacl.public.SealedBox(ownKey) decrypted = anonBox.decrypt(data, encoder=encoding) return decrypted diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index e1a3c88b..7a3f515c 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -18,6 +18,7 @@ along with this program. If not, see . ''' import onionrblockapi, logger, onionrexceptions, json, sqlite3 +import nacl.exceptions class OnionrUser: def __init__(self, coreInst, publicKey): self.trust = 0 @@ -61,15 +62,20 @@ class OnionrUser: retData = self._core._crypto.pubKeyEncrypt(data, forwardKey, encodedData=True, anonymous=True) else: raise onionrexceptions.InvalidPubkey("No valid forward key available for this user") - self.generateForwardKey() + #self.generateForwardKey() return (retData, forwardKey) def forwardDecrypt(self, encrypted): retData = "" + logger.error(self.publicKey) + logger.error(self.getGeneratedForwardKeys()) for key in self.getGeneratedForwardKeys(): - retData = self._core._crypto.pubKeyDecrypt(encrypted, pubkey=key[1], anonymous=True) - logger('decrypting ' + key + ' got ' + retData) - if retData != False: + logger.info(encrypted) + try: + retData = self._core._crypto.pubKeyDecrypt(encrypted, privkey=key[1], anonymous=True, encodedData=True) + except nacl.exceptions.CryptoError: + retData = False + else: break else: raise onionrexceptions.DecryptionError("Could not decrypt forward secrecy content") @@ -110,8 +116,8 @@ class OnionrUser: # Prepare the insert time = self._core._utils.getEpoch() newKeys = self._core._crypto.generatePubKey() - newPub = newKeys[0] - newPriv = newKeys[1] + newPub = self._core._utils.bytesToStr(newKeys[0]) + newPriv = self._core._utils.bytesToStr(newKeys[1]) time = self._core._utils.getEpoch() command = (self.publicKey, newPub, newPriv, time, expire) @@ -126,14 +132,18 @@ class OnionrUser: # Fetch the keys we generated for the peer, that are still around conn = sqlite3.connect(self._core.forwardKeysFile, timeout=10) c = conn.cursor() - command = (self.publicKey,) + pubkey = self.publicKey + pubkey = self._core._utils.bytesToStr(pubkey) + command = (pubkey,) keyList = [] # list of tuples containing pub, private for peer for result in c.execute("SELECT * FROM myForwardKeys where peer=?", command): keyList.append((result[1], result[2])) - return keyList + if len(keyList) == 0: + self.generateForwardKey() + keyList = self.getGeneratedForwardKeys() + return list(keyList) def addForwardKey(self, newKey, expire=432000): - logger.info(newKey) if not self._core._utils.validatePubKey(newKey): raise onionrexceptions.InvalidPubkey # Add a forward secrecy key for the peer diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index f4a06776..a17d27bc 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -262,7 +262,7 @@ class OnionrUtils: ''' myBlock = Block(blockHash, self._core) if myBlock.isEncrypted: - myBlock.decrypt() + logger.warn(myBlock.decrypt()) if (myBlock.isEncrypted and myBlock.decrypted) or (not myBlock.isEncrypted): blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks signer = self.bytesToStr(myBlock.signer) @@ -287,6 +287,7 @@ class OnionrUtils: else: self._core.updateBlockInfo(blockHash, 'expire', expireTime) else: + logger.info(myBlock.isEncrypted) logger.debug('Not processing metadata on encrypted block we cannot decrypt.') def escapeAnsi(self, line): From 220fda02ce1fcf6ba9b5ea44d38b3075fa0eb9be Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 19 Oct 2018 00:04:11 -0500 Subject: [PATCH 51/70] half way done with encryption plugin, fixed encryption bug in onionrcrypto when using non anonymous encryption --- onionr/core.py | 1 + onionr/onionrcrypto.py | 2 +- onionr/onionrusers.py | 5 +++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index df0430c2..65f9f26a 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -732,6 +732,7 @@ class Core: onionrusers.OnionrUser(self, asymPeer).generateForwardKey() else: logger.info(forwardEncrypted) + onionrusers.OnionrUser(self, asymPeer).generateForwardKey() fsKey = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys()[0] meta['newFSKey'] = fsKey[0] jsonMeta = json.dumps(meta) diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index 8285f74d..d93d7c90 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -128,7 +128,7 @@ class OnionrCrypto: encoding = nacl.encoding.RawEncoder if self.privKey != None and not anonymous: - ownKey = nacl.signing.SigningKey(seed=self.privKey, encoder=nacl.encoding.Base32Encoder) + ownKey = nacl.signing.SigningKey(seed=self.privKey, encoder=nacl.encoding.Base32Encoder).to_curve25519_private_key() key = nacl.signing.VerifyKey(key=pubkey, encoder=nacl.encoding.Base32Encoder).to_curve25519_public_key() ourBox = nacl.public.Box(ownKey, key) retVal = ourBox.encrypt(data.encode(), encoder=encoding) diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index 7a3f515c..f1ab241c 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -58,6 +58,7 @@ class OnionrUser: def forwardEncrypt(self, data): retData = '' forwardKey = self._getLatestForwardKey() + logger.info('using ' + forwardKey) if self._core._utils.validatePubKey(forwardKey): retData = self._core._crypto.pubKeyEncrypt(data, forwardKey, encodedData=True, anonymous=True) else: @@ -87,7 +88,7 @@ class OnionrUser: conn = sqlite3.connect(self._core.peerDB, timeout=10) c = conn.cursor() - for row in c.execute("SELECT forwardKey FROM forwardKeys WHERE peerKey = ? AND date=(SELECT max(date) FROM forwardKeys)", (self.publicKey,)): + for row in c.execute("SELECT forwardKey FROM forwardKeys WHERE peerKey = ? order by date desc", (self.publicKey,)): key = row[0] break @@ -99,7 +100,7 @@ class OnionrUser: conn = sqlite3.connect(self._core.peerDB, timeout=10) c = conn.cursor() keyList = [] - for row in c.execute("SELECT forwardKey FROM forwardKeys WHERE peerKey = ?", (self.publicKey,)): + for row in c.execute("SELECT forwardKey FROM forwardKeys WHERE peerKey = ? order by date desc", (self.publicKey,)): key = row[0] keyList.append(key) From 247ae540f9d5b143560aab939502f57d01754ee7 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 21 Oct 2018 00:07:35 -0500 Subject: [PATCH 52/70] reworked offline encryption --- onionr/core.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index 65f9f26a..296fec88 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -724,17 +724,18 @@ class Core: except AttributeError: pass - try: - forwardEncrypted = onionrusers.OnionrUser(self, asymPeer).forwardEncrypt(data) - data = forwardEncrypted[0] - meta['forwardEnc'] = True - except onionrexceptions.InvalidPubkey: + if encryptType == 'sym': + try: + forwardEncrypted = onionrusers.OnionrUser(self, asymPeer).forwardEncrypt(data) + data = forwardEncrypted[0] + meta['forwardEnc'] = True + except onionrexceptions.InvalidPubkey: + onionrusers.OnionrUser(self, asymPeer).generateForwardKey() + else: + logger.info(forwardEncrypted) onionrusers.OnionrUser(self, asymPeer).generateForwardKey() - else: - logger.info(forwardEncrypted) - onionrusers.OnionrUser(self, asymPeer).generateForwardKey() - fsKey = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys()[0] - meta['newFSKey'] = fsKey[0] + fsKey = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys()[0] + meta['newFSKey'] = fsKey[0] jsonMeta = json.dumps(meta) if sign: signature = self._crypto.edSign(jsonMeta.encode() + data, key=self._crypto.privKey, encodeResult=True) From 2c4d08631640b50b4817e2fb312097df8ffb6602 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 21 Oct 2018 11:21:43 -0500 Subject: [PATCH 53/70] reworked offline encryption --- .../default-plugins/encrypt/info.json | 5 + .../default-plugins/encrypt/main.py | 93 +++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 onionr/static-data/default-plugins/encrypt/info.json create mode 100644 onionr/static-data/default-plugins/encrypt/main.py diff --git a/onionr/static-data/default-plugins/encrypt/info.json b/onionr/static-data/default-plugins/encrypt/info.json new file mode 100644 index 00000000..d675197a --- /dev/null +++ b/onionr/static-data/default-plugins/encrypt/info.json @@ -0,0 +1,5 @@ +{ + "name" : "encrypt", + "version" : "1.0", + "author" : "onionr" +} diff --git a/onionr/static-data/default-plugins/encrypt/main.py b/onionr/static-data/default-plugins/encrypt/main.py new file mode 100644 index 00000000..48508e79 --- /dev/null +++ b/onionr/static-data/default-plugins/encrypt/main.py @@ -0,0 +1,93 @@ +''' + Onionr - P2P Microblogging Platform & Social network + + This default plugin allows users to encrypt/decrypt messages without using blocks +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' + +# Imports some useful libraries +import logger, config, threading, time, readline, datetime, sys, json +from onionrblockapi import Block +import onionrexceptions, onionrusers +import locale +locale.setlocale(locale.LC_ALL, '') + +class PlainEncryption: + def __init__(self, api): + self.api = api + return + def encrypt(self): + # peer, data + plaintext = "" + encrypted = "" + # detect if signing is enabled + sign = True + try: + if sys.argv[3].lower() == 'false': + sign = False + except IndexError: + pass + + try: + if not self.api.get_core()._utils.validatePubKey(sys.argv[2]): + raise onionrexceptions.InvalidPubkey + except (ValueError, IndexError) as e: + logger.error("Peer public key not specified") + except onionrexceptions.InvalidPubkey: + logger.error("Invalid public key") + else: + pubkey = sys.argv[2] + # Encrypt if public key is valid + logger.info("Please enter your message (ctrl-d or -q to stop):") + try: + for line in sys.stdin: + if line == '-q\n': + break + plaintext += line + except KeyboardInterrupt: + sys.exit(1) + # Build Message to encrypt + data = {} + myPub = self.api.get_core()._crypto.pubKey + if sign: + data['sig'] = self.api.get_core()._crypto.edSign(plaintext, key=self.api.get_core()._crypto.privKey, encodeResult=True) + data['sig'] = self.api.get_core()._utils.bytesToStr(data['sig']) + data['signer'] = myPub + data['data'] = plaintext + data = json.dumps(data) + plaintext = data + encrypted = self.api.get_core()._crypto.pubKeyEncrypt(plaintext, pubkey, anonymous=True, encodedData=True) + encrypted = self.api.get_core()._utils.bytesToStr(encrypted) + print('ONIONR ENCRYPTED DATA %s END ENCRYPTED DATA' % (encrypted,)) + def decrypt(self, data): + plaintext = "" + encrypted = data + + return + + +def on_init(api, data = None): + ''' + This event is called after Onionr is initialized, but before the command + inputted is executed. Could be called when daemon is starting or when + just the client is running. + ''' + + pluginapi = api + encrypt = PlainEncryption(pluginapi) + api.commands.register(['encrypt'], encrypt.encrypt) + api.commands.register(['decrypt'], encrypt.decrypt) + return \ No newline at end of file From b8644c0441d408c44ac942b82ea605a878af81e1 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 23 Oct 2018 23:54:28 -0500 Subject: [PATCH 54/70] work on offline decryption and fixed pubkey encrypt bug --- onionr/onionrcrypto.py | 2 +- onionr/static-data/default-plugins/encrypt/main.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index d93d7c90..678a4457 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -144,7 +144,7 @@ class OnionrCrypto: def pubKeyDecrypt(self, data, pubkey='', privkey='', anonymous=False, encodedData=False): '''pubkey decrypt (Curve25519, taken from Ed25519 pubkey)''' - retVal = False + decrypted = False if encodedData: encoding = nacl.encoding.Base64Encoder else: diff --git a/onionr/static-data/default-plugins/encrypt/main.py b/onionr/static-data/default-plugins/encrypt/main.py index 48508e79..d4d9306a 100644 --- a/onionr/static-data/default-plugins/encrypt/main.py +++ b/onionr/static-data/default-plugins/encrypt/main.py @@ -74,8 +74,16 @@ class PlainEncryption: print('ONIONR ENCRYPTED DATA %s END ENCRYPTED DATA' % (encrypted,)) def decrypt(self, data): plaintext = "" - encrypted = data - + encrypted = data.replace('ONIONR ENCRYPTED DATA ', '').replace('END ENCRYPTED DATA', '') + myPub = self.api.get_core()._crypto.pubKey + decrypted = self.api.get_core()._crypto.pubKeyDecrypt(encrypted, pubkey, anonymous=True, encodedData=True) + if decrypted == False: + print("Decryption failed") + else: + data = json.loads(decrypted) + if not self.api.get_core()._crypto.edVerify(data['data'], data['signer'], data['sig']): + print("WARNING: THIS MESSAGE HAS AN INVALID SIGNATURE") + print(self.api.get_core()._utils.escapeAnsi(data['data'])) return From a142e8a75288bcff9ea96be3d1d2057b4348c7aa Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 25 Oct 2018 19:56:02 -0500 Subject: [PATCH 55/70] offline encryption plugin can now decrypt --- onionr/onionrcrypto.py | 1 + .../default-plugins/encrypt/main.py | 26 +++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index 678a4457..7b66a7d9 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -159,6 +159,7 @@ class OnionrCrypto: anonBox = nacl.public.SealedBox(privkey) else: anonBox = nacl.public.SealedBox(ownKey) + print(data) decrypted = anonBox.decrypt(data, encoder=encoding) return decrypted diff --git a/onionr/static-data/default-plugins/encrypt/main.py b/onionr/static-data/default-plugins/encrypt/main.py index d4d9306a..2aa87dcb 100644 --- a/onionr/static-data/default-plugins/encrypt/main.py +++ b/onionr/static-data/default-plugins/encrypt/main.py @@ -72,18 +72,34 @@ class PlainEncryption: encrypted = self.api.get_core()._crypto.pubKeyEncrypt(plaintext, pubkey, anonymous=True, encodedData=True) encrypted = self.api.get_core()._utils.bytesToStr(encrypted) print('ONIONR ENCRYPTED DATA %s END ENCRYPTED DATA' % (encrypted,)) - def decrypt(self, data): + def decrypt(self): plaintext = "" + data = "" + logger.info("Please enter your message (ctrl-d or -q to stop):") + try: + for line in sys.stdin: + if line == '-q\n': + break + data += line + except KeyboardInterrupt: + sys.exit(1) + if len(data) <= 1: + return encrypted = data.replace('ONIONR ENCRYPTED DATA ', '').replace('END ENCRYPTED DATA', '') myPub = self.api.get_core()._crypto.pubKey - decrypted = self.api.get_core()._crypto.pubKeyDecrypt(encrypted, pubkey, anonymous=True, encodedData=True) + decrypted = self.api.get_core()._crypto.pubKeyDecrypt(encrypted, privkey=self.api.get_core()._crypto.privKey, anonymous=True, encodedData=True) if decrypted == False: print("Decryption failed") else: data = json.loads(decrypted) - if not self.api.get_core()._crypto.edVerify(data['data'], data['signer'], data['sig']): - print("WARNING: THIS MESSAGE HAS AN INVALID SIGNATURE") - print(self.api.get_core()._utils.escapeAnsi(data['data'])) + print(data['data']) + try: + logger.info("Signing public key: %s" % (data['signer'],)) + assert self.api.get_core()._crypto.edVerify(data['data'], data['signer'], data['sig']) != False + except (AssertionError, KeyError) as e: + logger.warn("WARNING: THIS MESSAGE HAS A MISSING OR INVALID SIGNATURE") + else: + logger.info("Message has good signature.") return From 3da06339f6c51dcbca6262d2230d4221d6816b91 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 26 Oct 2018 18:11:18 -0500 Subject: [PATCH 56/70] removed key sync --- onionr/communicator2.py | 17 ----------------- onionr/onionrcrypto.py | 1 - 2 files changed, 18 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index c3a104f2..19b4db31 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -94,7 +94,6 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.getBlocks'), requiresPeer=True) OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58) OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 65) - #OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True) OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True) OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True) netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) @@ -130,18 +129,6 @@ class OnionrCommunicatorDaemon: self._core._utils.localCommand('shutdown') # shutdown the api time.sleep(0.5) - def lookupKeys(self): - '''Lookup new keys''' - logger.debug('Looking up new keys...') - tryAmount = 1 - for i in range(tryAmount): # amount of times to ask peers for new keys - # Download new key list from random online peers - peer = self.pickOnlinePeer() - newKeys = self.peerAction(peer, action='kex') - self._core._utils.mergeKeys(newKeys) - self.decrementThreadCount('lookupKeys') - return - def lookupAdders(self): '''Lookup new peer addresses''' logger.info('LOOKING UP NEW ADDRESSES') @@ -465,10 +452,6 @@ class OnionrCommunicatorDaemon: open(self._core.dataDir + '.runcheck', 'w+').close() elif cmd[0] == 'connectedPeers': self.printOnlinePeers() - elif cmd[0] == 'kex': - for i in self.timers: - if i.timerFunction.__name__ == 'lookupKeys': - i.count = (i.frequency - 1) elif cmd[0] == 'pex': for i in self.timers: if i.timerFunction.__name__ == 'lookupAdders': diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index 7b66a7d9..678a4457 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -159,7 +159,6 @@ class OnionrCrypto: anonBox = nacl.public.SealedBox(privkey) else: anonBox = nacl.public.SealedBox(ownKey) - print(data) decrypted = anonBox.decrypt(data, encoder=encoding) return decrypted From c073020b80a4e7ffcf28c1965cc1ba22f002e0de Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 26 Oct 2018 22:29:25 -0500 Subject: [PATCH 57/70] Added API check in requests --- onionr/api.py | 8 ++++---- onionr/onionr.py | 5 ++--- onionr/onionrexceptions.py | 3 +++ onionr/onionrutils.py | 6 ++++++ 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index 146d43c5..94c7eeaa 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -22,7 +22,7 @@ from flask import request, Response, abort, send_from_directory from multiprocessing import Process from gevent.pywsgi import WSGIServer import sys, random, threading, hmac, hashlib, base64, time, math, os, json -from core import Core +import core from onionrblockapi import Block import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config @@ -69,7 +69,7 @@ class API: logger.debug('%s not in %s' % (path, mimetypes)) return 'text/plain' - def __init__(self, debug): + def __init__(self, debug, API_VERSION): ''' Initialize the api server, preping variables for later use @@ -88,7 +88,7 @@ class API: self.debug = debug self._privateDelayTime = 3 - self._core = Core() + self._core = core.Core() self._crypto = onionrcrypto.OnionrCrypto(self._core) self._utils = onionrutils.OnionrUtils(self._core) app = flask.Flask(__name__) @@ -133,7 +133,7 @@ class API: resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'" resp.headers['X-Frame-Options'] = 'deny' resp.headers['X-Content-Type-Options'] = "nosniff" - resp.headers['server'] = 'Onionr' + resp.headers['api'] = API_VERSION # reset to text/plain to help prevent browser attacks self.mimeType = 'text/plain' diff --git a/onionr/onionr.py b/onionr/onionr.py index 80737bde..eb56fb7c 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -30,7 +30,6 @@ import webbrowser from threading import Thread import api, core, config, logger, onionrplugins as plugins, onionrevents as events import onionrutils -from onionrutils import OnionrUtils from netcontroller import NetController from onionrblockapi import Block import onionrproofs, onionrexceptions, onionrusers @@ -98,7 +97,7 @@ class Onionr: logger.set_level(logger.LEVEL_INFO) self.onionrCore = core.Core() - self.onionrUtils = OnionrUtils(self.onionrCore) + self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore) # Handle commands @@ -636,7 +635,7 @@ class Onionr: ''' communicatorDaemon = './communicator2.py' - apiThread = Thread(target=api.API, args=(self.debug,)) + apiThread = Thread(target=api.API, args=(self.debug,API_VERSION)) apiThread.start() try: time.sleep(3) diff --git a/onionr/onionrexceptions.py b/onionr/onionrexceptions.py index f3cefe36..d450e3ae 100644 --- a/onionr/onionrexceptions.py +++ b/onionr/onionrexceptions.py @@ -65,6 +65,9 @@ class MissingPort(Exception): class InvalidAddress(Exception): pass +class InvalidAPIVersion(Exception): + pass + # file exceptions class DiskAllocationReached(Exception): diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index a17d27bc..5cc79e90 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -22,6 +22,7 @@ import getpass, sys, requests, os, socket, hashlib, logger, sqlite3, config, bin import nacl.signing, nacl.encoding from onionrblockapi import Block import onionrexceptions +from onionr import API_VERSION from defusedxml import minidom import onionrevents import pgpwords, onionrusers, storagecounter @@ -614,11 +615,16 @@ class OnionrUtils: try: proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)} r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30)) + # Check server is using same API version as us + if r.headers['api'] != str(API_VERSION): + raise onionrexceptions.InvalidAPIVersion retData = r.text except KeyboardInterrupt: raise KeyboardInterrupt except ValueError as e: logger.debug('Failed to make request', error = e) + except onionrexceptions.InvalidAPIVersion: + logger.debug("Node is using different API version :(") except requests.exceptions.RequestException as e: if not 'ConnectTimeoutError' in str(e) and not 'Request rejected or failed' in str(e): logger.debug('Error: %s' % str(e)) From 4e86604692f837fbcfe5fdf8310304318672e0d0 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 27 Oct 2018 14:59:15 -0500 Subject: [PATCH 58/70] handle missing api header --- onionr/onionrutils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 5cc79e90..f102a3a1 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -616,7 +616,10 @@ class OnionrUtils: proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)} r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30)) # Check server is using same API version as us - if r.headers['api'] != str(API_VERSION): + try: + if r.headers['api'] != str(API_VERSION): + raise onionrexceptions.InvalidAPIVersion + except KeyError: raise onionrexceptions.InvalidAPIVersion retData = r.text except KeyboardInterrupt: From 34aa892b6502ece4f4d920acf732073da01bb15c Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 28 Oct 2018 00:06:18 -0500 Subject: [PATCH 59/70] work on using dynamic/configurable POW --- onionr/onionrcrypto.py | 3 ++- onionr/onionrproofs.py | 7 ++++--- onionr/static-data/default_config.json | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index 678a4457..c717935c 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -269,7 +269,8 @@ class OnionrCrypto: pass difficulty = math.floor(dataLen / 1000000) - + if difficulty < int(config.get('general.minimum_block_pow')): + difficulty = int(config.get('general.minimum_block_pow')) mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode() puzzle = mainHash[:difficulty] diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py index b93d5724..0288343b 100644 --- a/onionr/onionrproofs.py +++ b/onionr/onionrproofs.py @@ -19,7 +19,7 @@ ''' import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys, base64, json -import core +import core, config class DataPOW: def __init__(self, data, forceDifficulty=0, threadCount = 5): @@ -27,6 +27,7 @@ class DataPOW: self.difficulty = 0 self.data = data self.threadCount = threadCount + config.reload() if forceDifficulty == 0: dataLen = sys.getsizeof(data) @@ -128,7 +129,7 @@ class POW: dataLen = len(data) + len(json.dumps(metadata)) self.difficulty = math.floor(dataLen / 1000000) if self.difficulty <= 2: - self.difficulty = 4 + self.difficulty = int(config.get('general.minimum_block_pow')) try: self.data = self.data.encode() @@ -144,7 +145,7 @@ class POW: for i in range(max(1, threadCount)): t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,myCore)) t.start() - + self.myCore = myCore return def pow(self, reporting = False, myCore = None): diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index de27f4c1..3841b05c 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -2,6 +2,7 @@ "general" : { "dev_mode": true, "display_header" : true, + "minimum_block_pow": 5, "direct_connect" : { "respond" : true, From 8c63d6c205242d732d818931fba95b9921a9b806 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 28 Oct 2018 14:01:57 -0500 Subject: [PATCH 60/70] work on improving block sync --- onionr/communicator2.py | 1 + onionr/onionrcrypto.py | 2 ++ onionr/onionrproofs.py | 23 +++++++++++++++++++++++ onionr/static-data/default_config.json | 3 ++- 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 19b4db31..fe4fb1ef 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -180,6 +180,7 @@ class OnionrCommunicatorDaemon: if not i in existingBlocks: # if block does not exist on disk and is not already in block queue if i not in self.blockQueue and not self._core._blacklist.inBlacklist(i): + # TODO ensure block starts with minimum difficulty before adding to queue self.blockQueue.append(i) # add blocks to download queue self.decrementThreadCount('lookupBlocks') return diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index c717935c..1105d844 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -24,6 +24,8 @@ if sys.version_info[0] == 3 and sys.version_info[1] < 6: from dependencies import secrets elif sys.version_info[0] == 3 and sys.version_info[1] >= 6: import secrets +import config +config.reload() class OnionrCrypto: def __init__(self, coreInstance): diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py index 0288343b..6c5045d0 100644 --- a/onionr/onionrproofs.py +++ b/onionr/onionrproofs.py @@ -20,6 +20,29 @@ import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys, base64, json import core, config +config.reload() + +def getHashDifficulty(h): + ''' + Return the amount of leading zeroes in a hex hash string (h) + ''' + difficulty = 0 + assert type(h) is str + for character in h: + if character == '0': + difficulty += 1 + return difficulty + +def hashMeetsDifficulty(h): + ''' + Return bool for a hash string to see if it meets pow difficulty defined in config + ''' + hashDifficulty = getHashDifficulty(h) + expected = int(config.get('minimum_block_pow')) + if hashDifficulty >= expected: + return True + else: + return False class DataPOW: def __init__(self, data, forceDifficulty=0, threadCount = 5): diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 3841b05c..768bb8ce 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -2,7 +2,8 @@ "general" : { "dev_mode": true, "display_header" : true, - "minimum_block_pow": 5, + "minimum_block_pow": 6, + "minimum_send_pow": 6, "direct_connect" : { "respond" : true, From 58aa8ce1ccd135faea9b8481eb70da7f6236cb22 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 30 Oct 2018 17:22:06 -0500 Subject: [PATCH 61/70] * Increased heartbeat timer * Tried to fix bug where wrong node was being reported as being announced to * Refactored core somewhat - Removed data dir encryption (TODO: just encrypt pub/priv key pair) - Removed simplecrypt dependency --- onionr/communicator2.py | 6 +- onionr/core.py | 62 ++----------------- onionr/dbcreator.py | 14 ++++- onionr/onionr.py | 41 ++++-------- onionr/onionrcrypto.py | 2 +- onionr/onionrdaemontools.py | 1 + onionr/onionrproofs.py | 2 +- .../static-data/default-plugins/cliui/main.py | 2 +- onionr/static-data/default_config.json | 4 +- onionr/tests.py | 32 +--------- requirements.txt | 1 - 11 files changed, 39 insertions(+), 128 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index fe4fb1ef..7f5f81fe 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -83,7 +83,7 @@ class OnionrCommunicatorDaemon: self._chat = onionrchat.OnionrChat(self) if debug or developmentMode: - OnionrCommunicatorTimers(self, self.heartbeat, 10) + OnionrCommunicatorTimers(self, self.heartbeat, 30) # Set timers, function reference, seconds # requiresPeer True means the timer function won't fire if we have no connected peers @@ -500,9 +500,7 @@ class OnionrCommunicatorDaemon: def announce(self, peer): '''Announce to peers our address''' - if self.daemonTools.announceNode(): - logger.info('Successfully introduced node to ' + peer) - else: + if self.daemonTools.announceNode() == False: logger.warn('Could not introduce node.') def detectAPICrash(self): diff --git a/onionr/core.py b/onionr/core.py index 296fec88..d15d30fb 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import sqlite3, os, sys, time, math, base64, tarfile, getpass, simplecrypt, hashlib, nacl, logger, json, netcontroller, math, config +import sqlite3, os, sys, time, math, base64, tarfile, nacl, logger, json, netcontroller, math, config from onionrblockapi import Block import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues @@ -276,14 +276,6 @@ class Core: return data - def _getSha3Hash(self, data): - hasher = hashlib.sha3_256() - if not type(data) is bytes: - data = data.encode() - hasher.update(data) - dataHash = hasher.hexdigest() - return dataHash - def setData(self, data): ''' Set the data assciated with a hash @@ -294,7 +286,7 @@ class Core: if not type(data) is bytes: data = data.encode() - dataHash = self._getSha3Hash(data) + dataHash = self._crypto.sha3Hash(data) if type(dataHash) is bytes: dataHash = dataHash.decode() @@ -319,42 +311,6 @@ class Core: return dataHash - def dataDirEncrypt(self, password): - ''' - Encrypt the data directory on Onionr shutdown - ''' - if os.path.exists('data.tar'): - os.remove('data.tar') - tar = tarfile.open("data.tar", "w") - for name in ['data']: - tar.add(name) - tar.close() - tarData = open('data.tar', 'r', encoding = "ISO-8859-1").read() - encrypted = simplecrypt.encrypt(password, tarData) - open('data-encrypted.dat', 'wb').write(encrypted) - os.remove('data.tar') - - return - - def dataDirDecrypt(self, password): - ''' - Decrypt the data directory on startup - ''' - if not os.path.exists('data-encrypted.dat'): - return (False, 'encrypted archive does not exist') - data = open('data-encrypted.dat', 'rb').read() - try: - decrypted = simplecrypt.decrypt(password, data) - except simplecrypt.DecryptionException: - return (False, 'wrong password (or corrupted archive)') - else: - open('data.tar', 'wb').write(decrypted) - tar = tarfile.open('data.tar') - tar.extractall() - tar.close() - - return (True, '') - def daemonQueue(self): ''' Gives commands to the communication proccess/daemon by reading an sqlite3 database @@ -363,7 +319,7 @@ class Core: ''' retData = False if not os.path.exists(self.queueDB): - self.makeDaemonDB() + self.dbCreate.createDaemonDB() else: conn = sqlite3.connect(self.queueDB, timeout=10) c = conn.cursor() @@ -372,7 +328,7 @@ class Core: retData = row break except sqlite3.OperationalError: - self.makeDaemonDB() + self.dbCreate.createDaemonDB() else: if retData != False: c.execute('DELETE FROM commands WHERE id=?;', (retData[3],)) @@ -383,16 +339,6 @@ class Core: return retData - def makeDaemonDB(self): - '''generate the daemon queue db''' - conn = sqlite3.connect(self.queueDB, timeout=10) - c = conn.cursor() - # Create table - c.execute('''CREATE TABLE commands - (id integer primary key autoincrement, command text, data text, date text)''') - conn.commit() - conn.close() - def daemonQueueAdd(self, command, data=''): ''' Add a command to the daemon queue, used by the communication daemon (communicator.py) diff --git a/onionr/dbcreator.py b/onionr/dbcreator.py index 7f970310..82816f43 100644 --- a/onionr/dbcreator.py +++ b/onionr/dbcreator.py @@ -132,4 +132,16 @@ class DBCreator: ''') conn.commit() conn.close() - return \ No newline at end of file + return + + def createDaemonDB(self): + ''' + Create the daemon queue database + ''' + conn = sqlite3.connect(self.core.queueDB, timeout=10) + c = conn.cursor() + # Create table + c.execute('''CREATE TABLE commands + (id integer primary key autoincrement, command text, data text, date text)''') + conn.commit() + conn.close() \ No newline at end of file diff --git a/onionr/onionr.py b/onionr/onionr.py index eb56fb7c..24880258 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -103,31 +103,21 @@ class Onionr: self.debug = False # Whole application debugging - if os.path.exists('data-encrypted.dat'): - while True: - print('Enter password to decrypt:') - password = getpass.getpass() - result = self.onionrCore.dataDirDecrypt(password) - if os.path.exists(self.dataDir): - break - else: - logger.error('Failed to decrypt: ' + result[1], timestamp = False) - else: - # If data folder does not exist - if not data_exists: - if not os.path.exists(self.dataDir + 'blocks/'): - os.mkdir(self.dataDir + 'blocks/') + # If data folder does not exist + if not data_exists: + if not os.path.exists(self.dataDir + 'blocks/'): + os.mkdir(self.dataDir + 'blocks/') - # Copy default plugins into plugins folder - if not os.path.exists(plugins.get_plugins_folder()): - if os.path.exists('static-data/default-plugins/'): - names = [f for f in os.listdir("static-data/default-plugins/") if not os.path.isfile(f)] - shutil.copytree('static-data/default-plugins/', plugins.get_plugins_folder()) + # Copy default plugins into plugins folder + if not os.path.exists(plugins.get_plugins_folder()): + if os.path.exists('static-data/default-plugins/'): + names = [f for f in os.listdir("static-data/default-plugins/") if not os.path.isfile(f)] + shutil.copytree('static-data/default-plugins/', plugins.get_plugins_folder()) - # Enable plugins - for name in names: - if not name in plugins.get_enabled_plugins(): - plugins.enable(name, self) + # Enable plugins + for name in names: + if not name in plugins.get_enabled_plugins(): + plugins.enable(name, self) for name in plugins.get_enabled_plugins(): if not os.path.exists(plugins.get_plugin_data_folder(name)): @@ -265,11 +255,6 @@ class Onionr: finally: self.execute(command) - if not self._developmentMode: - encryptionPassword = self.onionrUtils.getPassword('Enter password to encrypt directory: ') - self.onionrCore.dataDirEncrypt(encryptionPassword) - shutil.rmtree(self.dataDir) - return ''' diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index 1105d844..8bc5c847 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -25,10 +25,10 @@ if sys.version_info[0] == 3 and sys.version_info[1] < 6: elif sys.version_info[0] == 3 and sys.version_info[1] >= 6: import secrets import config -config.reload() class OnionrCrypto: def __init__(self, coreInstance): + config.reload() self._core = coreInstance self._keyFile = self._core.dataDir + 'keys.txt' self.keyPowFile = self._core.dataDir + 'keyPow.txt' diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index 8192e0de..71a89410 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -52,6 +52,7 @@ class DaemonTools: logger.info('Announcing node to ' + url) if self.daemon._core._utils.doPostRequest(url, data) == 'Success': + logger.info('Successfully introduced node to ' + peer) retData = True self.daemon.decrementThreadCount('announceNode') return retData diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py index 6c5045d0..40ae195a 100644 --- a/onionr/onionrproofs.py +++ b/onionr/onionrproofs.py @@ -20,7 +20,6 @@ import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys, base64, json import core, config -config.reload() def getHashDifficulty(h): ''' @@ -37,6 +36,7 @@ def hashMeetsDifficulty(h): ''' Return bool for a hash string to see if it meets pow difficulty defined in config ''' + config.reload() hashDifficulty = getHashDifficulty(h) expected = int(config.get('minimum_block_pow')) if hashDifficulty >= expected: diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py index 5fc24385..646aa7c2 100644 --- a/onionr/static-data/default-plugins/cliui/main.py +++ b/onionr/static-data/default-plugins/cliui/main.py @@ -53,7 +53,7 @@ class OnionrCLIUI: while showMenu: if firstRun: - print("please wait while Onionr starts...") + logger.info("please wait while Onionr starts...") daemon = subprocess.Popen(["./onionr.py", "start"], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL) time.sleep(30) firstRun = False diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 768bb8ce..8ce16361 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -2,8 +2,8 @@ "general" : { "dev_mode": true, "display_header" : true, - "minimum_block_pow": 6, - "minimum_send_pow": 6, + "minimum_block_pow": 5, + "minimum_send_pow": 5, "direct_connect" : { "respond" : true, diff --git a/onionr/tests.py b/onionr/tests.py index 008d2bed..c23db1fa 100755 --- a/onionr/tests.py +++ b/onionr/tests.py @@ -14,7 +14,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import unittest, sys, os, base64, tarfile, shutil, simplecrypt, logger +import unittest, sys, os, base64, tarfile, shutil, logger class OnionrTests(unittest.TestCase): def testPython3(self): @@ -61,36 +61,6 @@ class OnionrTests(unittest.TestCase): else: self.assertTrue(False) - def testData_b_Encrypt(self): - self.assertTrue(True) - return - - logger.debug('-'*26 + '\n') - logger.info('Running data dir encrypt test...') - - import core - myCore = core.Core() - myCore.dataDirEncrypt('password') - if os.path.exists('data-encrypted.dat'): - self.assertTrue(True) - else: - self.assertTrue(False) - - def testData_a_Decrypt(self): - self.assertTrue(True) - return - - logger.debug('-'*26 + '\n') - logger.info('Running data dir decrypt test...') - - import core - myCore = core.Core() - myCore.dataDirDecrypt('password') - if os.path.exists('data/'): - self.assertTrue(True) - else: - self.assertTrue(False) - def testConfig(self): logger.debug('-'*26 + '\n') logger.info('Running simple configuration test...') diff --git a/requirements.txt b/requirements.txt index 05797aed..545d7cf9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ PyNaCl==1.2.1 gevent==1.3.6 sha3==0.2.1 defusedxml==0.5.0 -simple_crypt==4.1.7 Flask==1.0.2 PySocks==1.6.8 stem==1.6.0 From b1752132cb5a89fa594372abf7a271b469ad91d5 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Wed, 31 Oct 2018 23:56:59 -0500 Subject: [PATCH 62/70] work on sentbox --- .../static-data/default-plugins/pms/main.py | 30 ++++++++++++++++++- requirements.txt | 2 +- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index a47983b0..28b7859c 100644 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -133,6 +133,34 @@ class OnionrMail: print(draw_border(self.myCore._utils.escapeAnsi(readBlock.bcontent.decode().strip()))) return + def sentbox(self): + ''' + Display sent mail messages + ''' + entering = True + while entering: + print('''1. List Sent Messages +2. Read sent message +3. Delete Sent Message +4. Main Menu +''') + try: + choice = logger.readline('>').lower() + except (KeyboardInterrupt, EOFError): + entering = False + else: + if choice in ('1', 'list'): + print(getSentList()) + elif choice in ('2', 'read'): + pass + else: + print('Not implemented') + + return + + def getSentList(self): + return "" + def draftMessage(self): message = '' newLine = '' @@ -185,7 +213,7 @@ class OnionrMail: if choice in (self.strings.mainMenuChoices[0], '1'): self.inbox() elif choice in (self.strings.mainMenuChoices[1], '2'): - logger.warn('not implemented yet') + self.sentbox() elif choice in (self.strings.mainMenuChoices[2], '3'): self.draftMessage() elif choice in (self.strings.mainMenuChoices[3], '4'): diff --git a/requirements.txt b/requirements.txt index 545d7cf9..754a0da9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ urllib3==1.23 -requests==2.18.4 +requests==2.20.0 PyNaCl==1.2.1 gevent==1.3.6 sha3==0.2.1 From f8867fb08e915a416bafc1874b30a92a535aa936 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 1 Nov 2018 14:32:50 -0500 Subject: [PATCH 63/70] work on sentbox --- .../default-plugins/pms/sentboxdb.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 onionr/static-data/default-plugins/pms/sentboxdb.py diff --git a/onionr/static-data/default-plugins/pms/sentboxdb.py b/onionr/static-data/default-plugins/pms/sentboxdb.py new file mode 100644 index 00000000..6a79f24d --- /dev/null +++ b/onionr/static-data/default-plugins/pms/sentboxdb.py @@ -0,0 +1,53 @@ +''' + Onionr - P2P Microblogging Platform & Social network + + This file handles the sentbox for the mail plugin +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import sqlite3, os +import core +class SentBox: + def __init__(self, core): + assert isinstance(core, core.Core) + self.dbLocation = core.dataDir + 'sentbox.db' + if not os.path.exists(self.dbLocation): + self.createDB() + self.conn = sqlite3.connect(self.dbLocation) + self.cursor = self.conn.cursor() + return + + def createDB(self): + self.cursor.execute('''CREATE TABLE sent( + hash id not null, + peer text not null, + message text not null, + date int not null + ); + ''') + self.conn.commit() + return + + def listSent(self): + retData = [] + for entry in self.cursor.execute('SELECT * FROM sent;'): + retData.append({'hash': entry[0], 'peer': entry[1], 'message': entry[2], 'date': entry[3]}) + return retData + + def addToSent(self, blockID): + return + + def removeSent(self, blockID): + return \ No newline at end of file From f270d3c5220d902b284a0f5ab4f3dd4bd11d7513 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 2 Nov 2018 22:24:14 -0500 Subject: [PATCH 64/70] work on sentbox --- onionr/onionrproofs.py | 2 - .../static-data/default-plugins/pms/main.py | 49 ++++++++++++------- .../default-plugins/pms/sentboxdb.py | 21 +++++--- 3 files changed, 47 insertions(+), 25 deletions(-) diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py index 40ae195a..9085943f 100644 --- a/onionr/onionrproofs.py +++ b/onionr/onionrproofs.py @@ -101,7 +101,6 @@ class DataPOW: endTime = math.floor(time.time()) if self.reporting: logger.debug('Found token after %s seconds: %s' % (endTime - startTime, token), timestamp=True) - logger.debug('Random value was: %s' % base64.b64encode(rand).decode()) self.result = (token, rand) def shutdown(self): @@ -201,7 +200,6 @@ class POW: endTime = math.floor(time.time()) if self.reporting: logger.debug('Found token after %s seconds: %s' % (endTime - startTime, token), timestamp=True) - logger.debug('Random value was: %s' % base64.b64encode(rand).decode()) def shutdown(self): self.hashing = False diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index 28b7859c..b6d29545 100644 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -22,9 +22,13 @@ import logger, config, threading, time, readline, datetime from onionrblockapi import Block import onionrexceptions, onionrusers -import locale +import locale, sys, os + locale.setlocale(locale.LC_ALL, '') +sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) +import sentboxdb # import after path insert + plugin_name = 'pms' PLUGIN_VERSION = '0.0.1' @@ -60,6 +64,9 @@ class OnionrMail: #self.dataFolder = pluginapi.get_data_folder() self.strings = MailStrings(self) + self.sentboxTools = sentboxdb.SentBox(self.myCore) + self.sentboxList = [] + self.sentMessages = {} return def inbox(self): @@ -139,27 +146,35 @@ class OnionrMail: ''' entering = True while entering: - print('''1. List Sent Messages -2. Read sent message -3. Delete Sent Message -4. Main Menu -''') + self.getSentList() + print('Enter block number or -q to return') try: - choice = logger.readline('>').lower() - except (KeyboardInterrupt, EOFError): + choice = input('>') + except (EOFError, KeyboardInterrupt) as e: entering = False else: - if choice in ('1', 'list'): - print(getSentList()) - elif choice in ('2', 'read'): - pass + if choice == '-q': + entering = False else: - print('Not implemented') - + try: + self.sentboxList[int(choice) - 1] + except IndexError: + print('Invalid block') + else: + logger.info('Sent to: ' + self.sentMessages[self.sentboxList[int(choice) - 1]][1]) + # Print ansi escaped sent message + print(self.myCore._utils.escapeAnsi(self.sentMessages[self.sentboxList[int(choice) - 1]][0])) + input('Press enter to continue...') + return def getSentList(self): - return "" + count = 1 + for i in self.sentboxTools.listSent(): + self.sentboxList.append(i['hash']) + self.sentMessages[i['hash']] = (i['message'], i['peer']) + print('%s. %s - %s - %s' % (count, i['hash'], i['peer'][:12], i['date'])) + count += 1 def draftMessage(self): message = '' @@ -197,8 +212,8 @@ class OnionrMail: print('Inserting encrypted message as Onionr block....') - self.myCore.insertBlock(message, header='pm', encryptType='asym', asymPeer=recip, sign=True) - + blockID = self.myCore.insertBlock(message, header='pm', encryptType='asym', asymPeer=recip, sign=True) + self.sentboxTools.addToSent(blockID, recip, message) def menu(self): choice = '' while True: diff --git a/onionr/static-data/default-plugins/pms/sentboxdb.py b/onionr/static-data/default-plugins/pms/sentboxdb.py index 6a79f24d..f2328ccb 100644 --- a/onionr/static-data/default-plugins/pms/sentboxdb.py +++ b/onionr/static-data/default-plugins/pms/sentboxdb.py @@ -20,24 +20,27 @@ import sqlite3, os import core class SentBox: - def __init__(self, core): - assert isinstance(core, core.Core) - self.dbLocation = core.dataDir + 'sentbox.db' + def __init__(self, mycore): + assert isinstance(mycore, core.Core) + self.dbLocation = mycore.dataDir + 'sentbox.db' if not os.path.exists(self.dbLocation): self.createDB() self.conn = sqlite3.connect(self.dbLocation) self.cursor = self.conn.cursor() + self.core = mycore return def createDB(self): - self.cursor.execute('''CREATE TABLE sent( + conn = sqlite3.connect(self.dbLocation) + cursor = conn.cursor() + cursor.execute('''CREATE TABLE sent( hash id not null, peer text not null, message text not null, date int not null ); ''') - self.conn.commit() + conn.commit() return def listSent(self): @@ -46,8 +49,14 @@ class SentBox: retData.append({'hash': entry[0], 'peer': entry[1], 'message': entry[2], 'date': entry[3]}) return retData - def addToSent(self, blockID): + def addToSent(self, blockID, peer, message): + args = (blockID, peer, message, self.core._utils.getEpoch()) + self.cursor.execute('INSERT INTO sent VALUES(?, ?, ?, ?)', args) + self.conn.commit() return def removeSent(self, blockID): + args = (blockID,) + self.cursor.execute('DELETE FROM sent where hash=?', args) + self.conn.commit() return \ No newline at end of file From a31a0fd264a07e8dab04ddee1f9910307041664a Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 3 Nov 2018 00:06:04 -0500 Subject: [PATCH 65/70] removed pubkey pow and bug fixes --- onionr/onionr.py | 3 --- onionr/onionrcrypto.py | 23 +------------------ .../static-data/default-plugins/pms/main.py | 2 +- 3 files changed, 2 insertions(+), 26 deletions(-) diff --git a/onionr/onionr.py b/onionr/onionr.py index 24880258..97781f0c 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -683,13 +683,10 @@ class Onionr: # define stats messages here totalBlocks = len(Block.getBlocks()) signedBlocks = len(Block.getBlocks(signed = True)) - powToken = self.onionrCore._crypto.pubKeyPowToken messages = { # info about local client 'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if self.onionrUtils.isCommunicatorRunning(timeout = 9) else logger.colors.fg.red + 'Offline'), 'Public Key' : self.onionrCore._crypto.pubKey, - 'POW Token' : powToken, - 'Combined' : self.onionrCore._crypto.pubKey + '-' + powToken, 'Human readable public key' : self.onionrCore._utils.getHumanReadableID(), 'Node Address' : self.get_hostname(), diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index 8bc5c847..045bf053 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -31,15 +31,11 @@ class OnionrCrypto: config.reload() self._core = coreInstance self._keyFile = self._core.dataDir + 'keys.txt' - self.keyPowFile = self._core.dataDir + 'keyPow.txt' self.pubKey = None self.privKey = None self.secrets = secrets - - self.pubKeyPowToken = None - #self.pubKeyPowHash = None - + self.HASH_ID_ROUNDS = 2000 # Load our own pub/priv Ed25519 keys, gen & save them if they don't exist @@ -48,29 +44,12 @@ class OnionrCrypto: keys = keys.read().split(',') self.pubKey = keys[0] self.privKey = keys[1] - try: - with open(self.keyPowFile, 'r') as powFile: - data = powFile.read() - self.pubKeyPowToken = data - except (FileNotFoundError, IndexError): - pass else: keys = self.generatePubKey() self.pubKey = keys[0] self.privKey = keys[1] with open(self._keyFile, 'w') as keyfile: keyfile.write(self.pubKey + ',' + self.privKey) - with open(self.keyPowFile, 'w') as keyPowFile: - proof = onionrproofs.DataPOW(self.pubKey) - logger.info('Doing necessary work to insert our public key') - while True: - time.sleep(0.2) - powToken = proof.getResult() - if powToken != False: - break - keyPowFile.write(base64.b64encode(powToken[1]).decode()) - self.pubKeyPowToken = powToken[1] - self.pubKeyPowHash = powToken[0] return def edVerify(self, data, key, sig, encodedData=True): diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index b6d29545..ea2ff988 100644 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -131,7 +131,7 @@ class OnionrMail: else: cancel = '' readBlock.verifySig() - print('Message recieved from %s' % (readBlock.signer,)) + print('Message recieved from %s' % (self.myCore._utils.bytesToStr(readBlock.signer,))) print('Valid signature:', readBlock.validSig) if not readBlock.validSig: logger.warn('This message has an INVALID signature. ANYONE could have sent this message.') From 3764c1a115618f94947a6383e53eab55b5539021 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 3 Nov 2018 22:26:18 -0500 Subject: [PATCH 66/70] fixed bug where process would not die --- onionr/onionr.py | 12 +++++++++--- onionr/static-data/bootstrap-nodes.txt | 2 -- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/onionr/onionr.py b/onionr/onionr.py index 97781f0c..eceb7639 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -640,7 +640,7 @@ class Onionr: logger.info('Our Public key: ' + self.onionrCore._crypto.pubKey) time.sleep(1) #TODO make runable on windows - subprocess.Popen([communicatorDaemon, "run", str(net.socksPort)]) + communicatorProc = subprocess.Popen([communicatorDaemon, "run", str(net.socksPort)]) # Print nice header thing :) if config.get('general.display_header', True): self.header() @@ -649,6 +649,9 @@ class Onionr: try: while True: time.sleep(5) + # Break if communicator process ends, so we don't have left over processes + if communicatorProc.poll() is not None: + break except KeyboardInterrupt: self.onionrCore.daemonQueueAdd('shutdown') self.onionrUtils.localCommand('shutdown') @@ -789,11 +792,14 @@ class Onionr: ''' Get a file from onionr blocks ''' - if len(sys.argv) >= 3: + try: fileName = sys.argv[2] + bHash = sys.argv[3] + except IndexError: + logger.error("Syntax %s %s" % (sys.argv[0], '/path/to/filename ')) + else: print(fileName) contents = None - bHash = sys.argv[3] if os.path.exists(fileName): logger.error("File already exists") return diff --git a/onionr/static-data/bootstrap-nodes.txt b/onionr/static-data/bootstrap-nodes.txt index 5473550f..e69de29b 100644 --- a/onionr/static-data/bootstrap-nodes.txt +++ b/onionr/static-data/bootstrap-nodes.txt @@ -1,2 +0,0 @@ -onionragxuddecmg.onion -dgyllprmtmym4gbk.onion From 293b36e3ade5edfab5afe99ba1d6ed843074d7e8 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 4 Nov 2018 10:06:24 -0600 Subject: [PATCH 67/70] mail plugin usable now --- onionr/api.py | 2 +- onionr/onionrcrypto.py | 2 +- onionr/onionri2p.py | 19 ------------------- onionr/onionrproofs.py | 2 +- .../static-data/default-plugins/pms/main.py | 9 +++------ 5 files changed, 6 insertions(+), 28 deletions(-) delete mode 100644 onionr/onionri2p.py diff --git a/onionr/api.py b/onionr/api.py index 94c7eeaa..a8a3b29e 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -1,5 +1,5 @@ ''' - Onionr - P2P Microblogging Platform & Social network + Onionr - P2P Anonymous Storage Network This file handles all incoming http requests to the client, using Flask ''' diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index 045bf053..fe13ae01 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -1,5 +1,5 @@ ''' - Onionr - P2P Microblogging Platform & Social network + Onionr - P2P Anonymous Storage Network This file handles Onionr's cryptography. ''' diff --git a/onionr/onionri2p.py b/onionr/onionri2p.py deleted file mode 100644 index baeb2977..00000000 --- a/onionr/onionri2p.py +++ /dev/null @@ -1,19 +0,0 @@ -''' - Onionr - P2P Microblogging Platform & Social network - - Funcitons for talking to I2P -''' -''' - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -''' diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py index 9085943f..30ca4b6c 100644 --- a/onionr/onionrproofs.py +++ b/onionr/onionrproofs.py @@ -1,5 +1,5 @@ ''' - Onionr - P2P Microblogging Platform & Social network + Onionr - P2P Anonymous Storage Network Proof of work module ''' diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index ea2ff988..981979ee 100644 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -1,5 +1,5 @@ ''' - Onionr - P2P Microblogging Platform & Social network + Onionr - P2P Anonymous Storage Network This default plugin handles private messages in an email like fashion ''' @@ -48,15 +48,14 @@ class MailStrings: self.mailInstance = mailInstance self.programTag = 'OnionrMail v%s' % (PLUGIN_VERSION) - choices = ['view inbox', 'view sentbox', 'send message', 'help', 'quit'] + choices = ['view inbox', 'view sentbox', 'send message', 'quit'] self.mainMenuChoices = choices self.mainMenu = '''\n ----------------- 1. %s 2. %s 3. %s -4. %s -5. %s''' % (choices[0], choices[1], choices[2], choices[3], choices[4]) +4. %s''' % (choices[0], choices[1], choices[2], choices[3]) class OnionrMail: def __init__(self, pluginapi): @@ -232,8 +231,6 @@ class OnionrMail: elif choice in (self.strings.mainMenuChoices[2], '3'): self.draftMessage() elif choice in (self.strings.mainMenuChoices[3], '4'): - logger.warn('not implemented yet') - elif choice in (self.strings.mainMenuChoices[4], '5'): logger.info('Goodbye.') break elif choice == '': From 6f72e8c06c3bed44d90b2eea1c70c4c1f6ac66cc Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 4 Nov 2018 17:01:58 -0600 Subject: [PATCH 68/70] better address validation and removed dependency --- onionr/onionrblacklist.py | 11 ++++++----- onionr/onionrutils.py | 6 ++++++ onionr/static-data/default-plugins/pms/main.py | 1 - requirements.txt | 1 - 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/onionr/onionrblacklist.py b/onionr/onionrblacklist.py index f87ccd65..1d7e83f7 100644 --- a/onionr/onionrblacklist.py +++ b/onionr/onionrblacklist.py @@ -32,7 +32,8 @@ class OnionrBlackList: retData = False if not hashed.isalnum(): raise Exception("Hashed data is not alpha numeric") - + if len(hashed) > 64: + raise Exception("Hashed data is too large") for i in self._dbExecute("select * from blacklist where hash='%s'" % (hashed,)): retData = True # this only executes if an entry is present by that hash break @@ -95,9 +96,8 @@ class OnionrBlackList: ''' # we hash the data so we can remove data entirely from our node's disk hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data)) - - if self.inBlacklist(hashed): - return + if len(hashed) > 64: + raise Exception("Hashed data is too large") if not hashed.isalnum(): raise Exception("Hashed data is not alpha numeric") @@ -109,7 +109,8 @@ class OnionrBlackList: int(expire) except ValueError: raise Exception("expire is not int") - #TODO check for length sanity + if self.inBlacklist(hashed): + return insert = (hashed,) blacklistDate = self._core._utils.getEpoch() self._dbExecute("insert into blacklist (hash, dataType, blacklistDate, expire) VALUES('%s', %s, %s, %s);" % (hashed, dataType, blacklistDate, expire)) diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index f102a3a1..3bab70b0 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -483,6 +483,12 @@ class OnionrUtils: retVal = False if not idNoDomain.isalnum(): retVal = False + + # Validate address is valid base32 (when capitalized and minus extension); v2/v3 onions and .b32.i2p use base32 + try: + base64.b32decode(idNoDomain.upper().encode()) + except binascii.Error: + retVal = False return retVal except: diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index 981979ee..6e7dda24 100644 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -60,7 +60,6 @@ class MailStrings: class OnionrMail: def __init__(self, pluginapi): self.myCore = pluginapi.get_core() - #self.dataFolder = pluginapi.get_data_folder() self.strings = MailStrings(self) self.sentboxTools = sentboxdb.SentBox(self.myCore) diff --git a/requirements.txt b/requirements.txt index 754a0da9..2375324d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,3 @@ defusedxml==0.5.0 Flask==1.0.2 PySocks==1.6.8 stem==1.6.0 -ntfy==2.6.0 From c0707a10f9007f6588f6a88ee2fe3758774e59f5 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 8 Nov 2018 23:22:43 -0600 Subject: [PATCH 69/70] fixed offline error --- onionr/api.py | 3 +++ onionr/communicator2.py | 5 ++++- onionr/onionrdaemontools.py | 2 +- onionr/static-data/default-plugins/pms/main.py | 1 + 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index a8a3b29e..3efc11b8 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -37,6 +37,9 @@ class API: ''' Validate that the client token matches the given token ''' + if len(self.clientToken) == 0: + logger.error("client password needs to be set") + return False try: if not hmac.compare_digest(self.clientToken, token): return False diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 7f5f81fe..f29e482e 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -147,10 +147,13 @@ class OnionrCommunicatorDaemon: newBlocks = '' existingBlocks = self._core.getBlockList() triedPeers = [] # list of peers we've tried this time around + maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion for i in range(tryAmount): - # check if disk allocation is used + if len(self.blockQueue) >= maxBacklog: + break if not self.isOnline: break + # check if disk allocation is used if self._core._utils.storageCounter.isFull(): logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used') break diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index 71a89410..486f4638 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -59,7 +59,7 @@ class DaemonTools: def netCheck(self): '''Check if we are connected to the internet or not when we can't connect to any peers''' - if len(self.daemon.onlinePeers) != 0: + if len(self.daemon.onlinePeers) == 0: if not self.daemon._core._utils.checkNetwork(torPort=self.daemon.proxyPort): logger.warn('Network check failed, are you connected to the internet?') self.daemon.isOnline = False diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index 6e7dda24..d05ff36f 100644 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -136,6 +136,7 @@ class OnionrMail: cancel = logger.readline('Press enter to continue to message, or -q to not open the message (recommended).') if cancel != '-q': print(draw_border(self.myCore._utils.escapeAnsi(readBlock.bcontent.decode().strip()))) + input("Press enter to continue") return def sentbox(self): From c0c5061f1e6c5f03a5cf717e0e42997966f74925 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 9 Nov 2018 13:07:26 -0600 Subject: [PATCH 70/70] fixed forward secrecy and delete keys --- onionr/communicator2.py | 7 ++++++- onionr/core.py | 2 +- onionr/dbcreator.py | 4 ++-- onionr/onionrdaemontools.py | 22 +++++++++++++++++++++- onionr/onionrusers.py | 36 +++++++++++++++++++++++++----------- onionr/onionrutils.py | 5 +++++ 6 files changed, 60 insertions(+), 16 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index f29e482e..08806569 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -99,11 +99,13 @@ class OnionrCommunicatorDaemon: netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 305, requiresPeer=True, maxThreads=1) cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requiresPeer=True) + forwardSecrecyTimer = OnionrCommunicatorTimers(self, self.daemonTools.cleanKeys, 15) # set loop to execute instantly to load up peer pool (replaced old pool init wait) peerPoolTimer.count = (peerPoolTimer.frequency - 1) cleanupTimer.count = (cleanupTimer.frequency - 60) announceTimer.count = (cleanupTimer.frequency - 60) + #forwardSecrecyTimer.count = (forwardSecrecyTimer.frequency - 990) self.socketServer = threading.Thread(target=onionrsockets.OnionrSocketServer, args=(self._core,)) self.socketServer.start() @@ -450,7 +452,10 @@ class OnionrCommunicatorDaemon: if cmd[0] == 'shutdown': self.shutdown = True elif cmd[0] == 'announceNode': - self.announce(cmd[1]) + if len(self.onlinePeers) > 0: + self.announce(cmd[1]) + else: + logger.warn("Not introducing, since I have no connected nodes.") elif cmd[0] == 'runCheck': logger.debug('Status check; looks good.') open(self._core.dataDir + '.runcheck', 'w+').close() diff --git a/onionr/core.py b/onionr/core.py index d15d30fb..fab7b04a 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -670,7 +670,7 @@ class Core: except AttributeError: pass - if encryptType == 'sym': + if encryptType == 'asym': try: forwardEncrypted = onionrusers.OnionrUser(self, asymPeer).forwardEncrypt(data) data = forwardEncrypted[0] diff --git a/onionr/dbcreator.py b/onionr/dbcreator.py index 82816f43..d11aa4fa 100644 --- a/onionr/dbcreator.py +++ b/onionr/dbcreator.py @@ -124,8 +124,8 @@ class DBCreator: c = conn.cursor() c.execute('''CREATE TABLE myForwardKeys( peer text not null, - public key text not null, - private key text not null, + publickey text not null, + privatekey text not null, date int not null, expire int not null ); diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index 486f4638..cac06f9f 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import onionrexceptions, onionrpeers, onionrproofs, base64, logger +import onionrexceptions, onionrpeers, onionrproofs, base64, logger, onionrusers, sqlite3 from dependencies import secrets class DaemonTools: def __init__(self, daemon): @@ -78,6 +78,26 @@ class DaemonTools: self.daemon._core._blacklist.addToDB(bHash) self.daemon._core.removeBlock(bHash) self.daemon.decrementThreadCount('cleanOldBlocks') + + def cleanKeys(self): + '''Delete expired forward secrecy keys''' + conn = sqlite3.connect(self.daemon._core.peerDB, timeout=10) + c = conn.cursor() + time = self.daemon._core._utils.getEpoch() + deleteKeys = [] + for entry in c.execute("SELECT * FROM forwardKeys where expire <= ?", (time,)): + logger.info(entry[1]) + deleteKeys.append(entry[1]) + + for key in deleteKeys: + logger.info('Deleting forward key '+ key) + c.execute("DELETE from forwardKeys where forwardKey = ?", (key,)) + conn.commit() + conn.close() + + onionrusers.deleteExpiredKeys(self.daemon._core) + + self.daemon.decrementThreadCount('cleanKeys') def cooldownPeer(self): '''Randomly add an online peer to cooldown, so we can connect a new one''' diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index f1ab241c..93ecd6b0 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -19,6 +19,19 @@ ''' import onionrblockapi, logger, onionrexceptions, json, sqlite3 import nacl.exceptions + +def deleteExpiredKeys(coreInst): + # Fetch the keys we generated for the peer, that are still around + conn = sqlite3.connect(coreInst.forwardKeysFile, timeout=10) + c = conn.cursor() + + curTime = coreInst._utils.getEpoch() + c.execute("DELETE from myForwardKeys where expire <= ?", (curTime,)) + conn.commit() + conn.execute("VACUUM") + conn.close() + return + class OnionrUser: def __init__(self, coreInst, publicKey): self.trust = 0 @@ -58,7 +71,7 @@ class OnionrUser: def forwardEncrypt(self, data): retData = '' forwardKey = self._getLatestForwardKey() - logger.info('using ' + forwardKey) + #logger.info('using ' + forwardKey) if self._core._utils.validatePubKey(forwardKey): retData = self._core._crypto.pubKeyEncrypt(data, forwardKey, encodedData=True, anonymous=True) else: @@ -68,9 +81,9 @@ class OnionrUser: def forwardDecrypt(self, encrypted): retData = "" - logger.error(self.publicKey) - logger.error(self.getGeneratedForwardKeys()) - for key in self.getGeneratedForwardKeys(): + #logger.error(self.publicKey) + #logger.error(self.getGeneratedForwardKeys(False)) + for key in self.getGeneratedForwardKeys(False): logger.info(encrypted) try: retData = self._core._crypto.pubKeyDecrypt(encrypted, privkey=key[1], anonymous=True, encodedData=True) @@ -109,7 +122,7 @@ class OnionrUser: return list(keyList) - def generateForwardKey(self, expire=432000): + def generateForwardKey(self, expire=604800): # Generate a forward secrecy key for the peer conn = sqlite3.connect(self._core.forwardKeysFile, timeout=10) @@ -121,7 +134,7 @@ class OnionrUser: newPriv = self._core._utils.bytesToStr(newKeys[1]) time = self._core._utils.getEpoch() - command = (self.publicKey, newPub, newPriv, time, expire) + command = (self.publicKey, newPub, newPriv, time, expire + time) c.execute("INSERT INTO myForwardKeys VALUES(?, ?, ?, ?, ?);", command) @@ -129,7 +142,7 @@ class OnionrUser: conn.close() return newPub - def getGeneratedForwardKeys(self): + def getGeneratedForwardKeys(self, genNew=True): # Fetch the keys we generated for the peer, that are still around conn = sqlite3.connect(self._core.forwardKeysFile, timeout=10) c = conn.cursor() @@ -140,11 +153,12 @@ class OnionrUser: for result in c.execute("SELECT * FROM myForwardKeys where peer=?", command): keyList.append((result[1], result[2])) if len(keyList) == 0: - self.generateForwardKey() - keyList = self.getGeneratedForwardKeys() + if genNew: + self.generateForwardKey() + keyList = self.getGeneratedForwardKeys() return list(keyList) - def addForwardKey(self, newKey, expire=432000): + def addForwardKey(self, newKey, expire=604800): if not self._core._utils.validatePubKey(newKey): raise onionrexceptions.InvalidPubkey # Add a forward secrecy key for the peer @@ -152,7 +166,7 @@ class OnionrUser: c = conn.cursor() # Prepare the insert time = self._core._utils.getEpoch() - command = (self.publicKey, newKey, time, expire) + command = (self.publicKey, newKey, time, time + expire) c.execute("INSERT INTO forwardKeys VALUES(?, ?, ?, ?);", command) diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 3bab70b0..558a6b20 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -263,14 +263,19 @@ class OnionrUtils: ''' myBlock = Block(blockHash, self._core) if myBlock.isEncrypted: + #pass logger.warn(myBlock.decrypt()) if (myBlock.isEncrypted and myBlock.decrypted) or (not myBlock.isEncrypted): blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks signer = self.bytesToStr(myBlock.signer) valid = myBlock.verifySig() + logger.info('Checking for fs key') if myBlock.getMetadata('newFSKey') is not None: onionrusers.OnionrUser(self._core, signer).addForwardKey(myBlock.getMetadata('newFSKey')) + else: + logger.warn('FS not used for this encrypted block') + logger.info(myBlock.bmetadata) try: if len(blockType) <= 10: