From a87d1bf1e8466c71d4270df347862a721904cb60 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 18 Mar 2019 00:22:31 -0500 Subject: [PATCH 01/19] bumped stem version and started work on new dir conn system --- onionr/netcontroller.py | 14 +---- onionr/onionrservices/__init__.py | 18 ++++++ onionr/onionrservices/bootstrapservice.py | 67 +++++++++++++++++++++++ onionr/onionrservices/connectionserver.py | 4 ++ onionr/onionrutils.py | 6 +- requirements.in | 2 +- requirements.txt | 4 +- 7 files changed, 99 insertions(+), 16 deletions(-) create mode 100644 onionr/onionrservices/__init__.py create mode 100644 onionr/onionrservices/bootstrapservice.py create mode 100644 onionr/onionrservices/connectionserver.py diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py index 2898256c..fcba14c2 100755 --- a/onionr/netcontroller.py +++ b/onionr/netcontroller.py @@ -17,12 +17,11 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' - -import subprocess, os, random, sys, logger, time, signal, config, base64, socket -from stem.control import Controller +import subprocess, os, random, sys, time, signal, base64, socket +from shutil import which +import logger, config from onionrblockapi import Block from dependencies import secrets -from shutil import which def getOpenPort(): # taken from (but modified) https://stackoverflow.com/a/2838309 @@ -69,13 +68,6 @@ class NetController: self.torBinary = 'tor' config.reload() - ''' - if os.path.exists(self.torConfigLocation): - torrc = open(self.torConfigLocation, 'r') - if not str(self.hsPort) in torrc.read(): - os.remove(self.torConfigLocation) - torrc.close() - ''' return diff --git a/onionr/onionrservices/__init__.py b/onionr/onionrservices/__init__.py new file mode 100644 index 00000000..68612cf7 --- /dev/null +++ b/onionr/onionrservices/__init__.py @@ -0,0 +1,18 @@ +import stem +import core +from . import connectionserver, connectionclient, bootstrapservice +class OnionrServices: + def __init__(self, onionr_core): + assert isinstance(onionr_core, core.Core) + self._core = onionr_core + self.servers = {} + self.clients = {} + return + + def create_server(self): + return + + def create_client(self, peer): + # Create ephemeral onion service to bootstrap connection + address = bootstrapservice.bootstrap_client_service(peer) + return address \ No newline at end of file diff --git a/onionr/onionrservices/bootstrapservice.py b/onionr/onionrservices/bootstrapservice.py new file mode 100644 index 00000000..7a2c3823 --- /dev/null +++ b/onionr/onionrservices/bootstrapservice.py @@ -0,0 +1,67 @@ +''' + Onionr - P2P Anonymous Storage Network + + Bootstrap onion direct connections for the clients +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +from gevent.pywsgi import WSGIServer, WSGIHandler +from stem.control import Controller +from flask import Flask +import core +from netcontroller import getOpenPort + +def bootstrap_client_service(peer, core_inst=None): + ''' + Bootstrap client services + ''' + if core_inst is None: + core_inst = core.Core() + + if not core_inst._utils.validatePubKey(peer): + raise ValueError('Peer must be valid base32 ed25519 public key') + + http_server = WSGIServer(('127.0.0.1', bootstrap_port), bootstrap_app, log=None) + bootstrap_port = getOpenPort() + bootstrap_app = flask.Flask(__name__) + + bootstrap_address = '' + + @bootstrap_app.route('/ping') + def get_ping(): + return "pong!" + + @bootstrap_app.route('/bs/
', methods=['POST']) + def get_bootstrap(address): + if core_inst._utils.validateID(address): + # Set the bootstrap address then close the server + bootstrap_address = address + http_server.stop() + + with Controller.from_port() as controller: + # Connect to the Tor process for Onionr + controller.authenticate() + # Create the v3 onion service + response = controller.create_ephemeral_hidden_service({80: bootstrap_port}, await_publication = True, key_type='ED25519-V3') + + core_inst.insertBlock(response.hostname, header='con', sign=True, encryptType='asym', + asymPeer=peer, disableForward=True) + + # Run the bootstrap server + http_server.serve_forever() + # This line reached when server is shutdown by being bootstrapped + + # Now that the bootstrap server has received a server, return the address + return bootstrap_address diff --git a/onionr/onionrservices/connectionserver.py b/onionr/onionrservices/connectionserver.py new file mode 100644 index 00000000..d602d8ce --- /dev/null +++ b/onionr/onionrservices/connectionserver.py @@ -0,0 +1,4 @@ +class ConnectionServer: + def __init__(self): + return + \ No newline at end of file diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index f8fd13af..082b14fb 100755 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -329,7 +329,8 @@ class OnionrUtils: retVal = True return retVal - def validateID(self, id): + @staticmethod + def validateID(id): ''' Validate if an address is a valid tor or i2p hidden service ''' @@ -381,7 +382,8 @@ class OnionrUtils: except: return False - def isIntegerString(self, data): + @staticmethod + def isIntegerString(data): '''Check if a string is a valid base10 integer (also returns true if already an int)''' try: int(data) diff --git a/requirements.in b/requirements.in index 6b1cb663..7bf8ff02 100755 --- a/requirements.in +++ b/requirements.in @@ -5,5 +5,5 @@ gevent==1.3.6 defusedxml==0.5.0 Flask==1.0.2 PySocks==1.6.8 -stem==1.6.0 +stem==1.7.1 deadsimplekv==0.0.1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index b2c93d7c..9bb58ea1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -185,8 +185,8 @@ six==1.12.0 \ --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \ # via pynacl -stem==1.6.0 \ - --hash=sha256:d7fe1fb13ed5a94d610b5ad77e9f1b3404db0ca0586ded7a34afd323e3b849ed +stem==1.7.1 \ + --hash=sha256:c9eaf3116cb60c15995cbd3dec3a5cbc50e9bb6e062c4d6d42201e566f498ca2 urllib3==1.23 \ --hash=sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf \ --hash=sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5 From 2da5e0479a9dec52751d3d6a0bb2453d0a4ab563 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 19 Mar 2019 00:09:53 -0500 Subject: [PATCH 02/19] work on direct connections --- onionr/communicator.py | 21 +++----------- onionr/core.py | 1 - onionr/onionrservices/__init__.py | 20 ++++++++++--- onionr/onionrservices/bootstrapservice.py | 18 ++++++------ onionr/onionrservices/connectionserver.py | 29 +++++++++++++++++-- .../default-plugins/metadataprocessor/main.py | 26 +++++------------ 6 files changed, 62 insertions(+), 53 deletions(-) diff --git a/onionr/communicator.py b/onionr/communicator.py index 12fbabbc..37c0cbdb 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -24,7 +24,7 @@ from dependencies import secrets from utils import networkmerger import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block from communicatorutils import onionrdaemontools -import onionrsockets, onionr, onionrproofs +import onionrservices, onionr, onionrproofs from communicatorutils import onionrcommunicatortimers, proxypicker OnionrCommunicatorTimers = onionrcommunicatortimers.OnionrCommunicatorTimers @@ -126,9 +126,9 @@ class OnionrCommunicatorDaemon: #forwardSecrecyTimer.count = (forwardSecrecyTimer.frequency - 990) if config.get('general.socket_servers'): - self.socketServer = threading.Thread(target=onionrsockets.OnionrSocketServer, args=(self._core,)) - self.socketServer.start() - self.socketClient = onionrsockets.OnionrSocketClient(self._core) + self.services = onionrservices.OnionrServices(self._core) + else: + self.services = None # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking try: @@ -565,19 +565,6 @@ class OnionrCommunicatorDaemon: i.count = (i.frequency - 1) elif cmd[0] == 'uploadBlock': self.blocksToUpload.append(cmd[1]) - elif cmd[0] == 'startSocket': - # Create our own socket server - socketInfo = json.loads(cmd[1]) - socketInfo['id'] = uuid.uuid4() - self._core.startSocket = socketInfo - elif cmd[0] == 'addSocket': - # Socket server was created for us - socketInfo = json.loads(cmd[1]) - peer = socketInfo['peer'] - reason = socketInfo['reason'] - threading.Thread(target=self.socketClient.startSocket, args=(peer, reason)).start() - else: - logger.info('Recieved daemonQueue command:' + cmd[0]) if cmd[0] not in ('', None): if response != '': diff --git a/onionr/core.py b/onionr/core.py index f178fefd..0c8e1527 100755 --- a/onionr/core.py +++ b/onionr/core.py @@ -355,7 +355,6 @@ class Core: conn = sqlite3.connect(self.queueDB, timeout=30) c = conn.cursor() t = (command, data, date, responseID) - try: c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t) conn.commit() diff --git a/onionr/onionrservices/__init__.py b/onionr/onionrservices/__init__.py index 68612cf7..5ea0ef3e 100644 --- a/onionr/onionrservices/__init__.py +++ b/onionr/onionrservices/__init__.py @@ -1,17 +1,29 @@ +import time import stem import core -from . import connectionserver, connectionclient, bootstrapservice +from . import connectionserver, bootstrapservice class OnionrServices: def __init__(self, onionr_core): assert isinstance(onionr_core, core.Core) self._core = onionr_core self.servers = {} self.clients = {} + self.shutdown = False return - def create_server(self): - return - + def create_server(self, peer, address): + assert self._core._utils.validateID(address) + BOOTSTRAP_TRIES = 10 + TRY_WAIT = 3 + for x in range(BOOTSTRAP_TRIES): + if self._core._utils.doGetRequest('http://' + address + '/ping') == 'pong!': + connectionserver.ConnectionServer(peer, address, core_inst=self._core) + return True + else: + time.sleep(TRY_WAIT) + else: + return False + def create_client(self, peer): # Create ephemeral onion service to bootstrap connection address = bootstrapservice.bootstrap_client_service(peer) diff --git a/onionr/onionrservices/bootstrapservice.py b/onionr/onionrservices/bootstrapservice.py index 7a2c3823..b0b2fb7d 100644 --- a/onionr/onionrservices/bootstrapservice.py +++ b/onionr/onionrservices/bootstrapservice.py @@ -23,7 +23,7 @@ from flask import Flask import core from netcontroller import getOpenPort -def bootstrap_client_service(peer, core_inst=None): +def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=120): ''' Bootstrap client services ''' @@ -32,10 +32,10 @@ def bootstrap_client_service(peer, core_inst=None): if not core_inst._utils.validatePubKey(peer): raise ValueError('Peer must be valid base32 ed25519 public key') - - http_server = WSGIServer(('127.0.0.1', bootstrap_port), bootstrap_app, log=None) + bootstrap_port = getOpenPort() - bootstrap_app = flask.Flask(__name__) + bootstrap_app = Flask(__name__) + http_server = WSGIServer(('127.0.0.1', bootstrap_port), bootstrap_app, log=None) bootstrap_address = '' @@ -50,14 +50,14 @@ def bootstrap_client_service(peer, core_inst=None): bootstrap_address = address http_server.stop() - with Controller.from_port() as controller: + with Controller.from_port(port=core_inst.config.get('tor.controlPort')) as controller: # Connect to the Tor process for Onionr - controller.authenticate() + controller.authenticate(core_inst.config.get('tor.controlpassword')) # Create the v3 onion service - response = controller.create_ephemeral_hidden_service({80: bootstrap_port}, await_publication = True, key_type='ED25519-V3') + response = controller.create_ephemeral_hidden_service({80: bootstrap_port}, await_publication = True, key_content = 'ED25519-V3') - core_inst.insertBlock(response.hostname, header='con', sign=True, encryptType='asym', - asymPeer=peer, disableForward=True) + core_inst.insertBlock(response.service_id, header='con', sign=True, encryptType='asym', + asymPeer=peer, disableForward=True, expire=(core_inst._utils.getEpoch() + bootstrap_timeout)) # Run the bootstrap server http_server.serve_forever() diff --git a/onionr/onionrservices/connectionserver.py b/onionr/onionrservices/connectionserver.py index d602d8ce..54ff03c3 100644 --- a/onionr/onionrservices/connectionserver.py +++ b/onionr/onionrservices/connectionserver.py @@ -1,4 +1,27 @@ +import stem, flask +import core class ConnectionServer: - def __init__(self): - return - \ No newline at end of file + def __init__(self, peer, address, core_inst=None): + if core_inst is None: + self.core_inst = core.Core() + else: + self.core_inst = core_inst + + if not core_inst._utils.validatePubKey(peer): + raise ValueError('Peer must be valid base32 ed25519 public key') + + service_app = flask.Flask(__name__) + service_port = getOpenPort() + http_server = WSGIServer(('127.0.0.1', service_port), service_app, log=None) + + @service_app.route('/ping') + def get_ping(): + return "pong!" + + with Controller.from_port(port=core_inst.config.get('tor.controlPort')) as controller: + # Connect to the Tor process for Onionr + controller.authenticate(core_inst.config.get('tor.controlpassword')) + # Create the v3 onion service + response = controller.create_ephemeral_hidden_service({80: service_port}, await_publication = True, key_type='ED25519-V3') + logger.info('hosting on ' + response.service_id) + http_server.serve_forever() \ No newline at end of file diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py index e7277c7d..54f7dc18 100755 --- a/onionr/static-data/default-plugins/metadataprocessor/main.py +++ b/onionr/static-data/default-plugins/metadataprocessor/main.py @@ -46,6 +46,8 @@ def on_processblocks(api, data=None): myBlock = api.data['block'] blockType = api.data['type'] logger.info('blockType is ' + blockType) + utils = api.get_utils() + core = api.get_core() # Process specific block types @@ -54,26 +56,12 @@ def on_processblocks(api, data=None): if api.data['validSig'] == True: _processForwardKey(api, myBlock) # socket blocks - elif blockType == 'socket': + elif blockType == 'con': if api.data['validSig'] == True and myBlock.decrypted: # we check if it is decrypted as a way of seeing if it was for us - logger.info('Detected socket advertised to us...') - try: - address = myBlock.getMetadata('address') - except KeyError: - raise onionrexceptions.MissingAddress("Missing address for new socket") - try: - port = myBlock.getMetadata('port') - except KeyError: - raise ValueError("Missing port for new socket") - try: - reason = myBlock.getMetadata('reason') - except KeyError: - raise ValueError("Missing socket reason") - - socketInfo = json.dumps({'peer': api.data['signer'], 'address': address, 'port': port, 'create': False, 'reason': reason}) - api.get_core().daemonQueueAdd('addSocket', socketInfo) - else: - logger.warn("socket is not for us or is invalid") + myBlock.bcontent = utils.bytesToStr(myBlock.bcontent) + if utils.validateID('%s.onion' % (myBlock.bcontent,)): + logger.info('Detected socket advertised to us...') + core.keyStore.put('con', (myBlock.content, myBlock.signer)) def on_init(api, data = None): From 85adef16606c5c0e0c98ee418ade1c3bcb938296 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 22 Mar 2019 21:58:09 -0500 Subject: [PATCH 03/19] bootstrap blocks now inserted and detected --- onionr/communicator.py | 2 ++ onionr/communicatorutils/onionrcommunicatortimers.py | 5 +++-- onionr/onionrservices/__init__.py | 1 + onionr/onionrservices/bootstrapservice.py | 5 +++-- onionr/onionrservices/connectionserver.py | 2 +- onionr/onionrutils.py | 2 ++ .../default-plugins/metadataprocessor/main.py | 9 +-------- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/onionr/communicator.py b/onionr/communicator.py index 37c0cbdb..e4a513d6 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -24,6 +24,7 @@ from dependencies import secrets from utils import networkmerger import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block from communicatorutils import onionrdaemontools +from communicatorutils import servicecreator import onionrservices, onionr, onionrproofs from communicatorutils import onionrcommunicatortimers, proxypicker @@ -107,6 +108,7 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1) OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1) OnionrCommunicatorTimers(self, self.detectAPICrash, 30, maxThreads=1) + OnionrCommunicatorTimers(self, servicecreator.service_creator, 5, maxThreads=10, myArgs=(self,)) deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1) netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) diff --git a/onionr/communicatorutils/onionrcommunicatortimers.py b/onionr/communicatorutils/onionrcommunicatortimers.py index c15f20fc..a94a7821 100755 --- a/onionr/communicatorutils/onionrcommunicatortimers.py +++ b/onionr/communicatorutils/onionrcommunicatortimers.py @@ -20,7 +20,7 @@ ''' import threading, onionrexceptions, logger class OnionrCommunicatorTimers: - def __init__(self, daemonInstance, timerFunction, frequency, makeThread=True, threadAmount=1, maxThreads=5, requiresPeer=False): + def __init__(self, daemonInstance, timerFunction, frequency, makeThread=True, threadAmount=1, maxThreads=5, requiresPeer=False, myArgs=[]): self.timerFunction = timerFunction self.frequency = frequency self.threadAmount = threadAmount @@ -29,6 +29,7 @@ class OnionrCommunicatorTimers: self.daemonInstance = daemonInstance self.maxThreads = maxThreads self._core = self.daemonInstance._core + self.args = myArgs self.daemonInstance.timers.append(self) self.count = 0 @@ -55,7 +56,7 @@ class OnionrCommunicatorTimers: logger.debug('%s is currently using the maximum number of threads, not starting another.' % self.timerFunction.__name__) else: self.daemonInstance.threadCounts[self.timerFunction.__name__] += 1 - newThread = threading.Thread(target=self.timerFunction) + newThread = threading.Thread(target=self.timerFunction, args=self.args) newThread.start() else: self.timerFunction() diff --git a/onionr/onionrservices/__init__.py b/onionr/onionrservices/__init__.py index 5ea0ef3e..7194ea22 100644 --- a/onionr/onionrservices/__init__.py +++ b/onionr/onionrservices/__init__.py @@ -2,6 +2,7 @@ import time import stem import core from . import connectionserver, bootstrapservice + class OnionrServices: def __init__(self, onionr_core): assert isinstance(onionr_core, core.Core) diff --git a/onionr/onionrservices/bootstrapservice.py b/onionr/onionrservices/bootstrapservice.py index b0b2fb7d..5dafe0f3 100644 --- a/onionr/onionrservices/bootstrapservice.py +++ b/onionr/onionrservices/bootstrapservice.py @@ -23,7 +23,7 @@ from flask import Flask import core from netcontroller import getOpenPort -def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=120): +def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300): ''' Bootstrap client services ''' @@ -54,7 +54,8 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=120): # Connect to the Tor process for Onionr controller.authenticate(core_inst.config.get('tor.controlpassword')) # Create the v3 onion service - response = controller.create_ephemeral_hidden_service({80: bootstrap_port}, await_publication = True, key_content = 'ED25519-V3') + #response = controller.create_ephemeral_hidden_service({80: bootstrap_port}, await_publication = True, key_content = 'ED25519-V3') + response = controller.create_ephemeral_hidden_service(80, key_type = 'NEW', await_publication = True) core_inst.insertBlock(response.service_id, header='con', sign=True, encryptType='asym', asymPeer=peer, disableForward=True, expire=(core_inst._utils.getEpoch() + bootstrap_timeout)) diff --git a/onionr/onionrservices/connectionserver.py b/onionr/onionrservices/connectionserver.py index 54ff03c3..32d2640a 100644 --- a/onionr/onionrservices/connectionserver.py +++ b/onionr/onionrservices/connectionserver.py @@ -22,6 +22,6 @@ class ConnectionServer: # Connect to the Tor process for Onionr controller.authenticate(core_inst.config.get('tor.controlpassword')) # Create the v3 onion service - response = controller.create_ephemeral_hidden_service({80: service_port}, await_publication = True, key_type='ED25519-V3') + response = controller.create_ephemeral_hidden_service({80: service_port}, await_publication = True, key_type='NEW') logger.info('hosting on ' + response.service_id) http_server.serve_forever() \ No newline at end of file diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 082b14fb..b9ae61c4 100755 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -182,6 +182,8 @@ class OnionrUtils: pass else: self._core.updateBlockInfo(blockHash, 'expire', expireTime) + if not blockType is None: + self._core.updateBlockInfo(blockHash, 'dataType', blockType) else: pass #logger.debug('Not processing metadata on encrypted block we cannot decrypt.') diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py index 54f7dc18..787b0174 100755 --- a/onionr/static-data/default-plugins/metadataprocessor/main.py +++ b/onionr/static-data/default-plugins/metadataprocessor/main.py @@ -55,14 +55,7 @@ def on_processblocks(api, data=None): if blockType == 'forwardKey': if api.data['validSig'] == True: _processForwardKey(api, myBlock) - # socket blocks - elif blockType == 'con': - if api.data['validSig'] == True and myBlock.decrypted: # we check if it is decrypted as a way of seeing if it was for us - myBlock.bcontent = utils.bytesToStr(myBlock.bcontent) - if utils.validateID('%s.onion' % (myBlock.bcontent,)): - logger.info('Detected socket advertised to us...') - core.keyStore.put('con', (myBlock.content, myBlock.signer)) - + def on_init(api, data = None): pluginapi = api From b9beef3cce6f642869824ae8752bd085e71ff8ce Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 23 Mar 2019 21:56:46 -0500 Subject: [PATCH 04/19] direct connections closer to working --- onionr/communicator.py | 12 ++++++------ onionr/communicatorutils/servicecreator.py | 17 +++++++++++++++++ onionr/onionrservices/__init__.py | 3 +-- onionr/onionrservices/bootstrapservice.py | 3 +-- onionr/static-data/default_config.json | 2 +- 5 files changed, 26 insertions(+), 11 deletions(-) create mode 100644 onionr/communicatorutils/servicecreator.py diff --git a/onionr/communicator.py b/onionr/communicator.py index e4a513d6..670e08fa 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -108,7 +108,12 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1) OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1) OnionrCommunicatorTimers(self, self.detectAPICrash, 30, maxThreads=1) - OnionrCommunicatorTimers(self, servicecreator.service_creator, 5, maxThreads=10, myArgs=(self,)) + if config.get('general.socket_servers', False): + self.services = onionrservices.OnionrServices(self._core) + self.active_services = [] + OnionrCommunicatorTimers(self, servicecreator.service_creator, 5, maxThreads=10, myArgs=(self,)) + else: + self.services = None deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1) netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) @@ -127,11 +132,6 @@ class OnionrCommunicatorDaemon: blockCleanupTimer.count = (blockCleanupTimer.frequency - 5) #forwardSecrecyTimer.count = (forwardSecrecyTimer.frequency - 990) - if config.get('general.socket_servers'): - self.services = onionrservices.OnionrServices(self._core) - else: - self.services = None - # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking try: while not self.shutdown: diff --git a/onionr/communicatorutils/servicecreator.py b/onionr/communicatorutils/servicecreator.py new file mode 100644 index 00000000..4914547a --- /dev/null +++ b/onionr/communicatorutils/servicecreator.py @@ -0,0 +1,17 @@ +import communicator, onionrblockapi +def service_creator(daemon): + assert isinstance(daemon, communicator.OnionrCommunicatorDaemon) + core = daemon._core + utils = core._utils + + # Find socket connection blocks + con_blocks = core.getBlocksByType('con') + for b in con_blocks: + if not b in daemon.active_services: + bl = onionrblockapi.Block(b, core=core, decrypt=True) + bs = utils.bytesToStr(bl.bcontent) + '.onion' + if utils.validatePubKey(bl.signer) and utils.validateID(bs): + daemon.active_services.append(b) + daemon.services.create_server(bl.signer, bs) + + daemon.decrementThreadCount('service_creator') \ No newline at end of file diff --git a/onionr/onionrservices/__init__.py b/onionr/onionrservices/__init__.py index 7194ea22..3f0e81e5 100644 --- a/onionr/onionrservices/__init__.py +++ b/onionr/onionrservices/__init__.py @@ -17,9 +17,8 @@ class OnionrServices: BOOTSTRAP_TRIES = 10 TRY_WAIT = 3 for x in range(BOOTSTRAP_TRIES): - if self._core._utils.doGetRequest('http://' + address + '/ping') == 'pong!': + if self._core._utils.doGetRequest('http://' + address + '/ping', port=self._core.config.get('tor.socksport')) == 'pong!': connectionserver.ConnectionServer(peer, address, core_inst=self._core) - return True else: time.sleep(TRY_WAIT) else: diff --git a/onionr/onionrservices/bootstrapservice.py b/onionr/onionrservices/bootstrapservice.py index 5dafe0f3..33c4e8b4 100644 --- a/onionr/onionrservices/bootstrapservice.py +++ b/onionr/onionrservices/bootstrapservice.py @@ -55,8 +55,7 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300): controller.authenticate(core_inst.config.get('tor.controlpassword')) # Create the v3 onion service #response = controller.create_ephemeral_hidden_service({80: bootstrap_port}, await_publication = True, key_content = 'ED25519-V3') - response = controller.create_ephemeral_hidden_service(80, key_type = 'NEW', await_publication = True) - + response = controller.create_ephemeral_hidden_service({80: bootstrap_port}, key_type = 'NEW', await_publication = True) core_inst.insertBlock(response.service_id, header='con', sign=True, encryptType='asym', asymPeer=peer, disableForward=True, expire=(core_inst._utils.getEpoch() + bootstrap_timeout)) diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 86261502..2b9d47df 100755 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -4,7 +4,7 @@ "display_header" : false, "minimum_block_pow": 4, "minimum_send_pow": 4, - "socket_servers": false, + "socket_servers": true, "security_level": 0, "max_block_age": 2678400, "bypass_tor_check": false, From dfbb0a512ca95d587e8791f3cc0af56e405da4d8 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 24 Mar 2019 22:32:17 -0500 Subject: [PATCH 05/19] cleanup, work, and bugfixes for direct connections --- onionr/api.py | 7 ++-- onionr/onionrblockapi.py | 5 ++- onionr/onionrservices/__init__.py | 4 ++- onionr/onionrservices/bootstrapservice.py | 12 ++++--- onionr/onionrservices/connectionserver.py | 42 ++++++++++++++++++++--- 5 files changed, 57 insertions(+), 13 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index 79694238..4f7276aa 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -39,7 +39,7 @@ class FDSafeHandler(WSGIHandler): except Timeout as ex: raise -def setBindIP(filePath): +def setBindIP(filePath, writeOut=True): '''Set a random localhost IP to a specified file (intended for private or public API localhost IPs)''' if config.get('general.random_bind_ip', True): hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))] @@ -55,8 +55,9 @@ def setBindIP(filePath): s.close() else: data = '127.0.0.1' - with open(filePath, 'w') as bindFile: - bindFile.write(data) + if writeOut: + with open(filePath, 'w') as bindFile: + bindFile.write(data) return data class PublicAPI: diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index e7a4395b..1f890bcc 100755 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -215,7 +215,10 @@ class Block: ''' if self.exists(): - os.remove(self.getBlockFile()) + try: + os.remove(self.getBlockFile()) + except TypeError: + pass self.getCore().removeBlock(self.getHash()) return True return False diff --git a/onionr/onionrservices/__init__.py b/onionr/onionrservices/__init__.py index 3f0e81e5..21ded6ec 100644 --- a/onionr/onionrservices/__init__.py +++ b/onionr/onionrservices/__init__.py @@ -16,8 +16,10 @@ class OnionrServices: assert self._core._utils.validateID(address) BOOTSTRAP_TRIES = 10 TRY_WAIT = 3 + base_url = 'http://%s/' % (address,) + socks = self._core.config.get('tor.socksport') for x in range(BOOTSTRAP_TRIES): - if self._core._utils.doGetRequest('http://' + address + '/ping', port=self._core.config.get('tor.socksport')) == 'pong!': + if self._core._utils.doGetRequest(base_url + 'ping', port=socks, ignoreAPI=True) == 'pong!': connectionserver.ConnectionServer(peer, address, core_inst=self._core) else: time.sleep(TRY_WAIT) diff --git a/onionr/onionrservices/bootstrapservice.py b/onionr/onionrservices/bootstrapservice.py index 33c4e8b4..f1e0b35c 100644 --- a/onionr/onionrservices/bootstrapservice.py +++ b/onionr/onionrservices/bootstrapservice.py @@ -17,11 +17,12 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' +import time from gevent.pywsgi import WSGIServer, WSGIHandler from stem.control import Controller from flask import Flask import core -from netcontroller import getOpenPort +from netcontroller import getOpenPort def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300): ''' @@ -38,6 +39,7 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300): http_server = WSGIServer(('127.0.0.1', bootstrap_port), bootstrap_app, log=None) bootstrap_address = '' + shutdown = False @bootstrap_app.route('/ping') def get_ping(): @@ -48,20 +50,22 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300): if core_inst._utils.validateID(address): # Set the bootstrap address then close the server bootstrap_address = address - http_server.stop() + shutdown = True + return "success" with Controller.from_port(port=core_inst.config.get('tor.controlPort')) as controller: # Connect to the Tor process for Onionr controller.authenticate(core_inst.config.get('tor.controlpassword')) # Create the v3 onion service - #response = controller.create_ephemeral_hidden_service({80: bootstrap_port}, await_publication = True, key_content = 'ED25519-V3') response = controller.create_ephemeral_hidden_service({80: bootstrap_port}, key_type = 'NEW', await_publication = True) core_inst.insertBlock(response.service_id, header='con', sign=True, encryptType='asym', asymPeer=peer, disableForward=True, expire=(core_inst._utils.getEpoch() + bootstrap_timeout)) # Run the bootstrap server - http_server.serve_forever() + threading.Thread(target=http_server.serve_forever).start() # This line reached when server is shutdown by being bootstrapped + while not shutdown and not core_inst.killSockets: + time.sleep(1) # Now that the bootstrap server has received a server, return the address return bootstrap_address diff --git a/onionr/onionrservices/connectionserver.py b/onionr/onionrservices/connectionserver.py index 32d2640a..60aff03e 100644 --- a/onionr/onionrservices/connectionserver.py +++ b/onionr/onionrservices/connectionserver.py @@ -1,5 +1,30 @@ -import stem, flask -import core +''' + Onionr - P2P Anonymous Storage Network + + This module does the second part of the bootstrap block handshake and creates the API server +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import threading, time +from gevent.pywsgi import WSGIServer, WSGIHandler +from stem.control import Controller +from flask import Flask +import core, logger +from netcontroller import getOpenPort +from api import setBindIP + class ConnectionServer: def __init__(self, peer, address, core_inst=None): if core_inst is None: @@ -10,10 +35,15 @@ class ConnectionServer: if not core_inst._utils.validatePubKey(peer): raise ValueError('Peer must be valid base32 ed25519 public key') - service_app = flask.Flask(__name__) + socks = core_inst.config.get('tor.socksport') # Load config for Tor socks port for proxy + service_app = Flask(__name__) # Setup Flask app for server. service_port = getOpenPort() + service_ip = setBindIP() http_server = WSGIServer(('127.0.0.1', service_port), service_app, log=None) + + # TODO define basic endpoints useful for direct connections like stats + # TODO load endpoints from plugins @service_app.route('/ping') def get_ping(): return "pong!" @@ -23,5 +53,9 @@ class ConnectionServer: controller.authenticate(core_inst.config.get('tor.controlpassword')) # Create the v3 onion service response = controller.create_ephemeral_hidden_service({80: service_port}, await_publication = True, key_type='NEW') + self.core_inst._utils.doPostRequest('http://' + address + '/bs/' + response.service_id, port=socks) logger.info('hosting on ' + response.service_id) - http_server.serve_forever() \ No newline at end of file + threading.Thread(target=http_server.serve_forever).start() + while not self.core_inst.killSockets: + time.sleep(1) + http_server.stop() \ No newline at end of file From d3f57fe3e78c9f43841b40c1b7abe4877e30dfc6 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 25 Mar 2019 18:46:25 -0500 Subject: [PATCH 06/19] bug fixes for direct connnections --- onionr/api.py | 4 ++-- onionr/communicator.py | 2 +- onionr/onionrservices/bootstrapservice.py | 28 +++++++++++++---------- onionr/onionrservices/connectionserver.py | 11 ++++----- 4 files changed, 23 insertions(+), 22 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index 4f7276aa..e542bd35 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -39,7 +39,7 @@ class FDSafeHandler(WSGIHandler): except Timeout as ex: raise -def setBindIP(filePath, writeOut=True): +def setBindIP(filePath=''): '''Set a random localhost IP to a specified file (intended for private or public API localhost IPs)''' if config.get('general.random_bind_ip', True): hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))] @@ -55,7 +55,7 @@ def setBindIP(filePath, writeOut=True): s.close() else: data = '127.0.0.1' - if writeOut: + if filePath != '': with open(filePath, 'w') as bindFile: bindFile.write(data) return data diff --git a/onionr/communicator.py b/onionr/communicator.py index 670e08fa..bb56ca19 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -111,7 +111,7 @@ class OnionrCommunicatorDaemon: if config.get('general.socket_servers', False): self.services = onionrservices.OnionrServices(self._core) self.active_services = [] - OnionrCommunicatorTimers(self, servicecreator.service_creator, 5, maxThreads=10, myArgs=(self,)) + OnionrCommunicatorTimers(self, servicecreator.service_creator, 5, maxThreads=50, myArgs=(self,)) else: self.services = None deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1) diff --git a/onionr/onionrservices/bootstrapservice.py b/onionr/onionrservices/bootstrapservice.py index f1e0b35c..95ae0158 100644 --- a/onionr/onionrservices/bootstrapservice.py +++ b/onionr/onionrservices/bootstrapservice.py @@ -17,10 +17,10 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import time +import time, threading, uuid from gevent.pywsgi import WSGIServer, WSGIHandler from stem.control import Controller -from flask import Flask +from flask import Flask, Response import core from netcontroller import getOpenPort @@ -40,6 +40,7 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300): bootstrap_address = '' shutdown = False + bs_id = str(uuid.uuid4()) @bootstrap_app.route('/ping') def get_ping(): @@ -47,25 +48,28 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300): @bootstrap_app.route('/bs/
', methods=['POST']) def get_bootstrap(address): - if core_inst._utils.validateID(address): + if core_inst._utils.validateID(address + '.onion'): # Set the bootstrap address then close the server - bootstrap_address = address - shutdown = True - return "success" + bootstrap_address = address + '.onion' + core_inst.keyStore.put(bs_id, bootstrap_address) + http_server.stop() + return Response("success") + else: + return Response("") with Controller.from_port(port=core_inst.config.get('tor.controlPort')) as controller: # Connect to the Tor process for Onionr controller.authenticate(core_inst.config.get('tor.controlpassword')) # Create the v3 onion service - response = controller.create_ephemeral_hidden_service({80: bootstrap_port}, key_type = 'NEW', await_publication = True) + response = controller.create_ephemeral_hidden_service({80: bootstrap_port}, key_type = 'NEW', key_content = 'ED25519-V3', await_publication = True) core_inst.insertBlock(response.service_id, header='con', sign=True, encryptType='asym', asymPeer=peer, disableForward=True, expire=(core_inst._utils.getEpoch() + bootstrap_timeout)) - # Run the bootstrap server - threading.Thread(target=http_server.serve_forever).start() + try: + http_server.serve_forever() + except TypeError: + pass # This line reached when server is shutdown by being bootstrapped - while not shutdown and not core_inst.killSockets: - time.sleep(1) # Now that the bootstrap server has received a server, return the address - return bootstrap_address + return core_inst.keyStore.get(bs_id) diff --git a/onionr/onionrservices/connectionserver.py b/onionr/onionrservices/connectionserver.py index 60aff03e..d8a5f773 100644 --- a/onionr/onionrservices/connectionserver.py +++ b/onionr/onionrservices/connectionserver.py @@ -23,7 +23,7 @@ from stem.control import Controller from flask import Flask import core, logger from netcontroller import getOpenPort -from api import setBindIP +import api class ConnectionServer: def __init__(self, peer, address, core_inst=None): @@ -38,9 +38,8 @@ class ConnectionServer: socks = core_inst.config.get('tor.socksport') # Load config for Tor socks port for proxy service_app = Flask(__name__) # Setup Flask app for server. service_port = getOpenPort() - service_ip = setBindIP() + service_ip = api.setBindIP() http_server = WSGIServer(('127.0.0.1', service_port), service_app, log=None) - # TODO define basic endpoints useful for direct connections like stats # TODO load endpoints from plugins @@ -52,10 +51,8 @@ class ConnectionServer: # Connect to the Tor process for Onionr controller.authenticate(core_inst.config.get('tor.controlpassword')) # Create the v3 onion service - response = controller.create_ephemeral_hidden_service({80: service_port}, await_publication = True, key_type='NEW') + response = controller.create_ephemeral_hidden_service({80: service_port}, await_publication = True, key_type='NEW', key_content = 'ED25519-V3') self.core_inst._utils.doPostRequest('http://' + address + '/bs/' + response.service_id, port=socks) logger.info('hosting on ' + response.service_id) - threading.Thread(target=http_server.serve_forever).start() - while not self.core_inst.killSockets: - time.sleep(1) + http_server.serve_forever() http_server.stop() \ No newline at end of file From c855a2ea6bc72115c88058487399e47e212d2e53 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 25 Mar 2019 23:25:46 -0500 Subject: [PATCH 07/19] more work on direct connections --- onionr/api.py | 17 +++-------------- onionr/communicator.py | 4 +++- onionr/onionrservices/bootstrapservice.py | 6 ++++++ onionr/onionrservices/connectionserver.py | 3 ++- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index e542bd35..7e552355 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -27,6 +27,7 @@ from onionrblockapi import Block import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config import httpapi from httpapi import friendsapi, simplecache +from onionrservices import httpheaders import onionr class FDSafeHandler(WSGIHandler): @@ -92,17 +93,9 @@ class PublicAPI: @app.after_request def sendHeaders(resp): '''Send api, access control headers''' - resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch, since we can't fully remove the header. - # CSP to prevent XSS. Mainly for client side attacks (if hostname protection could somehow be bypassed) - resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'" - # Prevent click jacking - resp.headers['X-Frame-Options'] = 'deny' - # No sniff is possibly not needed - resp.headers['X-Content-Type-Options'] = "nosniff" + resp = httpheaders.set_default_onionr_http_headers(resp) # Network API version resp.headers['X-API'] = onionr.API_VERSION - # Close connections to limit FD use - resp.headers['Connection'] = "close" self.lastRequest = clientAPI._core._utils.getRoundedEpoch(roundS=5) return resp @@ -300,15 +293,11 @@ class API: @app.after_request def afterReq(resp): # Security headers + resp = httpheaders.set_default_onionr_http_headers(resp) if request.endpoint == 'site': resp.headers['Content-Security-Policy'] = "default-src 'none'; style-src data: 'unsafe-inline'; img-src data:" else: resp.headers['Content-Security-Policy'] = "default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self'; img-src 'self'; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'self'" - resp.headers['X-Frame-Options'] = 'deny' - resp.headers['X-Content-Type-Options'] = "nosniff" - resp.headers['Server'] = '' - resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch. - resp.headers['Connection'] = "close" return resp @app.route('/board/', endpoint='board') diff --git a/onionr/communicator.py b/onionr/communicator.py index bb56ca19..47f8e5d6 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -111,6 +111,7 @@ class OnionrCommunicatorDaemon: if config.get('general.socket_servers', False): self.services = onionrservices.OnionrServices(self._core) self.active_services = [] + self.service_greenlets = [] OnionrCommunicatorTimers(self, servicecreator.service_creator, 5, maxThreads=50, myArgs=(self,)) else: self.services = None @@ -148,7 +149,8 @@ class OnionrCommunicatorDaemon: pass logger.info('Goodbye.') - self._core.killSockets = True + for server in self.service_greenlets: + server.stop() self._core._utils.localCommand('shutdown') # shutdown the api time.sleep(0.5) diff --git a/onionr/onionrservices/bootstrapservice.py b/onionr/onionrservices/bootstrapservice.py index 95ae0158..292df4ed 100644 --- a/onionr/onionrservices/bootstrapservice.py +++ b/onionr/onionrservices/bootstrapservice.py @@ -37,6 +37,12 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300): bootstrap_port = getOpenPort() bootstrap_app = Flask(__name__) http_server = WSGIServer(('127.0.0.1', bootstrap_port), bootstrap_app, log=None) + try: + core_inst.onionrInst.communicatorInst + except AttributeError: + pass + else: + core_inst.onionrInst.communicatorInst.service_greenlets.append(http_server) bootstrap_address = '' shutdown = False diff --git a/onionr/onionrservices/connectionserver.py b/onionr/onionrservices/connectionserver.py index d8a5f773..6b04645c 100644 --- a/onionr/onionrservices/connectionserver.py +++ b/onionr/onionrservices/connectionserver.py @@ -40,6 +40,7 @@ class ConnectionServer: service_port = getOpenPort() service_ip = api.setBindIP() http_server = WSGIServer(('127.0.0.1', service_port), service_app, log=None) + core_inst.onionrInst.communicatorInst.service_greenlets.append(http_server) # TODO define basic endpoints useful for direct connections like stats # TODO load endpoints from plugins @@ -53,6 +54,6 @@ class ConnectionServer: # Create the v3 onion service response = controller.create_ephemeral_hidden_service({80: service_port}, await_publication = True, key_type='NEW', key_content = 'ED25519-V3') self.core_inst._utils.doPostRequest('http://' + address + '/bs/' + response.service_id, port=socks) - logger.info('hosting on ' + response.service_id) + logger.info('hosting on %s with %s' % (response.service_id, peer)) http_server.serve_forever() http_server.stop() \ No newline at end of file From 15e321feeee2d2672c3fa6198e5b4008d4c643a0 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 26 Mar 2019 10:23:31 -0500 Subject: [PATCH 08/19] added httpheaders.py --- onionr/onionrservices/httpheaders.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 onionr/onionrservices/httpheaders.py diff --git a/onionr/onionrservices/httpheaders.py b/onionr/onionrservices/httpheaders.py new file mode 100644 index 00000000..2b2c1770 --- /dev/null +++ b/onionr/onionrservices/httpheaders.py @@ -0,0 +1,28 @@ +''' + Onionr - P2P Anonymous Storage Network + + Set default onionr http headers +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +def set_default_onionr_http_headers(flask_response): + '''Response headers''' + flask_response.headers['Content-Security-Policy'] = "default-src 'none'; style-src data: 'unsafe-inline'; img-src data:" + flask_response.headers['X-Frame-Options'] = 'deny' + flask_response.headers['X-Content-Type-Options'] = "nosniff" + flask_response.headers['Server'] = '' + flask_response.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch. + flask_response.headers['Connection'] = "close" + return flask_response \ No newline at end of file From fb32201df3c374021b2614dabc9105979386a035 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Wed, 27 Mar 2019 12:38:46 -0500 Subject: [PATCH 09/19] added clandestine --- onionr/onionrservices/bootstrapservice.py | 7 + onionr/onionrservices/connectionserver.py | 7 + onionr/onionrsockets.py | 172 ------------------ .../default-plugins/clandestine/controlapi.py | 26 +++ .../default-plugins/clandestine/info.json | 5 + .../default-plugins/clandestine/main.py | 46 +++++ 6 files changed, 91 insertions(+), 172 deletions(-) delete mode 100755 onionr/onionrsockets.py create mode 100644 onionr/static-data/default-plugins/clandestine/controlapi.py create mode 100755 onionr/static-data/default-plugins/clandestine/info.json create mode 100755 onionr/static-data/default-plugins/clandestine/main.py diff --git a/onionr/onionrservices/bootstrapservice.py b/onionr/onionrservices/bootstrapservice.py index 292df4ed..a2952563 100644 --- a/onionr/onionrservices/bootstrapservice.py +++ b/onionr/onionrservices/bootstrapservice.py @@ -23,6 +23,7 @@ from stem.control import Controller from flask import Flask, Response import core from netcontroller import getOpenPort +from . import httpheaders def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300): ''' @@ -52,6 +53,12 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300): def get_ping(): return "pong!" + @bootstrap_app.after_request + def afterReq(resp): + # Security headers + resp = httpheaders.set_default_onionr_http_headers(resp) + return resp + @bootstrap_app.route('/bs/
', methods=['POST']) def get_bootstrap(address): if core_inst._utils.validateID(address + '.onion'): diff --git a/onionr/onionrservices/connectionserver.py b/onionr/onionrservices/connectionserver.py index 6b04645c..ee586e2b 100644 --- a/onionr/onionrservices/connectionserver.py +++ b/onionr/onionrservices/connectionserver.py @@ -24,6 +24,7 @@ from flask import Flask import core, logger from netcontroller import getOpenPort import api +from . import httpheaders class ConnectionServer: def __init__(self, peer, address, core_inst=None): @@ -48,6 +49,12 @@ class ConnectionServer: def get_ping(): return "pong!" + @service_app.after_request + def afterReq(resp): + # Security headers + resp = httpheaders.set_default_onionr_http_headers(resp) + return resp + with Controller.from_port(port=core_inst.config.get('tor.controlPort')) as controller: # Connect to the Tor process for Onionr controller.authenticate(core_inst.config.get('tor.controlpassword')) diff --git a/onionr/onionrsockets.py b/onionr/onionrsockets.py deleted file mode 100755 index aa02c3db..00000000 --- a/onionr/onionrsockets.py +++ /dev/null @@ -1,172 +0,0 @@ -''' - Onionr - P2P Anonymous Storage Network - - Onionr Socket interface -''' -''' - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -''' -import stem.control -import threading -import socks, config, uuid -import onionrexceptions, time, requests, onionrblockapi, logger -from dependencies import secrets -from gevent.pywsgi import WSGIServer -from flask import request, Response, abort -import flask -class OnionrSocketServer: - def __init__(self, coreInst): - self._core = coreInst - app = flask.Flask(__name__) - self._core.socketServerConnData = {} - self.bindPort = 0 - - self.sockets = {} - - while self.bindPort < 1024: - self.bindPort = secrets.randbelow(65535) - - self.responseData = {} - - threading.Thread(target=self.detectShutdown).start() - threading.Thread(target=self.socketStarter).start() - app = flask.Flask(__name__) - self.http_server = WSGIServer(('127.0.0.1', self.bindPort), app) - self.http_server.serve_forever() - - @app.route('/dc/', methods=['POST']) - def acceptConn(self): - data = request.form['data'] - data = self._core._utils.bytesTorStr(data) - data = {'date': self._core._utils.getEpoch(), 'data': data} - myPeer = '' - retData = '' - for peer in self.sockets: - if self.sockets[peer] == request.host: - myPeer = peer - break - else: - return "" - - if request.host in self.sockets: - self._core.socketServerConnData[myPeer].append(data) - else: - self._core.socketServerConnData[myPeer] = [data] - - try: - retData = self._core.socketServerResponseData[myPeer] - except KeyError: - pass - else: - self._core.socketServerResponseData[myPeer] = '' - - return retData - - def socketStarter(self): - while not self._core.killSockets: - try: - self.addSocket(self._core.startSocket['peer'], reason=self._core.startSocket['reason']) - except KeyError: - pass - else: - logger.info('%s socket started with %s' % (self._core.startSocket['reason'], self._core.startSocket['peer'])) - self._core.startSocket = {} - time.sleep(1) - - def detectShutdown(self): - while not self._core.killSockets: - time.sleep(5) - - logger.debug('Killing socket server...') - self.http_server.stop() - - def addSocket(self, peer, reason=''): - bindPort = 1337 - - assert len(reason) <= 12 - - with stem.control.Controller.from_port(port=config.get('tor.controlPort')) as controller: - controller.authenticate(config.get('tor.controlpassword')) - - socket = controller.create_ephemeral_hidden_service({80: bindPort}, await_publication = True) - self.sockets[peer] = socket.service_id + '.onion' - - self.responseData[socket.service_id + '.onion'] = '' - - self._core.insertBlock(str(uuid.uuid4()), header='socket', sign=True, encryptType='asym', asymPeer=peer, meta={'reason': reason, 'address': socket.service_id + '.onion'}) - self._core.socketReasons[peer] = reason - return - -class OnionrSocketClient: - def __init__(self, coreInst): - self.sockets = {} # pubkey: tor address - self.connPool = {} - self.sendData = {} - self._core = coreInst - self.response = '' - self.request = '' - self.connected = False - self.killSocket = False - - def startSocket(self, peer, reason): - address = '' - logger.info('Trying to find socket server for %s' % (peer,)) - # Find the newest open socket for a given peer - for block in self._core.getBlocksByType('socket'): - block = onionrblockapi.Block(block, core=self._core) - if block.decrypt(): - theSigner = block.signer - try: - theSigner = theSigner.decode() - except AttributeError: - pass - if block.verifySig() and theSigner == peer: - address = block.getMetadata('address') - if self._core._utils.validateID(address): - # If we got their address, it is valid, and verified, we can break out - if block.getMetadata('reason') == reason: - break - else: - logger.error('The socket the peer opened is not for %s' % (reason,)) - else: - logger.error('Peer transport id is invalid for socket: %s' % (address,)) - address = '' - else: - logger.warn('Block has invalid sig or id, was for %s' % (theSigner,)) - if address != '': - logger.info('%s socket client started with %s' % (reason, peer)) - self.sockets[peer] = address - data = 'hey' - while not self.killSocket: - try: - data = self.sendData[peer] - logger.info('Sending %s to %s' % (data, peer)) - except KeyError: - pass - else: - self.sendData[peer] = '' - postData = {'data': data} - self.connPool[peer] = {'date': self._core._utils.getEpoch(), 'data': self._core._utils.doPostRequest('http://' + address + '/dc/', data=postData)} - time.sleep(2) - - def getResponse(self, peer): - retData = '' - try: - retData = self.connPool[peer] - except KeyError: - pass - return - - def sendData(self, peer, data): - self.sendData[peer] = data diff --git a/onionr/static-data/default-plugins/clandestine/controlapi.py b/onionr/static-data/default-plugins/clandestine/controlapi.py new file mode 100644 index 00000000..ce341f82 --- /dev/null +++ b/onionr/static-data/default-plugins/clandestine/controlapi.py @@ -0,0 +1,26 @@ +''' + Onionr - P2P Anonymous Storage Network + + HTTP endpoints for controlling IMs +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +from flask import Response, request, redirect, Blueprint, abort + +flask_blueprint = Blueprint('clandenstine', __name__) + +@flask_blueprint.route('/mail/ping') +def ping(): + return 'pong!' diff --git a/onionr/static-data/default-plugins/clandestine/info.json b/onionr/static-data/default-plugins/clandestine/info.json new file mode 100755 index 00000000..7c5a143c --- /dev/null +++ b/onionr/static-data/default-plugins/clandestine/info.json @@ -0,0 +1,5 @@ +{ + "name" : "clandestine", + "version" : "1.0", + "author" : "onionr" +} diff --git a/onionr/static-data/default-plugins/clandestine/main.py b/onionr/static-data/default-plugins/clandestine/main.py new file mode 100755 index 00000000..bc7ca112 --- /dev/null +++ b/onionr/static-data/default-plugins/clandestine/main.py @@ -0,0 +1,46 @@ +''' + Onionr - P2P Anonymous Storage Network + + Instant message conversations with Onionr peers +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' + +# Imports some useful libraries +import locale + +locale.setlocale(locale.LC_ALL, '') + +plugin_name = 'clandenstine' +PLUGIN_VERSION = '0.0.0' + +sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) +from . import controlapi +flask_blueprint = controlapi.flask_blueprint + +class Clandenstine: + def __init__(self, pluginapi): + self.myCore = pluginapi.get_core() + +def on_init(api, data = None): + ''' + This event is called after Onionr is initialized, but before the command + inputted is executed. Could be called when daemon is starting or when + just the client is running. + ''' + + pluginapi = api + chat = Clandenstine(pluginapi) + return From e1c62fee5bc342e467f6e46ef0748f61189214d9 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Wed, 27 Mar 2019 13:55:43 -0500 Subject: [PATCH 10/19] improved expiry --- onionr/communicator.py | 9 +++++++-- onionr/etc/onionrvalues.py | 3 ++- onionr/onionrutils.py | 10 +++++----- onionr/static-data/bootstrap-nodes.txt | 1 + onionr/static-data/default-plugins/clandestine/main.py | 2 +- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/onionr/communicator.py b/onionr/communicator.py index 47f8e5d6..ef94ae14 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -149,8 +149,13 @@ class OnionrCommunicatorDaemon: pass logger.info('Goodbye.') - for server in self.service_greenlets: - server.stop() + try: + self.service_greenlets + except AttributeError: + pass + else: + for server in self.service_greenlets: + server.stop() self._core._utils.localCommand('shutdown') # shutdown the api time.sleep(0.5) diff --git a/onionr/etc/onionrvalues.py b/onionr/etc/onionrvalues.py index 84203d88..338dc0d3 100755 --- a/onionr/etc/onionrvalues.py +++ b/onionr/etc/onionrvalues.py @@ -21,4 +21,5 @@ class OnionrValues: def __init__(self): self.passwordLength = 20 - self.blockMetadataLengths = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'powRandomToken': 1000, 'encryptType': 4, 'expire': 14} #TODO properly refine values to minimum needed \ No newline at end of file + self.blockMetadataLengths = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'powRandomToken': 1000, 'encryptType': 4, 'expire': 14} #TODO properly refine values to minimum needed + self.default_expire = 2678400 \ No newline at end of file diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index b9ae61c4..ea56b593 100755 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -25,7 +25,7 @@ import onionrexceptions from onionr import API_VERSION import onionrevents import storagecounter -from etc import pgpwords +from etc import pgpwords, onionrvalues from onionrusers import onionrusers if sys.version_info < (3, 6): try: @@ -179,8 +179,8 @@ class OnionrUtils: expireTime = myBlock.getHeader('expire') assert len(str(int(expireTime))) < 20 # test that expire time is an integer of sane length (for epoch) except (AssertionError, ValueError, TypeError) as e: - pass - else: + expireTime = onionrvalues.OnionrValues().default_expire + finally: self._core.updateBlockInfo(blockHash, 'expire', expireTime) if not blockType is None: self._core.updateBlockInfo(blockHash, 'dataType', blockType) @@ -253,7 +253,7 @@ class OnionrUtils: pass # Validate metadata dict for invalid keys to sizes that are too large - maxAge = config.get("general.max_block_age", 2678400) + maxAge = config.get("general.max_block_age", onionrvalues.OnionrValues().default_expire) if type(metadata) is dict: for i in metadata: try: @@ -285,7 +285,7 @@ class OnionrUtils: try: assert int(metadata[i]) > self.getEpoch() except AssertionError: - logger.warn('Block is expired: %s greater than %s' % (metadata[i], self.getEpoch())) + logger.warn('Block is expired: %s less than %s' % (metadata[i], self.getEpoch())) break elif i == 'encryptType': try: diff --git a/onionr/static-data/bootstrap-nodes.txt b/onionr/static-data/bootstrap-nodes.txt index e69de29b..374ea7b4 100755 --- a/onionr/static-data/bootstrap-nodes.txt +++ b/onionr/static-data/bootstrap-nodes.txt @@ -0,0 +1 @@ +huei7fugyuesltkf3yzj27an36skz2f2u55g7ganyaq6yjhauwicwfyd.onion diff --git a/onionr/static-data/default-plugins/clandestine/main.py b/onionr/static-data/default-plugins/clandestine/main.py index bc7ca112..93bfd2d3 100755 --- a/onionr/static-data/default-plugins/clandestine/main.py +++ b/onionr/static-data/default-plugins/clandestine/main.py @@ -19,7 +19,7 @@ ''' # Imports some useful libraries -import locale +import locale, sys, os locale.setlocale(locale.LC_ALL, '') From 815f1b1d4ed43cb01bb2a19633473f6a8dd9c9a7 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 29 Mar 2019 12:37:51 -0500 Subject: [PATCH 11/19] work on clandestine --- onionr/communicatorutils/onionrdaemontools.py | 2 +- onionr/communicatorutils/servicecreator.py | 1 + onionr/core.py | 6 +-- onionr/etc/onionrvalues.py | 2 +- onionr/httpapi/__init__.py | 4 +- onionr/onionrcommands/pubkeymanager.py | 4 +- onionr/onionrservices/connectionserver.py | 8 +++- .../default-plugins/clandestine/controlapi.py | 2 +- .../default-plugins/clandestine/main.py | 9 ++-- .../default-plugins/clandestine/peerserver.py | 42 +++++++++++++++++++ 10 files changed, 64 insertions(+), 16 deletions(-) create mode 100644 onionr/static-data/default-plugins/clandestine/peerserver.py diff --git a/onionr/communicatorutils/onionrdaemontools.py b/onionr/communicatorutils/onionrdaemontools.py index b68c693d..1c4e0220 100755 --- a/onionr/communicatorutils/onionrdaemontools.py +++ b/onionr/communicatorutils/onionrdaemontools.py @@ -98,7 +98,7 @@ class DaemonTools: if len(self.daemon.onlinePeers) == 0: if not netutils.checkNetwork(self.daemon._core._utils, torPort=self.daemon.proxyPort): if not self.daemon.shutdown: - logger.warn('Network check failed, are you connected to the internet?') + logger.warn('Network check failed, are you connected to the Internet, and is Tor working?') self.daemon.isOnline = False else: self.daemon.isOnline = True diff --git a/onionr/communicatorutils/servicecreator.py b/onionr/communicatorutils/servicecreator.py index 4914547a..ecd1543c 100644 --- a/onionr/communicatorutils/servicecreator.py +++ b/onionr/communicatorutils/servicecreator.py @@ -12,6 +12,7 @@ def service_creator(daemon): bs = utils.bytesToStr(bl.bcontent) + '.onion' if utils.validatePubKey(bl.signer) and utils.validateID(bs): daemon.active_services.append(b) + daemon.active_services.append(bl.signer) daemon.services.create_server(bl.signer, bs) daemon.decrementThreadCount('service_creator') \ No newline at end of file diff --git a/onionr/core.py b/onionr/core.py index 0c8e1527..09b589f0 100755 --- a/onionr/core.py +++ b/onionr/core.py @@ -796,11 +796,11 @@ class Core: retData = False else: # Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult - self._utils.localCommand('/waitforshare/' + retData, post=True) + if self._utils.localCommand('/ping', maxWait=10) == 'pong!': + self._utils.localCommand('/waitforshare/' + retData, post=True, maxWait=5) + self.daemonQueueAdd('uploadBlock', retData) self.addToBlockDB(retData, selfInsert=True, dataSaved=True) - #self.setBlockType(retData, meta['type']) self._utils.processBlockMetadata(retData) - self.daemonQueueAdd('uploadBlock', retData) if retData != False: if plaintextPeer == 'OVPCZLOXD6DC5JHX4EQ3PSOGAZ3T24F75HQLIUZSDSMYPEOXCPFA====': diff --git a/onionr/etc/onionrvalues.py b/onionr/etc/onionrvalues.py index 338dc0d3..8627d246 100755 --- a/onionr/etc/onionrvalues.py +++ b/onionr/etc/onionrvalues.py @@ -22,4 +22,4 @@ class OnionrValues: def __init__(self): self.passwordLength = 20 self.blockMetadataLengths = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'powRandomToken': 1000, 'encryptType': 4, 'expire': 14} #TODO properly refine values to minimum needed - self.default_expire = 2678400 \ No newline at end of file + self.default_expire = 2592000 \ No newline at end of file diff --git a/onionr/httpapi/__init__.py b/onionr/httpapi/__init__.py index 018d3abb..f38d80bd 100755 --- a/onionr/httpapi/__init__.py +++ b/onionr/httpapi/__init__.py @@ -19,11 +19,11 @@ ''' import onionrplugins -def load_plugin_blueprints(flaskapp): +def load_plugin_blueprints(flaskapp, blueprint='flask_blueprint'): '''Iterate enabled plugins and load any http endpoints they have''' for plugin in onionrplugins.get_enabled_plugins(): plugin = onionrplugins.get_plugin(plugin) try: - flaskapp.register_blueprint(getattr(plugin, 'flask_blueprint')) + flaskapp.register_blueprint(getattr(plugin, blueprint)) except AttributeError: pass \ No newline at end of file diff --git a/onionr/onionrcommands/pubkeymanager.py b/onionr/onionrcommands/pubkeymanager.py index ea55a3fe..00d17175 100644 --- a/onionr/onionrcommands/pubkeymanager.py +++ b/onionr/onionrcommands/pubkeymanager.py @@ -19,7 +19,7 @@ ''' import sys, getpass -import logger +import logger, onionrexceptions from onionrusers import onionrusers def add_ID(o_inst): try: @@ -38,7 +38,7 @@ def add_ID(o_inst): logger.info('Generating deterministic key. This can take a while.') newID, privKey = o_inst.onionrCore._crypto.generateDeterministic(pass1) except onionrexceptions.PasswordStrengthError: - logger.error('Must use at least 25 characters.') + logger.error('Passphrase must use at least %s characters.' % (o_inst.onionrCore._crypto.deterministicRequirement,)) sys.exit(1) else: logger.error('Passwords do not match.') diff --git a/onionr/onionrservices/connectionserver.py b/onionr/onionrservices/connectionserver.py index ee586e2b..ad34cae1 100644 --- a/onionr/onionrservices/connectionserver.py +++ b/onionr/onionrservices/connectionserver.py @@ -21,7 +21,7 @@ import threading, time from gevent.pywsgi import WSGIServer, WSGIHandler from stem.control import Controller from flask import Flask -import core, logger +import core, logger, httpapi from netcontroller import getOpenPort import api from . import httpheaders @@ -44,7 +44,9 @@ class ConnectionServer: core_inst.onionrInst.communicatorInst.service_greenlets.append(http_server) # TODO define basic endpoints useful for direct connections like stats - # TODO load endpoints from plugins + + httpapi.load_plugin_blueprints(service_app, blueprint='direct_blueprint') + @service_app.route('/ping') def get_ping(): return "pong!" @@ -60,7 +62,9 @@ class ConnectionServer: controller.authenticate(core_inst.config.get('tor.controlpassword')) # Create the v3 onion service response = controller.create_ephemeral_hidden_service({80: service_port}, await_publication = True, key_type='NEW', key_content = 'ED25519-V3') + self.core_inst.keyStore.put('dc-' + response.service_id, self.core_inst._utils.bytesToStr(peer)) self.core_inst._utils.doPostRequest('http://' + address + '/bs/' + response.service_id, port=socks) logger.info('hosting on %s with %s' % (response.service_id, peer)) http_server.serve_forever() + self.core_inst.keyStore.delete('dc-' + response.service_id) http_server.stop() \ No newline at end of file diff --git a/onionr/static-data/default-plugins/clandestine/controlapi.py b/onionr/static-data/default-plugins/clandestine/controlapi.py index ce341f82..5631a2a4 100644 --- a/onionr/static-data/default-plugins/clandestine/controlapi.py +++ b/onionr/static-data/default-plugins/clandestine/controlapi.py @@ -21,6 +21,6 @@ from flask import Response, request, redirect, Blueprint, abort flask_blueprint = Blueprint('clandenstine', __name__) -@flask_blueprint.route('/mail/ping') +@flask_blueprint.route('/clandenstine/ping') def ping(): return 'pong!' diff --git a/onionr/static-data/default-plugins/clandestine/main.py b/onionr/static-data/default-plugins/clandestine/main.py index 93bfd2d3..9a1e75a2 100755 --- a/onionr/static-data/default-plugins/clandestine/main.py +++ b/onionr/static-data/default-plugins/clandestine/main.py @@ -23,14 +23,15 @@ import locale, sys, os locale.setlocale(locale.LC_ALL, '') -plugin_name = 'clandenstine' +plugin_name = 'clandestine' PLUGIN_VERSION = '0.0.0' sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) -from . import controlapi +from . import controlapi, peerserver flask_blueprint = controlapi.flask_blueprint +direct_blueprint = peerserver.direct_blueprint -class Clandenstine: +class Clandestine: def __init__(self, pluginapi): self.myCore = pluginapi.get_core() @@ -42,5 +43,5 @@ def on_init(api, data = None): ''' pluginapi = api - chat = Clandenstine(pluginapi) + chat = Clandestine(pluginapi) return diff --git a/onionr/static-data/default-plugins/clandestine/peerserver.py b/onionr/static-data/default-plugins/clandestine/peerserver.py new file mode 100644 index 00000000..e10fa016 --- /dev/null +++ b/onionr/static-data/default-plugins/clandestine/peerserver.py @@ -0,0 +1,42 @@ +''' + Onionr - P2P Anonymous Storage Network + + HTTP endpoints for communicating with peers +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import core +from flask import Response, request, redirect, Blueprint, abort, g + +direct_blueprint = Blueprint('clandestine', __name__) +core_inst = core.Core() + +storage_dir = core_inst.dataDir + +@direct_blueprint.before_request +def request_setup(): + core_inst.keyStore.refresh() + host = request.host + host = host.strip('.b32.i2p') + host = host.strip('.onion') + g.host = host + g.peer = core_inst.keyStore.get('dc-' + g.host) + +@direct_blueprint.route('/clandestine/ping') +def pingdirect(): + return 'pong!' + g.peer + +@direct_blueprint.route('/clandestine/send') +def poll_chat \ No newline at end of file From 063330cfee16554a872a7dd9cf51c043b46aad31 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 30 Mar 2019 23:50:54 -0500 Subject: [PATCH 12/19] basic cs messages supported --- .../default-plugins/clandestine/controlapi.py | 51 +++++++++++++++++-- .../default-plugins/clandestine/peerserver.py | 21 ++++++-- 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/onionr/static-data/default-plugins/clandestine/controlapi.py b/onionr/static-data/default-plugins/clandestine/controlapi.py index 5631a2a4..f203c0af 100644 --- a/onionr/static-data/default-plugins/clandestine/controlapi.py +++ b/onionr/static-data/default-plugins/clandestine/controlapi.py @@ -17,10 +17,55 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -from flask import Response, request, redirect, Blueprint, abort +import json +from flask import Response, request, redirect, Blueprint, g +import core -flask_blueprint = Blueprint('clandenstine', __name__) +core_inst = core.Core() +flask_blueprint = Blueprint('clandestine_control', __name__) -@flask_blueprint.route('/clandenstine/ping') +@flask_blueprint.route('/clandestine/ping') def ping(): return 'pong!' + +@flask_blueprint.route('/clandestine/send/', methods=['POST']) +def send_message(peer): + data = request.get_json(force=True) + core_inst.keyStore.refresh() + existing = core_inst.keyStore.get('s' + peer) + if existing is None: + existing = [] + existing.append(data) + core_inst.keyStore.put('s' + peer, existing) + core_inst.keyStore.flush() + return Response('success') + +@flask_blueprint.route('/clandestine/gets/') +def get_sent(peer): + sent = core_inst.keyStore.get('s' + peer) + if sent is None: + sent = [] + return Response(json.dumps(sent)) + +@flask_blueprint.route('/clandestine/addrec/', methods=['POST']) +def add_rec(peer): + data = request.get_json(force=True) + core_inst.keyStore.refresh() + existing = core_inst.keyStore.get('r' + peer) + if existing is None: + existing = [] + existing.append(data) + core_inst.keyStore.put('r' + peer, existing) + core_inst.keyStore.flush() + return Response('success') + +@flask_blueprint.route('/clandestine/getrec/') +def get_messages(peer): + core_inst.keyStore.refresh() + existing = core_inst.keyStore.get('r' + peer) + if existing is None: + existing = [] + else: + existing = list(existing) + core_inst.keyStore.delete('r' + peer) + return Response(json.dumps(existing)) \ No newline at end of file diff --git a/onionr/static-data/default-plugins/clandestine/peerserver.py b/onionr/static-data/default-plugins/clandestine/peerserver.py index e10fa016..b5fb3913 100644 --- a/onionr/static-data/default-plugins/clandestine/peerserver.py +++ b/onionr/static-data/default-plugins/clandestine/peerserver.py @@ -17,9 +17,10 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' +import sys, os, json import core from flask import Response, request, redirect, Blueprint, abort, g - +sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) direct_blueprint = Blueprint('clandestine', __name__) core_inst = core.Core() @@ -36,7 +37,19 @@ def request_setup(): @direct_blueprint.route('/clandestine/ping') def pingdirect(): - return 'pong!' + g.peer + return 'pong!' -@direct_blueprint.route('/clandestine/send') -def poll_chat \ No newline at end of file +@direct_blueprint.route('/clandestine/sendto', methods=['POST', 'GET']) +def sendto(): + try: + msg = request.get_json(force=True) + except: + msg = '' + if msg == None or msg == '': + msg = json.dumps({'m': 'hello world', 't': core_inst._utils.getEpoch()}) + core_inst._utils.localCommand('/clandestine/addrec/%s' % (g.peer,), post=True, postData=msg) + return Response('success') + +@direct_blueprint.route('/clandestine/poll') +def poll_chat(): + return Response(core_inst._utils.localCommand('/clandestine/gets/%s' % (g.peer,))) \ No newline at end of file From faf42071eb38f67420697e856c2eb16333dc00b7 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 31 Mar 2019 12:16:09 -0500 Subject: [PATCH 13/19] plugin example of connecting to dir conn --- onionr/onionrservices/bootstrapservice.py | 4 ++-- onionr/onionrservices/connectionserver.py | 6 +++++ .../default-plugins/clandestine/main.py | 24 ++++++++++++++++++- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/onionr/onionrservices/bootstrapservice.py b/onionr/onionrservices/bootstrapservice.py index a2952563..160b716d 100644 --- a/onionr/onionrservices/bootstrapservice.py +++ b/onionr/onionrservices/bootstrapservice.py @@ -39,8 +39,8 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300): bootstrap_app = Flask(__name__) http_server = WSGIServer(('127.0.0.1', bootstrap_port), bootstrap_app, log=None) try: - core_inst.onionrInst.communicatorInst - except AttributeError: + assert core_inst.onionrInst.communicatorInst is not None + except (AttributeError, AssertionError) as e: pass else: core_inst.onionrInst.communicatorInst.service_greenlets.append(http_server) diff --git a/onionr/onionrservices/connectionserver.py b/onionr/onionrservices/connectionserver.py index ad34cae1..dea9f393 100644 --- a/onionr/onionrservices/connectionserver.py +++ b/onionr/onionrservices/connectionserver.py @@ -50,6 +50,12 @@ class ConnectionServer: @service_app.route('/ping') def get_ping(): return "pong!" + + @service_app.route('/shutdown') + def shutdown_server(): + core_inst.onionrInst.communicatorInst.service_greenlets.remove(http_server) + http_server.stop() + return Response('goodbye') @service_app.after_request def afterReq(resp): diff --git a/onionr/static-data/default-plugins/clandestine/main.py b/onionr/static-data/default-plugins/clandestine/main.py index 9a1e75a2..3fab2dd0 100755 --- a/onionr/static-data/default-plugins/clandestine/main.py +++ b/onionr/static-data/default-plugins/clandestine/main.py @@ -20,8 +20,9 @@ # Imports some useful libraries import locale, sys, os - locale.setlocale(locale.LC_ALL, '') +import onionrservices, logger +from onionrservices import bootstrapservice plugin_name = 'clandestine' PLUGIN_VERSION = '0.0.0' @@ -31,9 +32,29 @@ from . import controlapi, peerserver flask_blueprint = controlapi.flask_blueprint direct_blueprint = peerserver.direct_blueprint +def exit_with_error(text=''): + if text != '': + logger.error(text) + sys.exit(1) + class Clandestine: def __init__(self, pluginapi): self.myCore = pluginapi.get_core() + + def create(self): + try: + peer = sys.argv[2] + if not self.myCore._utils.validatePubKey(peer): + exit_with_error('Invalid public key specified') + except IndexError: + exit_with_error('You must specify a peer public key') + + # Ask peer for transport address by creating block for them + peer_transport_address = bootstrapservice.bootstrap_client_service(peer, self.myCore) + + print(peer_transport_address) + if self.myCore._utils.doGetRequest('http://%s/ping' % (peer_transport_address,), ignoreAPI=True, port=self.myCore.config.get('tor.socksport')) == 'pong!': + print('connected', peer_transport_address) def on_init(api, data = None): ''' @@ -44,4 +65,5 @@ def on_init(api, data = None): pluginapi = api chat = Clandestine(pluginapi) + api.commands.register(['clandestine'], chat.create) return From 7c57d95d1d07cd28ac69264043e2b567d78e9f20 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 2 Apr 2019 11:50:09 -0500 Subject: [PATCH 14/19] work on direct connections and clandestine --- onionr/api.py | 10 ++++++ onionr/communicatorutils/servicecreator.py | 5 +-- onionr/onionrservices/connectionserver.py | 2 +- .../default-plugins/clandestine/main.py | 34 ++++++++++++++++--- .../default-plugins/clandestine/peerserver.py | 2 ++ 5 files changed, 46 insertions(+), 7 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index 7e552355..3df4741b 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -318,6 +318,16 @@ class API: @app.route('/friends/', endpoint='friendsindex') def loadContacts(): return send_from_directory('static-data/www/friends/', 'index.html') + + @app.route('/serviceactive/') + def serviceActive(pubkey): + try: + if pubkey in self._core.onionrInst.communicatorInst.active_services: + return Response('true') + except AttributeError as e: + print('attribute error', str(e)) + pass + return Response('false') @app.route('/board/', endpoint='boardContent') def boardContent(path): diff --git a/onionr/communicatorutils/servicecreator.py b/onionr/communicatorutils/servicecreator.py index ecd1543c..fbb4c5b9 100644 --- a/onionr/communicatorutils/servicecreator.py +++ b/onionr/communicatorutils/servicecreator.py @@ -11,8 +11,9 @@ def service_creator(daemon): bl = onionrblockapi.Block(b, core=core, decrypt=True) bs = utils.bytesToStr(bl.bcontent) + '.onion' if utils.validatePubKey(bl.signer) and utils.validateID(bs): + signer = utils.bytesToStr(bl.signer) daemon.active_services.append(b) - daemon.active_services.append(bl.signer) - daemon.services.create_server(bl.signer, bs) + daemon.active_services.append(signer) + daemon.services.create_server(signer, bs) daemon.decrementThreadCount('service_creator') \ No newline at end of file diff --git a/onionr/onionrservices/connectionserver.py b/onionr/onionrservices/connectionserver.py index dea9f393..27a0b55f 100644 --- a/onionr/onionrservices/connectionserver.py +++ b/onionr/onionrservices/connectionserver.py @@ -70,7 +70,7 @@ class ConnectionServer: response = controller.create_ephemeral_hidden_service({80: service_port}, await_publication = True, key_type='NEW', key_content = 'ED25519-V3') self.core_inst.keyStore.put('dc-' + response.service_id, self.core_inst._utils.bytesToStr(peer)) self.core_inst._utils.doPostRequest('http://' + address + '/bs/' + response.service_id, port=socks) - logger.info('hosting on %s with %s' % (response.service_id, peer)) + logger.info('hosting on %s with %s' % (response.service_id, peer)) http_server.serve_forever() self.core_inst.keyStore.delete('dc-' + response.service_id) http_server.stop() \ No newline at end of file diff --git a/onionr/static-data/default-plugins/clandestine/main.py b/onionr/static-data/default-plugins/clandestine/main.py index 3fab2dd0..8f202dac 100755 --- a/onionr/static-data/default-plugins/clandestine/main.py +++ b/onionr/static-data/default-plugins/clandestine/main.py @@ -19,7 +19,7 @@ ''' # Imports some useful libraries -import locale, sys, os +import locale, sys, os, threading, json locale.setlocale(locale.LC_ALL, '') import onionrservices, logger from onionrservices import bootstrapservice @@ -40,7 +40,30 @@ def exit_with_error(text=''): class Clandestine: def __init__(self, pluginapi): self.myCore = pluginapi.get_core() + self.peer = None + self.transport = None + self.shutdown = False + def _sender_loop(self): + print('Enter a message to send, with ctrl-d or -s on a new line.') + print('-c on a new line or ctrl-c stops') + message = '' + while not self.shutdown: + try: + message += input() + if message == '-s': + raise EOFError + elif message == '-c': + raise KeyboardInterrupt + else: + message += '\n' + except EOFError: + message = json.dumps({'m': message, 't': self.myCore._utils.getEpoch()}) + print(self.myCore._utils.doPostRequest('http://%s/clandestine/sendto' % (self.transport,), port=self.socks, data=message)) + message = '' + except KeyboardInterrupt: + self.shutdown = True + def create(self): try: peer = sys.argv[2] @@ -48,13 +71,16 @@ class Clandestine: exit_with_error('Invalid public key specified') except IndexError: exit_with_error('You must specify a peer public key') - + self.peer = peer # Ask peer for transport address by creating block for them peer_transport_address = bootstrapservice.bootstrap_client_service(peer, self.myCore) + self.transport = peer_transport_address + self.socks = self.myCore.config.get('tor.socksport') - print(peer_transport_address) - if self.myCore._utils.doGetRequest('http://%s/ping' % (peer_transport_address,), ignoreAPI=True, port=self.myCore.config.get('tor.socksport')) == 'pong!': + print('connected with', peer, 'on', peer_transport_address) + if self.myCore._utils.doGetRequest('http://%s/ping' % (peer_transport_address,), ignoreAPI=True, port=self.socks) == 'pong!': print('connected', peer_transport_address) + threading.Thread(target=self._sender_loop).start() def on_init(api, data = None): ''' diff --git a/onionr/static-data/default-plugins/clandestine/peerserver.py b/onionr/static-data/default-plugins/clandestine/peerserver.py index b5fb3913..64e4eaeb 100644 --- a/onionr/static-data/default-plugins/clandestine/peerserver.py +++ b/onionr/static-data/default-plugins/clandestine/peerserver.py @@ -47,6 +47,8 @@ def sendto(): msg = '' if msg == None or msg == '': msg = json.dumps({'m': 'hello world', 't': core_inst._utils.getEpoch()}) + else: + msg = json.dumps(msg) core_inst._utils.localCommand('/clandestine/addrec/%s' % (g.peer,), post=True, postData=msg) return Response('success') From 93b3f998998c1ace7856542d36ffbc46cd56d70e Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 4 Apr 2019 16:56:18 -0500 Subject: [PATCH 15/19] added first block spec draft --- docs/block-spec.md | 64 ++++++++++++++++++++++++++++++++++++++ onionr/api.py | 1 - onionr/etc/onionrvalues.py | 2 +- onionr/onionrproofs.py | 2 +- onionr/subprocesspow.py | 2 +- 5 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 docs/block-spec.md diff --git a/docs/block-spec.md b/docs/block-spec.md new file mode 100644 index 00000000..0a5f1491 --- /dev/null +++ b/docs/block-spec.md @@ -0,0 +1,64 @@ +# Onionr Block Spec v1.0.0 + +# Block Description + +Onionr 'Blocks' are the primary means of sharing information in Onionr. Blocks are identified by a single hash value of their entire contents. + +They contain a JSON metadata section followed by a line break, with the main data following. + +In the future, the spec will be updated to use flags and MessagePack instead of JSON with english keys. + + +# Encryption and Signatures + +Onionr blocks may be encrypted or signed. In the reference client, this is done with libsodium, for both asymmetric and symmetric encryption. + +Unlike many similar projects, blocks may completely be in plaintext, making Onionr suitable for sharing information publicly. + +# Metadata Section + +The metadata section has the following fields. If a block contains any other field, it must be considered invalid. All metadata fields are technically optional, but many are useful and essentially necessary for most use cases. + +## meta + +Max byte size: 1000 + +Meta is a string field which can contain arbitrary sub fields. It is intended for applications and plugins to use it for arbitrary metadata information. In the reference client, if the data section is encrypted or signed, the meta section also is. + +Common meta fields, such as 'type' are used by the reference Onionr client to describe the type of a block. + +## sig + +Max byte size: 200 + +Sig is a field for storing public key signatures of the block, typically ed25519. In the reference client, this field is a base64 encoded signature of the meta field combined with the block data. (**Therefore, information outside of the meta and data fields cannot be trusted to be placed there by the signer, although it can still be assured that the particular block has not been modified.**) + +Note: the max field size is larger than a EdDSA signature (which is what is typically used) in order to allow other primitives for signing in alternative implementations or future versions. + +## signer + +Max byte size: 200 + +Signer is a field for specifying the public key which signed the block. In the reference client this is a base64 encoded ed25519 public key. + +## time + +Max byte size: 10 + +Time is an integer field for specifying the time of which a block was created. The trustworthiness of this field is based on one's trust of the block creator, however blocks with a time field set in the future (past a reasonable clock skew) are thrown out by the reference client. + +## expire + +Max byte size: 10 + +Expire is an integer field for specifying the time of which the block creator has indicated that the block should be deleted. The purpose of this is for voluntarily freeing the burden of unwanted blocks on the Onionr network, rather than security/privacy (since blocks could be trivially kept past expiration). Regardless, the reference client deletes blocks after a preset time if the expire field is either not set or longer than the preset time. + +## pow + +Max byte size: 1000 + +Pow is a field for placing the nonce found to make a block meet a target proof of work. In theory, a block could meet a target without a random token in this field. + +## encryptType + +encryptType is a field to specify the mode of encryption for a block. The values supported by Onionr are 'asym' and 'sym'. diff --git a/onionr/api.py b/onionr/api.py index 3df4741b..bba548c5 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -325,7 +325,6 @@ class API: if pubkey in self._core.onionrInst.communicatorInst.active_services: return Response('true') except AttributeError as e: - print('attribute error', str(e)) pass return Response('false') diff --git a/onionr/etc/onionrvalues.py b/onionr/etc/onionrvalues.py index 8627d246..9f636147 100755 --- a/onionr/etc/onionrvalues.py +++ b/onionr/etc/onionrvalues.py @@ -21,5 +21,5 @@ class OnionrValues: def __init__(self): self.passwordLength = 20 - self.blockMetadataLengths = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'powRandomToken': 1000, 'encryptType': 4, 'expire': 14} #TODO properly refine values to minimum needed + self.blockMetadataLengths = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'pow': 1000, 'encryptType': 4, 'expire': 14} #TODO properly refine values to minimum needed self.default_expire = 2592000 \ No newline at end of file diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py index 62dc215c..53c6d881 100755 --- a/onionr/onionrproofs.py +++ b/onionr/onionrproofs.py @@ -247,7 +247,7 @@ class POW: startNonce = nonce while self.hashing: #token = nacl.hash.blake2b(rand + self.data).decode() - self.metadata['powRandomToken'] = nonce + self.metadata['pow'] = nonce payload = json.dumps(self.metadata).encode() + b'\n' + self.data token = myCore._crypto.sha3Hash(payload) try: diff --git a/onionr/subprocesspow.py b/onionr/subprocesspow.py index a2fee7c6..fee2f2dd 100755 --- a/onionr/subprocesspow.py +++ b/onionr/subprocesspow.py @@ -75,7 +75,7 @@ class SubprocessPOW: difficulty = self.difficulty mcore = core.Core() while True: - metadata['powRandomToken'] = nonce + metadata['pow'] = nonce payload = json.dumps(metadata).encode() + b'\n' + data token = mcore._crypto.sha3Hash(payload) try: From 81cfc64658e2ae361647f15cd276ab45417e3ceb Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 9 Apr 2019 12:30:54 -0500 Subject: [PATCH 16/19] fixes for mail, work on specs, and added files for profile viewer --- README.md | 2 + docs/{api.md => http-api.md} | 0 docs/{ => specs}/block-spec.md | 3 +- onionr/api.py | 14 +++++- onionr/httpapi/profilesapi/__init__.py | 28 ++++++++++++ onionr/httpapi/profilesapi/profiles.py | 2 + onionr/static-data/bootstrap-nodes.txt | 2 +- .../default-plugins/pms/mailapi.py | 25 ++++++++--- onionr/static-data/default_config.json | 4 +- onionr/static-data/www/mail/index.html | 5 ++- onionr/static-data/www/mail/mail.css | 2 +- onionr/static-data/www/mail/mail.js | 44 +++++++++++-------- onionr/static-data/www/profiles/index.html | 20 +++++++++ onionr/static-data/www/profiles/profiles.js | 0 14 files changed, 119 insertions(+), 32 deletions(-) rename docs/{api.md => http-api.md} (100%) rename docs/{ => specs}/block-spec.md (96%) create mode 100644 onionr/httpapi/profilesapi/__init__.py create mode 100644 onionr/httpapi/profilesapi/profiles.py create mode 100644 onionr/static-data/www/profiles/index.html create mode 100644 onionr/static-data/www/profiles/profiles.js diff --git a/README.md b/README.md index 8b0de5c2..ebae2e10 100755 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ Users are identified by ed25519 public keys, which can be used to sign blocks or Onionr can be used for mail, as a social network, instant messenger, file sharing software, or for encrypted group discussion. +The whitepaper (subject to change prior to first alpha release) is available [here](docs/whitepaper.md). + ![Tor stinks slide image](docs/tor-stinks-02.png) ## Main Features diff --git a/docs/api.md b/docs/http-api.md similarity index 100% rename from docs/api.md rename to docs/http-api.md diff --git a/docs/block-spec.md b/docs/specs/block-spec.md similarity index 96% rename from docs/block-spec.md rename to docs/specs/block-spec.md index 0a5f1491..f9cd3509 100644 --- a/docs/block-spec.md +++ b/docs/specs/block-spec.md @@ -6,8 +6,7 @@ Onionr 'Blocks' are the primary means of sharing information in Onionr. Blocks a They contain a JSON metadata section followed by a line break, with the main data following. -In the future, the spec will be updated to use flags and MessagePack instead of JSON with english keys. - +In the future, the specification will likely be updated to use flags and MessagePack instead of JSON with english keys. # Encryption and Signatures diff --git a/onionr/api.py b/onionr/api.py index bba548c5..415707e5 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -26,7 +26,7 @@ import core from onionrblockapi import Block import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config import httpapi -from httpapi import friendsapi, simplecache +from httpapi import friendsapi, simplecache, profilesapi from onionrservices import httpheaders import onionr @@ -256,7 +256,8 @@ class API: self.bindPort = bindPort # Be extremely mindful of this. These are endpoints available without a password - self.whitelistEndpoints = ('site', 'www', 'onionrhome', 'board', 'boardContent', 'sharedContent', 'mail', 'mailindex', 'friends', 'friendsindex') + self.whitelistEndpoints = ('site', 'www', 'onionrhome', 'board', 'profiles', 'profilesindex', + 'boardContent', 'sharedContent', 'mail', 'mailindex', 'friends', 'friendsindex') self.clientToken = config.get('client.webpassword') self.timeBypassToken = base64.b16encode(os.urandom(32)).decode() @@ -272,6 +273,7 @@ class API: onionrInst.setClientAPIInst(self) app.register_blueprint(friendsapi.friends) app.register_blueprint(simplecache.simplecache) + app.register_blueprint(profilesapi.profile_BP) httpapi.load_plugin_blueprints(app) @app.before_request @@ -318,6 +320,14 @@ class API: @app.route('/friends/', endpoint='friendsindex') def loadContacts(): return send_from_directory('static-data/www/friends/', 'index.html') + + @app.route('/profiles/', endpoint='profiles') + def loadContacts(path): + return send_from_directory('static-data/www/profiles/', path) + + @app.route('/profiles/', endpoint='profilesindex') + def loadContacts(): + return send_from_directory('static-data/www/profiles/', 'index.html') @app.route('/serviceactive/') def serviceActive(pubkey): diff --git a/onionr/httpapi/profilesapi/__init__.py b/onionr/httpapi/profilesapi/__init__.py new file mode 100644 index 00000000..f3c3ba2f --- /dev/null +++ b/onionr/httpapi/profilesapi/__init__.py @@ -0,0 +1,28 @@ +''' + Onionr - P2P Anonymous Storage Network + + This file creates http endpoints for user profile pages +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import core +from flask import Blueprint, Response, request, abort +from . import profiles + +profile_BP = Blueprint('profile_BP', __name__) + +@profile_BP.route('/profile/get/', endpoint='profiles') +def get_profile_page(pubkey): + return Response(pubkey) \ No newline at end of file diff --git a/onionr/httpapi/profilesapi/profiles.py b/onionr/httpapi/profilesapi/profiles.py new file mode 100644 index 00000000..11a73b13 --- /dev/null +++ b/onionr/httpapi/profilesapi/profiles.py @@ -0,0 +1,2 @@ +def get_latest_user_profile(pubkey): + return '' \ No newline at end of file diff --git a/onionr/static-data/bootstrap-nodes.txt b/onionr/static-data/bootstrap-nodes.txt index 374ea7b4..52622b12 100755 --- a/onionr/static-data/bootstrap-nodes.txt +++ b/onionr/static-data/bootstrap-nodes.txt @@ -1 +1 @@ -huei7fugyuesltkf3yzj27an36skz2f2u55g7ganyaq6yjhauwicwfyd.onion +ecaufnpxx67xmvzwcriohapmmnqbj665h3ynemvwrr2juxl5oa5g7cad.onion diff --git a/onionr/static-data/default-plugins/pms/mailapi.py b/onionr/static-data/default-plugins/pms/mailapi.py index 87cdb678..0a60e67c 100644 --- a/onionr/static-data/default-plugins/pms/mailapi.py +++ b/onionr/static-data/default-plugins/pms/mailapi.py @@ -42,6 +42,7 @@ def mail_delete(block): if block not in existing: existing.append(block) kv.put('deleted_mail', existing) + kv.flush() return 'success' @flask_blueprint.route('/mail/getinbox') @@ -50,16 +51,30 @@ def list_inbox(): @flask_blueprint.route('/mail/getsentbox') def list_sentbox(): + kv.refresh() sentbox_list = sentboxdb.SentBox(c).listSent() - sentbox_list_copy = list(sentbox_list) + list_copy = list(sentbox_list) deleted = kv.get('deleted_mail') if deleted is None: deleted = [] - for x in range(len(sentbox_list_copy) - 1): + for x in list_copy: + if x['hash'] in deleted: + sentbox_list.remove(x) + return json.dumps(sentbox_list) +''' +@flask_blueprint.route('/mail/getsentbox') +def list_sentbox(): + sentbox_list = sentboxdb.SentBox(c).listSent() + sentbox_list_copy = list(sentbox_list) + kv.refresh() + deleted = kv.get('deleted_mail') + if deleted is None: + deleted = [] + for x in range(len(sentbox_list_copy)): if sentbox_list_copy[x]['hash'] in deleted: - x -= 1 - sentbox_list.pop(x) + sentbox_list.remove(sentbox_list_copy[x]['hash']) else: sentbox_list[x]['name'] = contactmanager.ContactManager(c, sentbox_list_copy[x]['peer'], saveUser=False).get_info('name') - return json.dumps(sentbox_list) \ No newline at end of file + return json.dumps(sentbox_list) +''' \ No newline at end of file diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 2b9d47df..988e91f0 100755 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -4,12 +4,12 @@ "display_header" : false, "minimum_block_pow": 4, "minimum_send_pow": 4, - "socket_servers": true, + "socket_servers": false, "security_level": 0, "max_block_age": 2678400, "bypass_tor_check": false, "public_key": "", - "random_bind_ip": true + "random_bind_ip": false }, "www" : { diff --git a/onionr/static-data/www/mail/index.html b/onionr/static-data/www/mail/index.html index 8af6f035..f3f337bd 100644 --- a/onionr/static-data/www/mail/index.html +++ b/onionr/static-data/www/mail/index.html @@ -36,8 +36,11 @@
From: Signature:
+
+ Subject: +
- +
diff --git a/onionr/static-data/www/mail/mail.css b/onionr/static-data/www/mail/mail.css index af80deb1..88afc0b9 100644 --- a/onionr/static-data/www/mail/mail.css +++ b/onionr/static-data/www/mail/mail.css @@ -92,7 +92,7 @@ input{ color: black; } -#replyBtn{ +.break-up{ margin-top: 1em; } diff --git a/onionr/static-data/www/mail/mail.js b/onionr/static-data/www/mail/mail.js index 302c84f6..cfbb7e89 100644 --- a/onionr/static-data/www/mail/mail.js +++ b/onionr/static-data/www/mail/mail.js @@ -26,7 +26,7 @@ threadContent = {} myPub = httpGet('/getActivePubkey') replyBtn = document.getElementById('replyBtn') -function openReply(bHash){ +function openReply(bHash, quote, subject){ var inbox = document.getElementsByClassName('threadEntry') var entry = '' var friendName = '' @@ -41,13 +41,23 @@ function openReply(bHash){ } key = entry.getAttribute('data-pubkey') document.getElementById('draftID').value = key + document.getElementById('draftSubject').value = 'RE: ' + subject + + // Add quoted reply + var splitQuotes = quote.split('\n') + for (var x = 0; x < splitQuotes.length; x++){ + splitQuotes[x] = '>' + splitQuotes[x] + } + quote = splitQuotes.join('\n') + document.getElementById('draftText').value = quote setActiveTab('send message') } -function openThread(bHash, sender, date, sigBool, pubkey){ +function openThread(bHash, sender, date, sigBool, pubkey, subjectLine){ var messageDisplay = document.getElementById('threadDisplay') var blockContent = httpGet('/getblockbody/' + bHash) document.getElementById('fromUser').value = sender + document.getElementById('subjectView').innerText = subjectLine messageDisplay.innerText = blockContent var sigEl = document.getElementById('sigValid') var sigMsg = 'signature' @@ -64,7 +74,7 @@ function openThread(bHash, sender, date, sigBool, pubkey){ sigEl.innerText = sigMsg overlay('messageDisplay') replyBtn.onclick = function(){ - openReply(bHash) + openReply(bHash, messageDisplay.innerText, subjectLine) } } @@ -174,7 +184,7 @@ function loadInboxEntries(bHash){ if (event.target.classList.contains('deleteBtn')){ return } - openThread(entry.getAttribute('data-hash'), senderInput.value, dateStr.innerText, resp['meta']['validSig'], entry.getAttribute('data-pubkey')) + openThread(entry.getAttribute('data-hash'), senderInput.value, dateStr.innerText, resp['meta']['validSig'], entry.getAttribute('data-pubkey'), subjectLine.innerText) } deleteBtn.onclick = function(){ @@ -216,7 +226,7 @@ function getSentbox(){ if (keys.length == 0){ threadPart.innerHTML = "nothing to show here yet." } - for (var i = 0; i < keys.length; i++){ + for (var i = 0; i < keys.length; i++) (function(i, resp){ var entry = document.createElement('div') var obj = resp[i] var toLabel = document.createElement('span') @@ -245,19 +255,18 @@ function getSentbox(){ entry.appendChild(toEl) entry.appendChild(preview) entry.appendChild(sentDate) - entry.onclick = (function(tree, el, msg) {return function() { - console.log(resp) - if (! entry.classList.contains('deleteBtn')){ - showSentboxWindow(el.value, msg) - } - };})(entry, toEl, message); - - deleteBtn.onclick = function(){ - entry.parentNode.removeChild(entry); - deleteMessage(entry.getAttribute('data-hash')) - } + threadPart.appendChild(entry) - } + + entry.onclick = function(e){ + if (e.target.classList.contains('deleteBtn')){ + deleteMessage(e.target.parentNode.getAttribute('data-hash')) + e.target.parentNode.parentNode.removeChild(e.target.parentNode) + return + } + showSentboxWindow() + } + })(i, resp) threadPart.appendChild(entry) }.bind(threadPart)) } @@ -304,7 +313,6 @@ for (var i = 0; i < document.getElementsByClassName('refresh').length; i++){ document.getElementsByClassName('refresh')[i].style.float = 'right' } - fetch('/friends/list', { headers: { "token": webpass diff --git a/onionr/static-data/www/profiles/index.html b/onionr/static-data/www/profiles/index.html new file mode 100644 index 00000000..a4dad9b7 --- /dev/null +++ b/onionr/static-data/www/profiles/index.html @@ -0,0 +1,20 @@ + + + + + + Onionr Profiles + + + + + + +
+

+
+ + + + + \ No newline at end of file diff --git a/onionr/static-data/www/profiles/profiles.js b/onionr/static-data/www/profiles/profiles.js new file mode 100644 index 00000000..e69de29b From fb15061ff7c172883884486e2e55d54e290983ad Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 9 Apr 2019 12:35:40 -0500 Subject: [PATCH 17/19] fixes for mail, work on specs, and added files for profile viewer --- onionr/communicator.py | 2 +- onionr/static-data/www/mail/mail.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/onionr/communicator.py b/onionr/communicator.py index ef94ae14..f10d37da 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -621,7 +621,7 @@ class OnionrCommunicatorDaemon: def detectAPICrash(self): '''exit if the api server crashes/stops''' if self._core._utils.localCommand('ping', silent=False) not in ('pong', 'pong!'): - for i in range(8): + for i in range(12): if self._core._utils.localCommand('ping') in ('pong', 'pong!') or self.shutdown: break # break for loop time.sleep(1) diff --git a/onionr/static-data/www/mail/mail.js b/onionr/static-data/www/mail/mail.js index cfbb7e89..09b8714a 100644 --- a/onionr/static-data/www/mail/mail.js +++ b/onionr/static-data/www/mail/mail.js @@ -264,7 +264,7 @@ function getSentbox(){ e.target.parentNode.parentNode.removeChild(e.target.parentNode) return } - showSentboxWindow() + showSentboxWindow(resp[i]['peer'], message) } })(i, resp) threadPart.appendChild(entry) From 50eb095039f16529c335e8b04184dc04418a6cd8 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 11 Apr 2019 17:01:18 -0500 Subject: [PATCH 18/19] work on docs --- README.md | 8 +++---- docs/http-api.md | 52 ++++++++++++++++++++++++++++++++++++++-- docs/specs/block-spec.md | 4 ++-- onionr/api.py | 4 +++- 4 files changed, 59 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ebae2e10..58dac596 100755 --- a/README.md +++ b/README.md @@ -17,11 +17,11 @@ **The main repository for this software is at https://gitlab.com/beardog/Onionr/** -# Summary +# About Onionr is a decentralized, peer-to-peer data storage network, designed to be anonymous and resistant to (meta)data analysis and spam/disruption. -Onionr stores data in independent packages referred to as 'blocks'. The blocks are synced to all other nodes in the network. Blocks and user IDs cannot be easily proven to have been created by particular nodes (only inferred). Even if there is enough evidence to believe a particular node created a block, nodes still operate behind Tor or I2P and as such are not trivially known to be at a particular IP address. +Onionr stores data in independent packages referred to as 'blocks'. The blocks are synced to all other nodes in the network. Blocks and user IDs cannot be easily proven to have been created by a particular user. Even if there is enough evidence to believe that a specific user created a block, nodes still operate behind Tor or I2P and as such cannot be trivially unmasked. Users are identified by ed25519 public keys, which can be used to sign blocks or send encrypted data. @@ -36,7 +36,7 @@ The whitepaper (subject to change prior to first alpha release) is available [he * [X] 🌐 Fully p2p/decentralized, no trackers or other single points of failure * [X] 🔒 End to end encryption of user data * [X] 📢 Optional non-encrypted blocks, useful for blog posts or public file sharing -* [X] 💻 Easy API system for integration to websites +* [X] 💻 Easy API for integration to websites * [X] 🕵️ Metadata analysis resistance and anonymity * [X] 📡 Transport agnosticism (no internet required) @@ -54,7 +54,7 @@ Friend/contact manager Encrypted, metadata-masking mail application screenshot -Encrypted, metadata-masking mail application. +Encrypted, metadata-masking mail application. Perhaps the first distributed mail system to have basic forward secrecy. # Install and Run on Linux diff --git a/docs/http-api.md b/docs/http-api.md index 52a55368..b2fae877 100755 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -1,2 +1,50 @@ -HTTP API -TODO +# Onionr HTTP API + +# About HTTP API + +All HTTP interfaces in the Onionr reference client use the [Flask](http://flask.pocoo.org/) web framework with the [gevent](http://www.gevent.org/) WSGI server. + +## Client & Public difference + +The client API server is a locked down interface intended for authenticated local communication. + +The public API server is available only remotely from Tor & I2P. It is the interface in which peers use to communicate with one another. + +# Client API + +Please note: endpoints that simply provide static web app files are not documented here. + +(Client API docs coming soon) + +# Public API + +* / + - Methods: GET + - Returns a basic HTML informational banner describing Onionr. +* /getblocklist + - Methods: GET + - URI Parameters: + - date: unix epoch timestamp for offset + - Returns a list of block hashes stored on the node since an offset (all blocks if no timestamp is specified) +* /getdata/block-hash + - Methods: GET + - Returns data for a block based on a provided hash +* /www/file-path + - Methods: GET + - Returns file data. Intended for manually sharing file data directly from an Onionr node. +* /ping + - Methods: GET + - Returns 'pong!' +* /pex + - Methods: GET + - Returns a list of peer addresses reached within recent time +* /announce + - Methods: POST + - Accepts form data for 'node' (valid node address) and 'random' which is a nonce when hashed (blake2b_256) in the format `hash(peerAddress+serverAddress+nonce)`, begins with at least 5 zeros. + - Returns 200 with 'Success' if no error occurs. If the post is invalid, 'failure' with code 406 is returned. +* /upload + - Methods: POST + - Accepts form data for 'block' as a 'file' upload. + - Returns 200 with 'success' if no error occurs. If the block cannot be accepted, 'failure' with 400 is returned. + +# Direct Connection API \ No newline at end of file diff --git a/docs/specs/block-spec.md b/docs/specs/block-spec.md index f9cd3509..348cedae 100644 --- a/docs/specs/block-spec.md +++ b/docs/specs/block-spec.md @@ -2,9 +2,9 @@ # Block Description -Onionr 'Blocks' are the primary means of sharing information in Onionr. Blocks are identified by a single hash value of their entire contents. +Onionr Blocks are the primary means of sharing information in Onionr. Blocks are identified by a single hash value of their entire contents, using SHA3_256. -They contain a JSON metadata section followed by a line break, with the main data following. +Blocks contain a JSON metadata section followed by a line break, with the main data section comprising the rest. In the future, the specification will likely be updated to use flags and MessagePack instead of JSON with english keys. diff --git a/onionr/api.py b/onionr/api.py index 415707e5..df8b09f2 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -186,7 +186,7 @@ class PublicAPI: powHash = powHash.decode() except AttributeError: pass - if powHash.startswith('0000'): + if powHash.startswith('00000'): newNode = clientAPI._core._utils.bytesToStr(newNode) if clientAPI._core._utils.validateID(newNode) and not newNode in clientAPI._core.onionrInst.communicatorInst.newPeers: clientAPI._core.onionrInst.communicatorInst.newPeers.append(newNode) @@ -194,6 +194,8 @@ class PublicAPI: else: logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash) resp = Response(resp) + if resp == 'failure': + return resp, 406 return resp @app.route('/upload', methods=['post']) From d403f490e9f1b76fb04f026e3218f4cfd378c81a Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 12 Apr 2019 12:14:16 -0500 Subject: [PATCH 19/19] updated docs and cleaned up api somewhat --- README.md | 7 +++ docs/http-api.md | 60 ++++++++++++++++++++++++-- onionr/api.py | 40 +++-------------- onionr/static-data/www/mail/index.html | 2 +- 4 files changed, 71 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 58dac596..b63fe020 100755 --- a/README.md +++ b/README.md @@ -56,6 +56,13 @@ Friend/contact manager Encrypted, metadata-masking mail application. Perhaps the first distributed mail system to have basic forward secrecy. +# Documentation + +More docs coming soon. + +* [Block specification](docs/specs/block-spec.md) +* [HTTP API](docs/http-api.md) + # Install and Run on Linux The following applies to Ubuntu Bionic. Other distros may have different package or command names. diff --git a/docs/http-api.md b/docs/http-api.md index b2fae877..d2ee56a5 100755 --- a/docs/http-api.md +++ b/docs/http-api.md @@ -1,7 +1,5 @@ # Onionr HTTP API -# About HTTP API - All HTTP interfaces in the Onionr reference client use the [Flask](http://flask.pocoo.org/) web framework with the [gevent](http://www.gevent.org/) WSGI server. ## Client & Public difference @@ -14,10 +12,66 @@ The public API server is available only remotely from Tor & I2P. It is the inter Please note: endpoints that simply provide static web app files are not documented here. -(Client API docs coming soon) +* /serviceactive/pubkey + - Methods: GET + - Returns true or false based on if a given public key has an active direct connection service. +* /queueResponseAdd/key (DEPRECATED) + - Methods: POST + - Accepts form key 'data' to set queue response information from a plugin + - Returns success if no error occurs +* /queueResponse/key (DEPRECATED) + - Methods: GET + - Returns the queue response for a key. Returns failure with a 404 code if a code is not set. +* /ping + - Methods: GET + - Returns "pong!" +* /getblocksbytype/type + - Methods: GET + - Returns a list of stored blocks by a given type +* /getblockbody/hash + - Methods: GET + - Returns the main data section of a block +* /getblockdata/hash + - Methods: GET + - Returns the entire data contents of a block, including metadata. +* /getblockheader/hash + - Methods: GET + - Returns the header (metadata section) of a block. +* /lastconnect + - Methods: GET + - Returns the epoch timestamp of when the last incoming connection to the public API server was logged +* /site/hash + - Methods: GET + - Returns HTML content out of a block +* /waitforshare/hash + - Methods: POST + - Prevents the public API server from listing or sharing a block until it has been uploaded to at least 1 peer. +* /shutdown + - Methods: GET + - Shutdown Onionr. You should probably use /shutdownclean instead. +* /shutdownclean + - Methods: GET + - Tells the communicator daemon to shutdown Onionr. Slower but cleaner. +* /getstats + - Methods: GET + - Returns some JSON serialized statistics +* /getuptime + - Methods: GET + - Returns uptime in seconds +* /getActivePubkey + - Methods: GET + - Returns the current active public key in base32 format +* /getHumanReadable/pubkey + - Methods: GET + - Echos the specified public key in mnemonic format +* /insertblock + - Methods: POST + - Accepts JSON data for creating a new block. 'message' contains the block data, 'to' specifies the peer's public key to encrypt the data to, 'sign' is a boolean for signing the message. # Public API +v0 + * / - Methods: GET - Returns a basic HTML informational banner describing Onionr. diff --git a/onionr/api.py b/onionr/api.py index df8b09f2..ae6368e2 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -270,7 +270,6 @@ class API: logger.info('Running api on %s:%s' % (self.host, self.bindPort)) self.httpServer = '' - self.pluginResponses = {} # Responses for plugin endpoints self.queueResponse = {} onionrInst.setClientAPIInst(self) app.register_blueprint(friendsapi.friends) @@ -369,7 +368,10 @@ class API: pass else: del self.queueResponse[name] - return Response(resp) + if resp == 'failure': + return resp, 404 + else: + return resp @app.route('/ping') def ping(): @@ -522,36 +524,6 @@ class API: pass threading.Thread(target=self._core.insertBlock, args=(message,), kwargs={'header': bType, 'encryptType': encryptType, 'sign':sign, 'asymPeer': to, 'meta': meta}).start() return Response('success') - - @app.route('/apipoints/', methods=['POST', 'GET']) - def pluginEndpoints(subpath=''): - '''Send data to plugins''' - # TODO have a variable for the plugin to set data to that we can use for the response - pluginResponseCode = str(uuid.uuid4()) - resp = 'success' - responseTimeout = 20 - startTime = self._core._utils.getEpoch() - postData = {} - if request.method == 'POST': - postData = request.form['postData'] - if len(subpath) > 1: - data = subpath.split('/') - if len(data) > 1: - plName = data[0] - events.event('pluginRequest', {'name': plName, 'path': subpath, 'pluginResponse': pluginResponseCode, 'postData': postData}, onionr=onionrInst) - while True: - try: - resp = self.pluginResponses[pluginResponseCode] - except KeyError: - time.sleep(0.2) - if self._core._utils.getEpoch() - startTime > responseTimeout: - abort(504) - break - else: - break - else: - abort(404) - return Response(resp) self.httpServer = WSGIServer((self.host, bindPort), app, log=None, handler_class=FDSafeHandler) self.httpServer.serve_forever() @@ -578,8 +550,8 @@ class API: def getUptime(self): while True: try: - return self._utils.getEpoch - startTime - except AttributeError: + return self._utils.getEpoch() - self.startTime + except (AttributeError, NameError): # Don't error on race condition with startup pass diff --git a/onionr/static-data/www/mail/index.html b/onionr/static-data/www/mail/index.html index f3f337bd..08441075 100644 --- a/onionr/static-data/www/mail/index.html +++ b/onionr/static-data/www/mail/index.html @@ -58,7 +58,7 @@
-
+ To: Subject: