From ca70b275f6928c598b16c3ca03717671b8eb9fb4 Mon Sep 17 00:00:00 2001 From: Kevin Date: Sun, 9 Dec 2018 17:29:39 +0000 Subject: [PATCH] Merge wot --- .gitignore | 1 + CONTRIBUTING.md | 26 ++++- Makefile | 6 +- README.md | 2 +- onionr/api.py | 87 +++++++------- onionr/communicator2.py | 46 +++++--- onionr/config.py | 4 +- onionr/core.py | 50 ++++----- onionr/dbcreator.py | 8 +- onionr/keymanager.py | 80 +++++++++++++ onionr/netcontroller.py | 26 +++-- onionr/onionr.py | 106 ++++++++++++++---- onionr/onionrcrypto.py | 42 +++++-- onionr/onionrdaemontools.py | 47 ++++++-- onionr/onionrexceptions.py | 3 + onionr/onionrgui.py | 58 ++++++++++ onionr/onionrplugins.py | 4 +- onionr/onionrproofs.py | 7 +- onionr/onionrusers.py | 15 ++- onionr/onionrutils.py | 34 +++--- .../static-data/default-plugins/cliui/main.py | 3 +- .../static-data/default-plugins/flow/main.py | 56 +++++---- .../default-plugins/metadataprocessor/main.py | 5 +- .../default-plugins/pluginmanager/main.py | 2 +- .../static-data/default-plugins/pms/main.py | 99 ++++++++-------- .../default-plugins/pms/sentboxdb.py | 8 +- onionr/static-data/default_config.json | 19 ++-- RUN-LINUX.sh => run-linux | 0 RUN-WINDOWS.bat => run-windows.bat | 0 29 files changed, 571 insertions(+), 273 deletions(-) create mode 100644 onionr/keymanager.py create mode 100755 onionr/onionrgui.py rename RUN-LINUX.sh => run-linux (100%) rename RUN-WINDOWS.bat => run-windows.bat (100%) diff --git a/.gitignore b/.gitignore index 6fcdd586..0aa47977 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ __pycache__/ onionr/data/config.ini onionr/data/*.db onionr/data-old/* +onionr/data* onionr/*.pyc onionr/*.log onionr/data/hs/hostname diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ac158606..91a0e227 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,8 @@ # Contributing to Onionr -One of the great things about open source projects is that they allow for many people to contribute to the project. This file should serve as a guideline when contributing to Onionr. + +One of the great things about open source projects is that they allow for many people to contribute to the project. + +This file serves to provide guidelines on how to successfully contribute to Onionr. ## Code of Conduct @@ -7,16 +10,29 @@ See our [Code of Conduct](https://github.com/beardog108/onionr/blob/master/CODE_ ## Reporting Bugs -Bugs can be reported using GitHub issues. +Bugs can be reported using GitLab issues. Please try to see if an issue is already opened for a particular thing. -TODO +Please provide the following information when reporting a bug: + +* Operating system +* Python version +* Onionr version +* Onionr logs or output with errors, any possible relevant information. +* A description of what you were doing before the bug was experienced +* Screenshots can often be helpful, but videos are rarely helpful. + +If a bug is a security issue, please contact us privately. + +And most importantly, please be patient. Onionr is an open source project done by volunteers. ## Asking Questions -TODO +If you need help with Onionr, you can ask in our ## Contributing Code -TODO +For any non-trivial changes, please get in touch with us first to discuss your plans. + +Please try to use a similar coding style as the project. **Thanks for contributing to Onionr!** diff --git a/Makefile b/Makefile index 9ee7df8d..7e235b4f 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ uninstall: rm -f $(DESTDIR)$(PREFIX)/bin/onionr test: - @./RUN-LINUX.sh stop + @./run-linux stop @sleep 1 @rm -rf onionr/data-backup @mv onionr/data onionr/data-backup | true > /dev/null 2>&1 @@ -29,7 +29,7 @@ test: soft-reset: @echo "Soft-resetting Onionr..." rm -f onionr/data/blocks/*.dat onionr/data/*.db onionr/data/block-nonces.dat | true > /dev/null 2>&1 - @./RUN-LINUX.sh version | grep -v "Failed" --color=always + @./run-linux version | grep -v "Failed" --color=always reset: @echo "Hard-resetting Onionr..." @@ -40,4 +40,4 @@ reset: plugins-reset: @echo "Resetting plugins..." rm -rf onionr/data/plugins/ | true > /dev/null 2>&1 - @./RUN-LINUX.sh version | grep -v "Failed" --color=always + @./run-linux version | grep -v "Failed" --color=always diff --git a/README.md b/README.md index a01a65b2..5d1aa37b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ![Onionr logo](./docs/onionr-logo.png) -v0.3.0 (***experimental, not safe or easy to use yet***) +(***experimental, not safe or easy to use yet***) [![Open Source Love](https://badges.frapsoft.com/os/v3/open-source.png?v=103)](https://github.com/ellerbrock/open-source-badges/) diff --git a/onionr/api.py b/onionr/api.py index 6e637136..94cf2cd9 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -86,21 +86,18 @@ class API: app = flask.Flask(__name__) bindPort = int(config.get('client.port', 59496)) self.bindPort = bindPort - self.clientToken = config.get('client.hmac') + self.clientToken = config.get('client.webpassword') self.timeBypassToken = base64.b16encode(os.urandom(32)).decode() self.i2pEnabled = config.get('i2p.host', False) - self.mimeType = 'text/plain' - self.overrideCSP = False - self.hideBlocks = [] # Blocks to be denied sharing with open(self._core.dataDir + 'time-bypass.txt', 'w') as bypass: bypass.write(self.timeBypassToken) if not debug and not self._developmentMode: - hostOctets = [127, random.randint(0x02, 0xFF), random.randint(0x02, 0xFF), random.randint(0x02, 0xFF)] + hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))] self.host = '.'.join(hostOctets) else: self.host = '127.0.0.1' @@ -122,19 +119,26 @@ class API: resp.headers['Access-Control-Allow-Origin'] = '*' #else: # resp.headers['server'] = 'Onionr' - resp.headers['Content-Type'] = self.mimeType - if not self.overrideCSP: - resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'" + resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'" resp.headers['X-Frame-Options'] = 'deny' resp.headers['X-Content-Type-Options'] = "nosniff" resp.headers['X-API'] = API_VERSION - - # reset to text/plain to help prevent browser attacks - self.mimeType = 'text/plain' - self.overrideCSP = False - + resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch. return resp + @app.route('/site/') + def site(block): + self.validateHost('private') + bHash = block + resp = 'Not Found' + if self._core._utils.validateHash(bHash): + resp = Block(bHash).bcontent + try: + resp = base64.b64decode(resp) + except: + pass + return Response(resp) + @app.route('/www/private/') def www_private(path): startTime = math.floor(time.time()) @@ -149,9 +153,6 @@ class API: self.validateHost('private') - if config.get('www.public.guess_mime', True): - self.mimeType = API.guessMime(path) - endTime = math.floor(time.time()) elapsed = endTime - startTime @@ -168,9 +169,6 @@ class API: self.validateHost('public') - if config.get('www.public.guess_mime', True): - self.mimeType = API.guessMime(path) - return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path) @app.route('/ui/') @@ -201,12 +199,11 @@ class API: time.sleep(self._privateDelayTime - elapsed) ''' - self.mimeType = API.guessMime(path) - self.overrideCSP = True + mime = API.guessMime(path) - logger.debug('Serving %s (mime: %s)' % (path, self.mimeType)) + logger.debug('Serving %s (mime: %s)' % (path, mime)) - return send_from_directory('static-data/www/ui/dist/', path, mimetype = API.guessMime(path)) + return send_from_directory('static-data/www/ui/dist/', path) @app.route('/client/') def private_handler(): @@ -233,6 +230,8 @@ class API: self.validateHost('private') if action == 'hello': resp = Response('Hello, World! ' + request.host) + elif action == 'getIP': + resp = Response(self.host) elif action == 'waitForShare': if self._core._utils.validateHash(data): if data not in self.hideBlocks: @@ -248,16 +247,8 @@ class API: resp = Response('Goodbye') elif action == 'ping': resp = Response('pong') - elif action == 'site': - block = data - siteData = self._core.getData(data) - response = 'not found' - if siteData != '' and siteData != False: - self.mimeType = 'text/html' - response = siteData.split(b'-', 2)[-1] - resp = Response(response) elif action == 'info': - resp = Response(json.dumps({'pubkey' : self._core._crypto.pubKey, 'host' : self._core.hsAddress})) + resp = Response(json.dumps({'pubkey' : self._core._crypto.pubKey, 'host' : self._core.hsAddress}), mimetype='text/plain') elif action == "insertBlock": response = {'success' : False, 'reason' : 'An unknown error occurred'} @@ -368,12 +359,12 @@ class API: else: response = {'success' : False, 'reason' : 'Missing `data` parameter.', 'blocks' : {}} - resp = Response(json.dumps(response)) + resp = Response(json.dumps(response), mimetype='text/plain') elif action in API.callbacks['private']: - resp = Response(str(getCallback(action, scope = 'private')(request))) + resp = Response(str(getCallback(action, scope = 'private')(request)), mimetype='text/plain') else: - resp = Response('(O_o) Dude what? (invalid command)') + resp = Response('invalid command') endTime = math.floor(time.time()) elapsed = endTime - startTime @@ -386,11 +377,10 @@ class API: @app.route('/') def banner(): - self.mimeType = 'text/html' self.validateHost('public') try: with open('static-data/index.html', 'r') as html: - resp = Response(html.read()) + resp = Response(html.read(), mimetype='text/html') except FileNotFoundError: resp = Response("") return resp @@ -461,6 +451,8 @@ class API: def public_handler(): # Public means it is publicly network accessible self.validateHost('public') + if config.get('general.security_level') != 0: + abort(403) action = request.args.get('action') requestingPeer = request.args.get('myID') data = request.args.get('data') @@ -489,9 +481,10 @@ class API: elif action == 'getData': resp = '' if self._utils.validateHash(data): - if os.path.exists(self._core.dataDir + 'blocks/' + data + '.dat'): - block = Block(hash=data.encode(), core=self._core) - resp = base64.b64encode(block.getRaw().encode()).decode() + if data not in self.hideBlocks: + if os.path.exists(self._core.dataDir + 'blocks/' + data + '.dat'): + block = Block(hash=data.encode(), core=self._core) + resp = base64.b64encode(block.getRaw().encode()).decode() if len(resp) == 0: abort(404) resp = "" @@ -531,8 +524,8 @@ class API: resp = Response("Invalid request") return resp - if not os.environ.get("WERKZEUG_RUN_MAIN") == "true": - logger.info('Starting client on ' + self.host + ':' + str(bindPort) + '...', timestamp=False) + + logger.info('Starting client on ' + self.host + ':' + str(bindPort), timestamp=False) try: while len(self._core.hsAddress) == 0: @@ -545,7 +538,6 @@ class API: except Exception as e: logger.error(str(e)) logger.fatal('Failed to start client on ' + self.host + ':' + str(bindPort) + ', exiting...') - exit(1) def validateHost(self, hostType): ''' @@ -571,13 +563,16 @@ class API: if not self.i2pEnabled and request.host.endswith('i2p'): abort(403) + ''' if not self._developmentMode: try: request.headers['X-Requested-With'] except: - # we exit rather than abort to avoid fingerprinting - logger.debug('Avoiding fingerprinting, exiting...') - sys.exit(1) + pass + # we exit rather than abort to avoid fingerprinting + logger.debug('Avoiding fingerprinting, exiting...') + #sys.exit(1) + ''' def setCallback(action, callback, scope = 'public'): if not scope in API.callbacks: diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 08527f8f..0bce1511 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -21,7 +21,7 @@ ''' import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block -import onionrdaemontools, onionrsockets, onionrchat, onionr +import onionrdaemontools, onionrsockets, onionrchat, onionr, onionrproofs from dependencies import secrets from defusedxml import minidom @@ -70,6 +70,9 @@ class OnionrCommunicatorDaemon: # list of blocks currently downloading, avoid s self.currentDownloading = [] + # timestamp when the last online node was seen + self.lastNodeSeen = None + # Clear the daemon queue for any dead messages if os.path.exists(self._core.queueDB): self._core.clearDaemonQueue() @@ -98,23 +101,31 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True) OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True) OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1) + OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1) + deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1) + netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) - announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 305, requiresPeer=True, maxThreads=1) + if config.get('general.security_level') == 0: + announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 86400, requiresPeer=True, maxThreads=1) + announceTimer.count = (announceTimer.frequency - 120) + else: + logger.debug('Will not announce node.') cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requiresPeer=True) forwardSecrecyTimer = OnionrCommunicatorTimers(self, self.daemonTools.cleanKeys, 15) # set loop to execute instantly to load up peer pool (replaced old pool init wait) peerPoolTimer.count = (peerPoolTimer.frequency - 1) cleanupTimer.count = (cleanupTimer.frequency - 60) - announceTimer.count = (cleanupTimer.frequency - 60) + deniableBlockTimer.count = (deniableBlockTimer.frequency - 175) #forwardSecrecyTimer.count = (forwardSecrecyTimer.frequency - 990) - self.socketServer = threading.Thread(target=onionrsockets.OnionrSocketServer, args=(self._core,)) - self.socketServer.start() - self.socketClient = onionrsockets.OnionrSocketClient(self._core) + if config.get('general.socket_servers'): + self.socketServer = threading.Thread(target=onionrsockets.OnionrSocketServer, args=(self._core,)) + self.socketServer.start() + self.socketClient = onionrsockets.OnionrSocketClient(self._core) - # Loads chat messages into memory - threading.Thread(target=self._chat.chatHandler).start() + # Loads chat messages into memory + threading.Thread(target=self._chat.chatHandler).start() # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking try: @@ -187,8 +198,8 @@ class OnionrCommunicatorDaemon: if not i in existingBlocks: # if block does not exist on disk and is not already in block queue if i not in self.blockQueue and not self._core._blacklist.inBlacklist(i): - # TODO ensure block starts with minimum difficulty before adding to queue - self.blockQueue.append(i) # add blocks to download queue + if onionrproofs.hashMeetsDifficulty(i): + self.blockQueue.append(i) # add blocks to download queue self.decrementThreadCount('lookupBlocks') return @@ -230,7 +241,6 @@ class OnionrCommunicatorDaemon: content = content.decode() # decode here because sha3Hash needs bytes above metas = self._core._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata metadata = metas[0] - #meta = metas[1] if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce if self._core._crypto.verifyPow(content): # check if POW is enough/correct logger.info('Attempting to save block %s...' % blockHash) @@ -317,11 +327,14 @@ class OnionrCommunicatorDaemon: self.connectNewPeer(useBootstrap=True) else: self.connectNewPeer() + if self.shutdown: break else: if len(self.onlinePeers) == 0: - logger.debug('Couldn\'t connect to any peers.') + logger.debug('Couldn\'t connect to any peers.' + (' Last node seen %s ago.' % self.daemonTools.humanReadableTime(time.time() - self.lastNodeSeen) if not self.lastNodeSeen is None else '')) + else: + self.lastNodeSeen = time.time() self.decrementThreadCount('getOnlinePeers') def addBootstrapListToPeerList(self, peerList): @@ -352,7 +365,7 @@ class OnionrCommunicatorDaemon: self.addBootstrapListToPeerList(peerList) for address in peerList: - if not config.get('tor.v3_onions') and len(address) == 62: + if not config.get('tor.v3onions') and len(address) == 62: continue if len(address) == 0 or address in tried or address in self.onlinePeers or address in self.cooldownPeer: continue @@ -445,7 +458,7 @@ class OnionrCommunicatorDaemon: def heartbeat(self): '''Show a heartbeat debug message''' currentTime = self._core._utils.getEpoch() - self.startTime - logger.debug('Heartbeat. Node online for %s.' % self.daemonTools.humanReadableTime(currentTime)) + logger.debug('Heartbeat. Node running for %s.' % self.daemonTools.humanReadableTime(currentTime)) self.decrementThreadCount('heartbeat') def daemonCommands(self): @@ -456,14 +469,13 @@ class OnionrCommunicatorDaemon: if cmd is not False: events.event('daemon_command', onionr = None, data = {'cmd' : cmd}) - if cmd[0] == 'shutdown': self.shutdown = True elif cmd[0] == 'announceNode': if len(self.onlinePeers) > 0: self.announce(cmd[1]) else: - logger.warn("Not introducing, since I have no connected nodes.") + logger.debug("No nodes connected. Will not introduce node.") elif cmd[0] == 'runCheck': # deprecated logger.debug('Status check; looks good.') open(self._core.dataDir + '.runcheck', 'w+').close() @@ -584,7 +596,7 @@ class OnionrCommunicatorTimers: if self.makeThread: for i in range(self.threadAmount): if self.daemonInstance.threadCounts[self.timerFunction.__name__] >= self.maxThreads: - logger.warn('%s is currently using the maximum number of threads, not starting another.' % self.timerFunction.__name__) + logger.debug('%s is currently using the maximum number of threads, not starting another.' % self.timerFunction.__name__) else: self.daemonInstance.threadCounts[self.timerFunction.__name__] += 1 newThread = threading.Thread(target=self.timerFunction) diff --git a/onionr/config.py b/onionr/config.py index 1e782dbe..7c986b83 100644 --- a/onionr/config.py +++ b/onionr/config.py @@ -30,7 +30,7 @@ except KeyError: _configfile = os.path.abspath(dataDir + 'config.json') _config = {} -def get(key, default = None): +def get(key, default = None, save = False): ''' Gets the key from configuration, or returns `default` ''' @@ -46,6 +46,8 @@ def get(key, default = None): data = data[item] if not last in data: + if save: + set(key, default, savefile = True) return default return data[last] diff --git a/onionr/core.py b/onionr/core.py index 781c6527..0219a875 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -113,7 +113,7 @@ class Core: with open(self.dataDir + '/hs/hostname', 'r') as hs: self.hsAddress = hs.read().strip() - def addPeer(self, peerID, powID, name=''): + def addPeer(self, peerID, name=''): ''' Adds a public key to the key database (misleading function name) ''' @@ -121,16 +121,13 @@ class Core: # This function simply adds a peer to the DB if not self._utils.validatePubKey(peerID): return False - if sys.getsizeof(powID) > 120: - logger.warn("POW token for pubkey base64 representation exceeded 120 bytes, is " + str(sys.getsizeof(powID))) - return False events.event('pubkey_add', data = {'key': peerID}, onionr = None) conn = sqlite3.connect(self.peerDB, timeout=10) hashID = self._crypto.pubKeyHashID(peerID) c = conn.cursor() - t = (peerID, name, 'unknown', hashID, powID, 0) + t = (peerID, name, 'unknown', hashID, 0) for i in c.execute("SELECT * FROM peers WHERE id = ?;", (peerID,)): try: @@ -141,7 +138,7 @@ class Core: pass except IndexError: pass - c.execute('INSERT INTO peers (id, name, dateSeen, pow, hashID, trust) VALUES(?, ?, ?, ?, ?, ?);', t) + c.execute('INSERT INTO peers (id, name, dateSeen, hashID, trust) VALUES(?, ?, ?, ?, ?);', t) conn.commit() conn.close() @@ -154,6 +151,8 @@ class Core: if address == config.get('i2p.ownAddr', None) or address == self.hsAddress: return False + if type(address) is type(None) or len(address) == 0: + return False if self._utils.validateID(address): conn = sqlite3.connect(self.addressDB, timeout=10) c = conn.cursor() @@ -231,21 +230,18 @@ class Core: ''' Generate the address database ''' - self.dbCreate.createAddressDB() def createPeerDB(self): ''' Generate the peer sqlite3 database and populate it with the peers table. ''' - self.dbCreate.createPeerDB() def createBlockDB(self): ''' Create a database for blocks ''' - self.dbCreate.createBlockDB() def addToBlockDB(self, newHash, selfInsert=False, dataSaved=False): @@ -374,7 +370,6 @@ class Core: retData = False self.daemonQueue() events.event('queue_push', data = {'command': command, 'data': data}, onionr = None) - return retData def clearDaemonQueue(self): @@ -464,18 +459,14 @@ class Core: name text, 1 adders text, 2 dateSeen not null, 3 - bytesStored int, 4 - trust int 5 - pubkeyExchanged int 6 - hashID text 7 - pow text 8 + trust int 4 + hashID text 5 ''' conn = sqlite3.connect(self.peerDB, timeout=10) c = conn.cursor() command = (peer,) - - infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'bytesStored': 4, 'trust': 5, 'pubkeyExchanged': 6, 'hashID': 7} + infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'trust': 4, 'hashID': 5} info = infoNumbers[info] iterCount = 0 retVal = '' @@ -503,7 +494,7 @@ class Core: command = (data, peer) # TODO: validate key on whitelist - if key not in ('id', 'name', 'pubkey', 'blockDBHash', 'forwardKey', 'dateSeen', 'bytesStored', 'trust'): + if key not in ('id', 'name', 'pubkey', 'forwardKey', 'dateSeen', 'trust'): raise Exception("Got invalid database key when setting peer info") c.execute('UPDATE peers SET ' + key + ' = ? WHERE id=?', command) @@ -524,13 +515,15 @@ class Core: DBHash text, 5 failure int 6 lastConnect 7 + trust 8 + introduced 9 ''' conn = sqlite3.connect(self.addressDB, timeout=10) c = conn.cursor() command = (address,) - infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'DBHash': 5, 'failure': 6, 'lastConnect': 7} + infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'DBHash': 5, 'failure': 6, 'lastConnect': 7, 'trust': 8, 'introduced': 9} info = infoNumbers[info] iterCount = 0 retVal = '' @@ -555,9 +548,8 @@ class Core: c = conn.cursor() command = (data, address) - - # TODO: validate key on whitelist - if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'DBHash', 'failure', 'lastConnect', 'lastConnectAttempt'): + + if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'DBHash', 'failure', 'lastConnect', 'lastConnectAttempt', 'trust', 'introduced'): raise Exception("Got invalid database key when setting address info") else: c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command) @@ -679,7 +671,7 @@ class Core: return True - def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = None, expire=None): + def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None): ''' Inserts a block into the network encryptType must be specified to encrypt a block @@ -710,9 +702,8 @@ class Core: # metadata is full block metadata, meta is internal, user specified metadata # only use header if not set in provided meta - if not header is None: - meta['type'] = header - meta['type'] = str(meta['type']) + + meta['type'] = str(header) if encryptType in ('asym', 'sym', ''): metadata['encryptType'] = encryptType @@ -731,8 +722,6 @@ class Core: meta['forwardEnc'] = True except onionrexceptions.InvalidPubkey: onionrusers.OnionrUser(self, asymPeer).generateForwardKey() - else: - logger.info(forwardEncrypted) onionrusers.OnionrUser(self, asymPeer).generateForwardKey() fsKey = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys()[0] meta['newFSKey'] = fsKey[0] @@ -763,6 +752,7 @@ class Core: data = self._crypto.pubKeyEncrypt(data, asymPeer, encodedData=True, anonymous=True).decode() signature = self._crypto.pubKeyEncrypt(signature, asymPeer, encodedData=True, anonymous=True).decode() signer = self._crypto.pubKeyEncrypt(signer, asymPeer, encodedData=True, anonymous=True).decode() + onionrusers.OnionrUser(self, asymPeer, saveUser=True) else: raise onionrexceptions.InvalidPubkey(asymPeer + ' is not a valid base32 encoded ed25519 key') @@ -770,7 +760,7 @@ class Core: metadata['meta'] = jsonMeta metadata['sig'] = signature metadata['signer'] = signer - metadata['time'] = str(self._utils.getEpoch()) + metadata['time'] = self._utils.getRoundedEpoch() + self._crypto.secrets.randbelow(301) # ensure expire is integer and of sane length if type(expire) is not type(None): @@ -798,7 +788,7 @@ class Core: Introduces our node into the network by telling X many nodes our HS address ''' - if(self._utils.isCommunicatorRunning()): + if(self._utils.isCommunicatorRunning(timeout=30)): announceAmount = 2 nodeList = self.listAdders() diff --git a/onionr/dbcreator.py b/onionr/dbcreator.py index d11aa4fa..f2f12f07 100644 --- a/onionr/dbcreator.py +++ b/onionr/dbcreator.py @@ -44,7 +44,8 @@ class DBCreator: failure int, lastConnect int, lastConnectAttempt int, - trust int + trust int, + introduced int ); ''') conn.commit() @@ -62,11 +63,8 @@ class DBCreator: name text, adders text, dateSeen not null, - bytesStored int, trust int, - pubkeyExchanged int, - hashID text, - pow text not null); + hashID text); ''') c.execute('''CREATE TABLE forwardKeys( peerKey text not null, diff --git a/onionr/keymanager.py b/onionr/keymanager.py new file mode 100644 index 00000000..ff89401b --- /dev/null +++ b/onionr/keymanager.py @@ -0,0 +1,80 @@ +''' + Onionr - P2P Anonymous Storage Network + + Load, save, and delete the user's public key pairs (does not handle peer keys) +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import onionrcrypto +class KeyManager: + def __init__(self, crypto): + assert isinstance(crypto, onionrcrypto.OnionrCrypto) + self._core = crypto._core + self._utils = self._core._utils + self.keyFile = crypto._keyFile + self.crypto = crypto + + def addKey(self, pubKey=None, privKey=None): + if type(pubKey) is type(None) and type(privKey) is type(None): + pubKey, privKey = self.crypto.generatePubKey() + pubKey = self.crypto._core._utils.bytesToStr(pubKey) + privKey = self.crypto._core._utils.bytesToStr(privKey) + try: + if pubKey in self.getPubkeyList(): + raise ValueError('Pubkey already in list: %s' % (pubKey,)) + except FileNotFoundError: + pass + with open(self.keyFile, "a") as keyFile: + keyFile.write(pubKey + ',' + privKey + '\n') + return (pubKey, privKey) + + def removeKey(self, pubKey): + '''Remove a key pair by pubkey''' + keyList = self.getPubkeyList() + keyData = '' + try: + keyList.remove(pubKey) + except ValueError: + return False + else: + keyData = ','.join(keyList) + with open(self.keyFile, "w") as keyFile: + keyFile.write(keyData) + + def getPubkeyList(self): + '''Return a list of the user's keys''' + keyList = [] + with open(self.keyFile, "r") as keyFile: + keyData = keyFile.read() + keyData = keyData.split('\n') + for pair in keyData: + if len(pair) > 0: keyList.append(pair.split(',')[0]) + return keyList + + def getPrivkey(self, pubKey): + privKey = None + with open(self.keyFile, "r") as keyFile: + keyData = keyFile.read() + for pair in keyData.split('\n'): + if pubKey in pair: + privKey = pair.split(',')[1] + return privKey + + def changeActiveKey(self, pubKey): + '''Change crypto.pubKey and crypto.privKey to a given key pair by specifying the public key''' + if not pubKey in self.getPubkeyList(): + raise ValueError('That pubkey does not exist') + self.crypto.pubKey = pubKey + self.crypto.privKey = self.getPrivkey(pubKey) \ No newline at end of file diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py index a6141aeb..76d8ece6 100644 --- a/onionr/netcontroller.py +++ b/onionr/netcontroller.py @@ -27,7 +27,7 @@ class NetController: This class handles hidden service setup on Tor and I2P ''' - def __init__(self, hsPort): + def __init__(self, hsPort, apiServerIP='127.0.0.1'): try: self.dataDir = os.environ['ONIONR_HOME'] if not self.dataDir.endswith('/'): @@ -41,6 +41,7 @@ class NetController: self.hsPort = hsPort self._torInstnace = '' self.myID = '' + self.apiServerIP = apiServerIP if os.path.exists('./tor'): self.torBinary = './tor' @@ -65,9 +66,9 @@ class NetController: Generate a torrc file for our tor instance ''' hsVer = '# v2 onions' - if config.get('tor.v3_onions'): + if config.get('tor.v3onions'): hsVer = 'HiddenServiceVersion 3' - logger.info('Using v3 onions :)') + logger.debug('Using v3 onions :)') if os.path.exists(self.torConfigLocation): os.remove(self.torConfigLocation) @@ -88,14 +89,16 @@ class NetController: break torrcData = '''SocksPort ''' + str(self.socksPort) + ''' -HiddenServiceDir ''' + self.dataDir + '''hs/ -\n''' + hsVer + '''\n -HiddenServicePort 80 127.0.0.1:''' + str(self.hsPort) + ''' DataDirectory ''' + self.dataDir + '''tordata/ CookieAuthentication 1 ControlPort ''' + str(controlPort) + ''' HashedControlPassword ''' + str(password) + ''' ''' + if config.get('general.security_level') == 0: + torrcData += '''\nHiddenServiceDir ''' + self.dataDir + '''hs/ +\n''' + hsVer + '''\n +HiddenServicePort 80 ''' + self.apiServerIP + ''':''' + str(self.hsPort) + torrc = open(self.torConfigLocation, 'w') torrc.write(torrcData) torrc.close() @@ -143,13 +146,16 @@ HashedControlPassword ''' + str(password) + ''' except KeyboardInterrupt: logger.fatal('Got keyboard interrupt.', timestamp = false, level = logger.LEVEL_IMPORTANT) return False - + logger.debug('Finished starting Tor.', timestamp=True) self.readyState = True - myID = open(self.dataDir + 'hs/hostname', 'r') - self.myID = myID.read().replace('\n', '') - myID.close() + try: + myID = open(self.dataDir + 'hs/hostname', 'r') + self.myID = myID.read().replace('\n', '') + myID.close() + except FileNotFoundError: + self.myID = "" torPidFile = open(self.dataDir + 'torPid.txt', 'w') torPidFile.write(str(tor.pid)) diff --git a/onionr/onionr.py b/onionr/onionr.py index d7117421..e2f88225 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -23,7 +23,7 @@ import sys if sys.version_info[0] == 2 or sys.version_info[1] < 5: - print('Error, Onionr requires Python 3.4+') + print('Error, Onionr requires Python 3.5+') sys.exit(1) import os, base64, random, getpass, shutil, subprocess, requests, time, platform, datetime, re, json, getpass, sqlite3 import webbrowser @@ -40,7 +40,7 @@ except ImportError: raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)") ONIONR_TAGLINE = 'Anonymous P2P Platform - GPLv3 - https://Onionr.VoidNet.Tech' -ONIONR_VERSION = '0.3.2' # for debugging and stuff +ONIONR_VERSION = '0.5.0' # for debugging and stuff ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION) API_VERSION = '5' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes know how to communicate without learning too much information about you. @@ -103,8 +103,8 @@ class Onionr: self.onionrCore.createAddressDB() # Get configuration - if type(config.get('client.hmac')) is type(None): - config.set('client.hmac', base64.b16encode(os.urandom(32)).decode('utf-8'), savefile=True) + if type(config.get('client.webpassword')) is type(None): + config.set('client.webpassword', base64.b16encode(os.urandom(32)).decode('utf-8'), savefile=True) if type(config.get('client.port')) is type(None): randomPort = 0 while randomPort < 1024: @@ -115,7 +115,6 @@ class Onionr: if type(config.get('client.api_version')) is type(None): config.set('client.api_version', API_VERSION, savefile=True) - self.cmds = { '': self.showHelpSuggestion, 'help': self.showHelp, @@ -168,6 +167,10 @@ class Onionr: 'add-file': self.addFile, 'addfile': self.addFile, + 'addhtml': self.addWebpage, + 'add-html': self.addWebpage, + 'add-site': self.addWebpage, + 'addsite': self.addWebpage, 'get-file': self.getFile, 'getfile': self.getFile, @@ -197,7 +200,9 @@ class Onionr: 'chat': self.startChat, - 'friend': self.friendCmd + 'friend': self.friendCmd, + 'add-id': self.addID, + 'change-id': self.changeID } self.cmdhelp = { @@ -226,7 +231,9 @@ class Onionr: 'pex': 'exchange addresses with peers (done automatically)', 'blacklist-block': 'deletes a block by hash and permanently removes it from your node', 'introduce': 'Introduce your node to the public Onionr network', - 'friend': '[add|remove] [public key/id]' + 'friend': '[add|remove] [public key/id]', + 'add-id': 'Generate a new ID (key pair)', + 'change-id': 'Change active ID' } # initialize plugins @@ -257,6 +264,48 @@ class Onionr: for detail in details: logger.info('%s%s: \n%s%s\n' % (logger.colors.fg.lightgreen, detail, logger.colors.fg.green, details[detail]), sensitive = True) + def addID(self): + try: + sys.argv[2] + assert sys.argv[2] == 'true' + except (IndexError, AssertionError) as e: + newID = self.onionrCore._crypto.keyManager.addKey()[0] + else: + logger.warn('Deterministic keys require random and long passphrases.') + logger.warn('If a good password is not used, your key can be easily stolen.') + pass1 = getpass.getpass(prompt='Enter at least %s characters: ' % (self.onionrCore._crypto.deterministicRequirement,)) + pass2 = getpass.getpass(prompt='Confirm entry: ') + if self.onionrCore._crypto.safeCompare(pass1, pass2): + try: + logger.info('Generating deterministic key. This can take a while.') + newID, privKey = self.onionrCore._crypto.generateDeterministic(pass1) + except onionrexceptions.PasswordStrengthError: + logger.error('Must use at least 25 characters.') + sys.exit(1) + else: + logger.error('Passwords do not match.') + sys.exit(1) + self.onionrCore._crypto.keyManager.addKey(pubKey=newID, + privKey=privKey) + logger.info('Added ID: %s' % (self.onionrUtils.bytesToStr(newID),)) + + def changeID(self): + try: + key = sys.argv[2] + except IndexError: + logger.error('Specify pubkey to use') + else: + if self.onionrUtils.validatePubKey(key): + if key in self.onionrCore._crypto.keyManager.getPubkeyList(): + config.set('general.public_key', key) + config.save() + logger.info('Set active key to: %s' % (key,)) + logger.info('Restart Onionr if it is running.') + else: + logger.error('That key does not exist') + else: + logger.error('Invalid key %s' % (key,)) + def startChat(self): try: data = json.dumps({'peer': sys.argv[2], 'reason': 'chat'}) @@ -334,13 +383,9 @@ class Onionr: friend = sys.argv[3] if not self.onionrUtils.validatePubKey(friend): raise onionrexceptions.InvalidPubkey('Public key is invalid') - if friend not in self.onionrCore.listPeers(): - raise onionrexceptions.KeyNotKnown friend = onionrusers.OnionrUser(self.onionrCore, friend) except IndexError: logger.error('Friend ID is required.') - except onionrexceptions.KeyNotKnown: - logger.error('That peer is not in our database') else: if action == 'add': friend.setTrust(1) @@ -381,7 +426,7 @@ class Onionr: logger.info(i) def getWebPassword(self): - return config.get('client.hmac') + return config.get('client.webpassword') def printWebPassword(self): logger.info(self.getWebPassword(), sensitive = True) @@ -506,6 +551,7 @@ class Onionr: try: newAddress = sys.argv[2] + newAddress = newAddress.replace('http:', '').replace('/', '') except: pass else: @@ -589,7 +635,7 @@ class Onionr: if len(sys.argv) >= 3: try: - plugin_name = re.sub('[^0-9a-zA-Z]+', '', str(sys.argv[2]).lower()) + plugin_name = re.sub('[^0-9a-zA-Z_]+', '', str(sys.argv[2]).lower()) if not plugins.exists(plugin_name): logger.info('Creating plugin "%s"...' % plugin_name) @@ -672,17 +718,26 @@ class Onionr: time.sleep(1) self.onionrUtils.localCommand('shutdown') else: + apiHost = '127.0.0.1' if apiThread.isAlive(): - # configure logger and stuff + try: + with open(self.onionrCore.dataDir + 'host.txt', 'r') as hostFile: + apiHost = hostFile.read() + except FileNotFoundError: + pass Onionr.setupConfig('data/', self = self) if self._developmentMode: logger.warn('DEVELOPMENT MODE ENABLED (THIS IS LESS SECURE!)', timestamp = False) - net = NetController(config.get('client.port', 59496)) + net = NetController(config.get('client.port', 59496), apiServerIP=apiHost) logger.debug('Tor is starting...') if not net.startTor(): + self.onionrUtils.localCommand('shutdown') sys.exit(1) - logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID)) + if len(net.myID) > 0 and config.get('general.security_level') == 0: + logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID)) + else: + logger.debug('.onion service disabled') logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey)) time.sleep(1) @@ -717,7 +772,7 @@ class Onionr: Shutdown the Onionr daemon ''' - logger.warn('Killing the running daemon...', timestamp = False) + logger.warn('Stopping the running daemon...', timestamp = False) try: events.event('daemon_stop', onionr = self) net = NetController(config.get('client.port', 59496)) @@ -729,7 +784,6 @@ class Onionr: net.killTor() except Exception as e: logger.error('Failed to shutdown daemon.', error = e, timestamp = False) - return def showStats(self): @@ -822,6 +876,8 @@ class Onionr: try: with open('./' + self.dataDir + 'hs/hostname', 'r') as hostname: return hostname.read().strip() + except FileNotFoundError: + return "Not Generated" except Exception: return None @@ -863,7 +919,13 @@ class Onionr: Block.mergeChain(bHash, fileName) return - def addFile(self): + def addWebpage(self): + ''' + Add a webpage to the onionr network + ''' + self.addFile(singleBlock=True, blockType='html') + + def addFile(self, singleBlock=False, blockType='txt'): ''' Adds a file to the onionr network ''' @@ -877,7 +939,11 @@ class Onionr: return logger.info('Adding file... this might take a long time.') try: - blockhash = Block.createChain(file = filename) + if singleBlock: + with open(filename, 'rb') as singleFile: + blockhash = self.onionrCore.insertBlock(base64.b64encode(singleFile.read()), header=blockType) + else: + blockhash = Block.createChain(file = filename) logger.info('File %s saved in block %s.' % (filename, blockhash)) except: logger.error('Failed to save file in block.', timestamp = False) diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index fe13ae01..35a8050d 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -17,8 +17,8 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import nacl.signing, nacl.encoding, nacl.public, nacl.hash, nacl.secret, os, binascii, base64, hashlib, logger, onionrproofs, time, math, sys - +import nacl.signing, nacl.encoding, nacl.public, nacl.hash, nacl.pwhash, nacl.utils, nacl.secret, os, binascii, base64, hashlib, logger, onionrproofs, time, math, sys, hmac +import onionrexceptions, keymanager # secrets module was added into standard lib in 3.6+ if sys.version_info[0] == 3 and sys.version_info[1] < 6: from dependencies import secrets @@ -36,20 +36,22 @@ class OnionrCrypto: self.secrets = secrets + self.deterministicRequirement = 25 # Min deterministic password/phrase length self.HASH_ID_ROUNDS = 2000 + self.keyManager = keymanager.KeyManager(self) # Load our own pub/priv Ed25519 keys, gen & save them if they don't exist if os.path.exists(self._keyFile): - with open(self._core.dataDir + 'keys.txt', 'r') as keys: - keys = keys.read().split(',') - self.pubKey = keys[0] - self.privKey = keys[1] + if len(config.get('general.public_key', '')) > 0: + self.pubKey = config.get('general.public_key') + else: + self.pubKey = self.keyManager.getPubkeyList()[0] + self.privKey = self.keyManager.getPrivkey(self.pubKey) else: keys = self.generatePubKey() self.pubKey = keys[0] self.privKey = keys[1] - with open(self._keyFile, 'w') as keyfile: - keyfile.write(self.pubKey + ',' + self.privKey) + self.keyManager.addKey(self.pubKey, self.privKey) return def edVerify(self, data, key, sig, encodedData=True): @@ -196,6 +198,27 @@ class OnionrCrypto: private_key = nacl.signing.SigningKey.generate() public_key = private_key.verify_key.encode(encoder=nacl.encoding.Base32Encoder()) return (public_key.decode(), private_key.encode(encoder=nacl.encoding.Base32Encoder()).decode()) + + def generateDeterministic(self, passphrase, bypassCheck=False): + '''Generate a Ed25519 public key pair from a password''' + passStrength = self.deterministicRequirement + passphrase = self._core._utils.strToBytes(passphrase) # Convert to bytes if not already + # Validate passphrase length + if not bypassCheck: + if len(passphrase) < passStrength: + raise onionrexceptions.PasswordStrengthError("Passphase must be at least %s characters" % (passStrength,)) + # KDF values + kdf = nacl.pwhash.argon2id.kdf + salt = b"U81Q7llrQcdTP0Ux" # Does not need to be unique or secret, but must be 16 bytes + ops = nacl.pwhash.argon2id.OPSLIMIT_SENSITIVE + mem = nacl.pwhash.argon2id.MEMLIMIT_SENSITIVE + + key = kdf(nacl.secret.SecretBox.KEY_SIZE, passphrase, salt, opslimit=ops, memlimit=mem) + key = nacl.public.PrivateKey(key, nacl.encoding.RawEncoder()) + publicKey = key.public_key + + return (publicKey.encode(encoder=nacl.encoding.Base32Encoder()), + key.encode(encoder=nacl.encoding.Base32Encoder())) def pubKeyHashID(self, pubkey=''): '''Accept a ed25519 public key, return a truncated result of X many sha3_256 hash rounds''' @@ -262,3 +285,6 @@ class OnionrCrypto: logger.debug("Invalid token, bad proof") return retData + + def safeCompare(self, one, two): + return hmac.compare_digest(one, two) diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index 28d7f964..890603dd 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -23,6 +23,9 @@ import base64, sqlite3, os from dependencies import secrets class DaemonTools: + ''' + Class intended for use by Onionr Communicator + ''' def __init__(self, daemon): self.daemon = daemon self.announceCache = {} @@ -30,7 +33,8 @@ class DaemonTools: def announceNode(self): '''Announce our node to our peers''' retData = False - + announceFail = False + # Announce to random online peers for i in self.daemon.onlinePeers: if not i in self.announceCache: @@ -50,14 +54,21 @@ class DaemonTools: data['random'] = self.announceCache[peer] else: proof = onionrproofs.DataPOW(combinedNodes, forceDifficulty=4) - data['random'] = base64.b64encode(proof.waitForResult()[1]) - self.announceCache[peer] = data['random'] - - logger.info('Announcing node to ' + url) - if self.daemon._core._utils.doPostRequest(url, data) == 'Success': - logger.info('Successfully introduced node to ' + peer) - retData = True - self.daemon.decrementThreadCount('announceNode') + try: + data['random'] = base64.b64encode(proof.waitForResult()[1]) + except TypeError: + # Happens when we failed to produce a proof + logger.error("Failed to produce a pow for announcing to " + peer) + announceFail = True + else: + self.announceCache[peer] = data['random'] + if not announceFail: + logger.info('Announcing node to ' + url) + if self.daemon._core._utils.doPostRequest(url, data) == 'Success': + logger.info('Successfully introduced node to ' + peer) + retData = True + self.daemon._core.setAddressInfo(peer, 'introduced', 1) + self.daemon.decrementThreadCount('announceNode') return retData def netCheck(self): @@ -66,6 +77,8 @@ class DaemonTools: if not self.daemon._core._utils.checkNetwork(torPort=self.daemon.proxyPort): logger.warn('Network check failed, are you connected to the internet?') self.daemon.isOnline = False + else: + self.daemon.isOnline = True self.daemon.decrementThreadCount('netCheck') def cleanOldBlocks(self): @@ -92,11 +105,11 @@ class DaemonTools: deleteKeys = [] for entry in c.execute("SELECT * FROM forwardKeys WHERE expire <= ?", (time,)): - logger.info(entry[1]) + logger.debug('Forward key: %s' % entry[1]) deleteKeys.append(entry[1]) for key in deleteKeys: - logger.info('Deleting forward key '+ key) + logger.debug('Deleting forward key %s' % key) c.execute("DELETE from forwardKeys where forwardKey = ?", (key,)) conn.commit() conn.close() @@ -120,7 +133,7 @@ class DaemonTools: del self.daemon.cooldownPeer[peer] # Cool down a peer, if we have max connections alive for long enough - if onlinePeerAmount >= self.daemon._core.config.get('peers.max_connect', 10): + if onlinePeerAmount >= self.daemon._core.config.get('peers.max_connect', 10, save = True): finding = True while finding: @@ -164,3 +177,13 @@ class DaemonTools: build += '%s %s' % (amnt_unit, unit) + ('s' if amnt_unit != 1 else '') + ' ' return build.strip() + + def insertDeniableBlock(self): + '''Insert a fake block in order to make it more difficult to track real blocks''' + fakePeer = self.daemon._core._crypto.generatePubKey()[0] + chance = 10 + if secrets.randbelow(chance) == (chance - 1): + data = secrets.token_hex(secrets.randbelow(500) + 1) + self.daemon._core.insertBlock(data, header='pm', encryptType='asym', asymPeer=fakePeer) + self.daemon.decrementThreadCount('insertDeniableBlock') + return \ No newline at end of file diff --git a/onionr/onionrexceptions.py b/onionr/onionrexceptions.py index d450e3ae..a0c468a3 100644 --- a/onionr/onionrexceptions.py +++ b/onionr/onionrexceptions.py @@ -40,6 +40,9 @@ class KeyNotKnown(Exception): class DecryptionError(Exception): pass +class PasswordStrengthError(Exception): + pass + # block exceptions class InvalidMetadata(Exception): pass diff --git a/onionr/onionrgui.py b/onionr/onionrgui.py new file mode 100755 index 00000000..510ea594 --- /dev/null +++ b/onionr/onionrgui.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +from tkinter import * +import core +class OnionrGUI: + def __init__(self): + self.dataDir = "/programming/onionr/data/" + self.root = Tk() + self.root.geometry("450x250") + self.core = core.Core() + menubar = Menu(self.root) + + # create a pulldown menu, and add it to the menu bar + filemenu = Menu(menubar, tearoff=0) + filemenu.add_command(label="Open", command=None) + filemenu.add_command(label="Save", command=None) + filemenu.add_separator() + filemenu.add_command(label="Exit", command=self.root.quit) + menubar.add_cascade(label="File", menu=filemenu) + + settingsmenu = Menu(menubar, tearoff=0) + menubar.add_cascade(label="Settings", menu=settingsmenu) + + helpmenu = Menu(menubar, tearoff=0) + menubar.add_cascade(label="Help", menu=helpmenu) + + self.root.config(menu=menubar) + + self.menuFrame = Frame(self.root) + self.mainButton = Button(self.menuFrame, text="Main View") + self.mainButton.grid(row=0, column=0, padx=0, pady=2, sticky=N+W) + self.tabButton1 = Button(self.menuFrame, text="Mail") + self.tabButton1.grid(row=0, column=1, padx=0, pady=2, sticky=N+W) + self.tabButton2 = Button(self.menuFrame, text="Message Flow") + self.tabButton2.grid(row=0, column=3, padx=0, pady=2, sticky=N+W) + + self.menuFrame.grid(row=0, column=0, padx=2, pady=0, sticky=N+W) + + + self.idFrame = Frame(self.root) + + self.ourIDLabel = Label(self.idFrame, text="ID: ") + self.ourIDLabel.grid(row=2, column=0, padx=1, pady=1, sticky=N+W) + self.ourID = Entry(self.idFrame) + self.ourID.insert(0, self.core._crypto.pubKey) + self.ourID.grid(row=2, column=1, padx=1, pady=1, sticky=N+W) + self.ourID.config(state='readonly') + self.idFrame.grid(row=1, column=0, padx=2, pady=2, sticky=N+W) + + self.syncStatus = Label(self.root, text="Sync Status: 15/100") + self.syncStatus.place(relx=1.0, rely=1.0, anchor=S+E) + self.peerCount = Label(self.root, text="Connected Peers: 3") + self.peerCount.place(relx=0.0, rely=1.0, anchor='sw') + + self.root.wm_title("Onionr") + self.root.mainloop() + return + +OnionrGUI() \ No newline at end of file diff --git a/onionr/onionrplugins.py b/onionr/onionrplugins.py index aa45dcca..7d81e08c 100644 --- a/onionr/onionrplugins.py +++ b/onionr/onionrplugins.py @@ -192,7 +192,7 @@ def get_enabled_plugins(): config.reload() - return config.get('plugins.enabled', list()) + return list(config.get('plugins.enabled', list())) def is_enabled(name): ''' @@ -212,7 +212,7 @@ def get_plugins_folder(name = None, absolute = True): path = _pluginsfolder else: # only allow alphanumeric characters - path = _pluginsfolder + re.sub('[^0-9a-zA-Z]+', '', str(name).lower()) + path = _pluginsfolder + re.sub('[^0-9a-zA-Z_]+', '', str(name).lower()) if absolute is True: path = os.path.abspath(path) diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py index 30ca4b6c..9665a4cb 100644 --- a/onionr/onionrproofs.py +++ b/onionr/onionrproofs.py @@ -30,6 +30,8 @@ def getHashDifficulty(h): for character in h: if character == '0': difficulty += 1 + else: + break return difficulty def hashMeetsDifficulty(h): @@ -38,7 +40,10 @@ def hashMeetsDifficulty(h): ''' config.reload() hashDifficulty = getHashDifficulty(h) - expected = int(config.get('minimum_block_pow')) + try: + expected = int(config.get('general.minimum_block_pow')) + except TypeError: + raise ValueError('Missing general.minimum_block_pow config') if hashDifficulty >= expected: return True else: diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index b9bc3c2c..5267112e 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -33,11 +33,20 @@ def deleteExpiredKeys(coreInst): return class OnionrUser: - def __init__(self, coreInst, publicKey): + def __init__(self, coreInst, publicKey, saveUser=False): + ''' + OnionrUser is an abstraction for "users" of the network. + + Takes an instance of onionr core, a base32 encoded ed25519 public key, and a bool saveUser + saveUser determines if we should add a user to our peer database or not. + ''' self.trust = 0 self._core = coreInst self.publicKey = publicKey + if saveUser: + self._core.addPeer(publicKey) + self.trust = self._core.getPeerInfo(self.publicKey, 'trust') return @@ -71,7 +80,6 @@ class OnionrUser: def forwardEncrypt(self, data): retData = '' forwardKey = self._getLatestForwardKey() - #logger.info('using ' + forwardKey) if self._core._utils.validatePubKey(forwardKey): retData = self._core._crypto.pubKeyEncrypt(data, forwardKey, encodedData=True, anonymous=True) else: @@ -81,10 +89,7 @@ class OnionrUser: def forwardDecrypt(self, encrypted): retData = "" - #logger.error(self.publicKey) - #logger.error(self.getGeneratedForwardKeys(False)) for key in self.getGeneratedForwardKeys(False): - logger.info(encrypted) try: retData = self._core._crypto.pubKeyDecrypt(encrypted, privkey=key[1], anonymous=True, encodedData=True) except nacl.exceptions.CryptoError: diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index b185b0bb..dbcf01e6 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -125,11 +125,11 @@ class OnionrUtils: for adder in newAdderList.split(','): adder = adder.strip() if not adder in self._core.listAdders(randomOrder = False) and adder != self.getMyAddress() and not self._core._blacklist.inBlacklist(adder): - if not config.get('tor.v3_onions') and len(adder) == 62: + if not config.get('tor.v3onions') and len(adder) == 62: continue if self._core.addAddress(adder): # Check if we have the maxmium amount of allowed stored peers - if config.get('peers.max_stored') > len(self._core.listAdders()): + if config.get('peers.max_stored_peers') > len(self._core.listAdders()): logger.info('Added %s to db.' % adder, timestamp = True) retVal = True else: @@ -146,6 +146,8 @@ class OnionrUtils: try: with open('./' + self._core.dataDir + 'hs/hostname', 'r') as hostname: return hostname.read().strip() + except FileNotFoundError: + return "" except Exception as error: logger.error('Failed to read my address.', error = error) return None @@ -163,7 +165,7 @@ class OnionrUtils: hostname = host.read() except FileNotFoundError: return False - payload = 'http://%s:%s/client/?action=%s&token=%s&timingToken=%s' % (hostname, config.get('client.port'), command, config.get('client.hmac'), self.timingToken) + payload = 'http://%s:%s/client/?action=%s&token=%s&timingToken=%s' % (hostname, config.get('client.port'), command, config.get('client.webpassword'), self.timingToken) if data != '': payload += '&data=' + urllib.parse.quote_plus(data) try: @@ -265,20 +267,14 @@ class OnionrUtils: ''' myBlock = Block(blockHash, self._core) if myBlock.isEncrypted: - #pass - logger.warn(myBlock.decrypt()) + myBlock.decrypt() if (myBlock.isEncrypted and myBlock.decrypted) or (not myBlock.isEncrypted): blockType = myBlock.getMetadata('type') # we would use myBlock.getType() here, but it is bugged with encrypted blocks signer = self.bytesToStr(myBlock.signer) valid = myBlock.verifySig() - - logger.info('Checking for fs key') if myBlock.getMetadata('newFSKey') is not None: onionrusers.OnionrUser(self._core, signer).addForwardKey(myBlock.getMetadata('newFSKey')) - else: - logger.warn('FS not used for this encrypted block') - logger.info(myBlock.bmetadata) - + try: if len(blockType) <= 10: self._core.updateBlockInfo(blockHash, 'dataType', blockType) @@ -295,7 +291,6 @@ class OnionrUtils: else: self._core.updateBlockInfo(blockHash, 'expire', expireTime) else: - logger.info(myBlock.isEncrypted) logger.debug('Not processing metadata on encrypted block we cannot decrypt.') def escapeAnsi(self, line): @@ -616,7 +611,7 @@ class OnionrUtils: retData = False return retData - def doGetRequest(self, url, port=0, proxyType='tor'): + def doGetRequest(self, url, port=0, proxyType='tor', ignoreAPI=False): ''' Do a get request through a local tor or i2p instance ''' @@ -635,12 +630,13 @@ class OnionrUtils: proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)} r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30)) # Check server is using same API version as us - try: - response_headers = r.headers - if r.headers['X-API'] != str(API_VERSION): + if not ignoreAPI: + try: + response_headers = r.headers + if r.headers['X-API'] != str(API_VERSION): + raise onionrexceptions.InvalidAPIVersion + except KeyError: raise onionrexceptions.InvalidAPIVersion - except KeyError: - raise onionrexceptions.InvalidAPIVersion retData = r.text except KeyboardInterrupt: raise KeyboardInterrupt @@ -701,7 +697,7 @@ class OnionrUtils: connectURLs = connectTest.read().split(',') for url in connectURLs: - if self.doGetRequest(url, port=torPort) != False: + if self.doGetRequest(url, port=torPort, ignoreAPI=True) != False: retData = True break except FileNotFoundError: diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py index f9fb691f..e40eb4b4 100644 --- a/onionr/static-data/default-plugins/cliui/main.py +++ b/onionr/static-data/default-plugins/cliui/main.py @@ -33,7 +33,8 @@ class OnionrCLIUI: def subCommand(self, command): try: - subprocess.run(["./onionr.py", command]) + #subprocess.run(["./onionr.py", command]) + subprocess.Popen(['./onionr.py', command], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) except KeyboardInterrupt: pass diff --git a/onionr/static-data/default-plugins/flow/main.py b/onionr/static-data/default-plugins/flow/main.py index a7f8842b..6f430e82 100644 --- a/onionr/static-data/default-plugins/flow/main.py +++ b/onionr/static-data/default-plugins/flow/main.py @@ -29,13 +29,19 @@ class OnionrFlow: self.myCore = pluginapi.get_core() self.alreadyOutputed = [] self.flowRunning = False + self.channel = None return def start(self): + logger.warn("Please note: everything said here is public, even if a random channel name is used.") message = "" self.flowRunning = True newThread = threading.Thread(target=self.showOutput) newThread.start() + try: + self.channel = logger.readline("Enter a channel name or none for default:") + except (KeyboardInterrupt, EOFError) as e: + self.flowRunning = False while self.flowRunning: try: message = logger.readline('\nInsert message into flow:').strip().replace('\n', '\\n').replace('\r', '\\r') @@ -43,33 +49,39 @@ class OnionrFlow: pass except KeyboardInterrupt: self.flowRunning = False - if message == "q": - self.flowRunning = False - expireTime = self.myCore._utils.getEpoch() + 43200 - if len(message) > 0: - Block(content = message, type = 'txt', expire=expireTime, core = self.myCore).save() + else: + if message == "q": + self.flowRunning = False + expireTime = self.myCore._utils.getEpoch() + 43200 + if len(message) > 0: + insertBL = Block(content = message, type = 'txt', expire=expireTime, core = self.myCore) + insertBL.setMetadata('ch', self.channel) + insertBL.save() logger.info("Flow is exiting, goodbye") return def showOutput(self): - while self.flowRunning: - for block in Block.getBlocks(type = 'txt', core = self.myCore): - if block.getHash() in self.alreadyOutputed: - continue - if not self.flowRunning: - break - logger.info('\n------------------------', prompt = False) - content = block.getContent() - # Escape new lines, remove trailing whitespace, and escape ansi sequences - content = self.myCore._utils.escapeAnsi(content.replace('\n', '\\n').replace('\r', '\\r').strip()) - logger.info(block.getDate().strftime("%m/%d %H:%M") + ' - ' + logger.colors.reset + content, prompt = False) - self.alreadyOutputed.append(block.getHash()) - try: - time.sleep(5) - except KeyboardInterrupt: - self.flowRunning = False - pass + while type(self.channel) is type(None) and self.flowRunning: + time.sleep(1) + try: + while self.flowRunning: + for block in Block.getBlocks(type = 'txt', core = self.myCore): + if block.getMetadata('ch') != self.channel: + continue + if block.getHash() in self.alreadyOutputed: + continue + if not self.flowRunning: + break + logger.info('\n------------------------', prompt = False) + content = block.getContent() + # Escape new lines, remove trailing whitespace, and escape ansi sequences + content = self.myCore._utils.escapeAnsi(content.replace('\n', '\\n').replace('\r', '\\r').strip()) + logger.info(block.getDate().strftime("%m/%d %H:%M") + ' - ' + logger.colors.reset + content, prompt = False) + self.alreadyOutputed.append(block.getHash()) + time.sleep(5) + except KeyboardInterrupt: + self.flowRunning = False def on_init(api, data = None): ''' diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py index 6b0b5a28..c0d3d38d 100644 --- a/onionr/static-data/default-plugins/metadataprocessor/main.py +++ b/onionr/static-data/default-plugins/metadataprocessor/main.py @@ -42,8 +42,9 @@ def _processUserInfo(api, newBlock): except onionrexceptions.InvalidMetadata: pass else: - api.get_core().setPeerInfo(signer, 'name', peerName) - logger.info('%s is now using the name %s.' % (signer, api.get_utils().escapeAnsi(peerName))) + if signer in self.api.get_core().listPeers(): + api.get_core().setPeerInfo(signer, 'name', peerName) + logger.info('%s is now using the name %s.' % (signer, api.get_utils().escapeAnsi(peerName))) def _processForwardKey(api, myBlock): ''' diff --git a/onionr/static-data/default-plugins/pluginmanager/main.py b/onionr/static-data/default-plugins/pluginmanager/main.py index 45f2b47f..aa5c2887 100644 --- a/onionr/static-data/default-plugins/pluginmanager/main.py +++ b/onionr/static-data/default-plugins/pluginmanager/main.py @@ -151,7 +151,7 @@ def check(): # plugin management def sanitize(name): - return re.sub('[^0-9a-zA-Z]+', '', str(name).lower())[:255] + return re.sub('[^0-9a-zA-Z_]+', '', str(name).lower())[:255] def blockToPlugin(block): try: diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index 5ba910eb..0cf7e2eb 100644 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -79,28 +79,26 @@ class OnionrMail: for blockHash in self.myCore.getBlocksByType('pm'): pmBlocks[blockHash] = Block(blockHash, core=self.myCore) pmBlocks[blockHash].decrypt() - - while choice not in ('-q', 'q', 'quit'): blockCount = 0 - for blockHash in pmBlocks: - if not pmBlocks[blockHash].decrypted: - continue - blockCount += 1 - pmBlockMap[blockCount] = blockHash + for blockHash in pmBlocks: + if not pmBlocks[blockHash].decrypted: + continue + blockCount += 1 + pmBlockMap[blockCount] = blockHash - block = pmBlocks[blockHash] - senderKey = block.signer - try: - senderKey = senderKey.decode() - except AttributeError: - pass - senderDisplay = onionrusers.OnionrUser(self.myCore, senderKey).getName() - if senderDisplay == 'anonymous': - senderDisplay = senderKey + block = pmBlocks[blockHash] + senderKey = block.signer + try: + senderKey = senderKey.decode() + except AttributeError: + pass + senderDisplay = onionrusers.OnionrUser(self.myCore, senderKey).getName() + if senderDisplay == 'anonymous': + senderDisplay = senderKey - blockDate = pmBlocks[blockHash].getDate().strftime("%m/%d %H:%M") - displayList.append('%s. %s - %s: %s' % (blockCount, blockDate, senderDisplay[:12], blockHash)) - #displayList.reverse() + blockDate = pmBlocks[blockHash].getDate().strftime("%m/%d %H:%M") + displayList.append('%s. %s - %s: %s' % (blockCount, blockDate, senderDisplay[:12], blockHash)) + while choice not in ('-q', 'q', 'quit'): for i in displayList: logger.info(i) try: @@ -138,7 +136,9 @@ class OnionrMail: cancel = logger.readline('Press enter to continue to message, or -q to not open the message (recommended).') if cancel != '-q': print(draw_border(self.myCore._utils.escapeAnsi(readBlock.bcontent.decode().strip()))) - logger.readline("Press enter to continue") + reply = logger.readline("Press enter to continue, or enter %s to reply" % ("-r",)) + if reply == "-r": + self.draftMessage(self.myCore._utils.bytesToStr(readBlock.signer,)) return def sentbox(self): @@ -148,29 +148,36 @@ class OnionrMail: entering = True while entering: self.getSentList() - logger.info('Enter block number or -q to return') + logger.info('Enter a block number or -q to return') try: choice = input('>') except (EOFError, KeyboardInterrupt) as e: entering = False else: - if choice == '-q': - entering = False + try: + choice = int(choice) - 1 + except ValueError: + pass else: try: - self.sentboxList[int(choice) - 1] - except IndexError: + self.sentboxList[int(choice)] + except (IndexError, ValueError) as e: logger.warn('Invalid block.') else: - logger.info('Sent to: ' + self.sentMessages[self.sentboxList[int(choice) - 1]][1]) + logger.info('Sent to: ' + self.sentMessages[self.sentboxList[int(choice)]][1]) # Print ansi escaped sent message - logger.info(self.myCore._utils.escapeAnsi(self.sentMessages[self.sentboxList[int(choice) - 1]][0])) + logger.info(self.myCore._utils.escapeAnsi(self.sentMessages[self.sentboxList[int(choice)]][0])) input('Press enter to continue...') + finally: + if choice == '-q': + entering = False return def getSentList(self): count = 1 + self.sentboxList = [] + self.sentMessages = {} for i in self.sentboxTools.listSent(): self.sentboxList.append(i['hash']) self.sentMessages[i['hash']] = (i['message'], i['peer']) @@ -178,28 +185,28 @@ class OnionrMail: logger.info('%s. %s - %s - %s' % (count, i['hash'], i['peer'][:12], i['date'])) count += 1 - def draftMessage(self): + def draftMessage(self, recip=''): message = '' newLine = '' - recip = '' - entering = True - - while entering: - try: - recip = logger.readline('Enter peer address, or q to stop:').strip() - if recip in ('-q', 'q'): - raise EOFError - if not self.myCore._utils.validatePubKey(recip): - raise onionrexceptions.InvalidPubkey('Must be a valid ed25519 base32 encoded public key') - except onionrexceptions.InvalidPubkey: - logger.warn('Invalid public key') - except (KeyboardInterrupt, EOFError): - entering = False + entering = False + if len(recip) == 0: + entering = True + while entering: + try: + recip = logger.readline('Enter peer address, or -q to stop:').strip() + if recip in ('-q', 'q'): + raise EOFError + if not self.myCore._utils.validatePubKey(recip): + raise onionrexceptions.InvalidPubkey('Must be a valid ed25519 base32 encoded public key') + except onionrexceptions.InvalidPubkey: + logger.warn('Invalid public key') + except (KeyboardInterrupt, EOFError): + entering = False + else: + break else: - break - else: - # if -q or ctrl-c/d, exit function here, otherwise we successfully got the public key - return + # if -q or ctrl-c/d, exit function here, otherwise we successfully got the public key + return logger.info('Enter your message, stop by entering -q on a new line.') while newLine != '-q': diff --git a/onionr/static-data/default-plugins/pms/sentboxdb.py b/onionr/static-data/default-plugins/pms/sentboxdb.py index f2328ccb..2d6207e6 100644 --- a/onionr/static-data/default-plugins/pms/sentboxdb.py +++ b/onionr/static-data/default-plugins/pms/sentboxdb.py @@ -29,7 +29,7 @@ class SentBox: self.cursor = self.conn.cursor() self.core = mycore return - + def createDB(self): conn = sqlite3.connect(self.dbLocation) cursor = conn.cursor() @@ -48,15 +48,15 @@ class SentBox: for entry in self.cursor.execute('SELECT * FROM sent;'): retData.append({'hash': entry[0], 'peer': entry[1], 'message': entry[2], 'date': entry[3]}) return retData - + def addToSent(self, blockID, peer, message): args = (blockID, peer, message, self.core._utils.getEpoch()) self.cursor.execute('INSERT INTO sent VALUES(?, ?, ?, ?)', args) self.conn.commit() return - + def removeSent(self, blockID): args = (blockID,) self.cursor.execute('DELETE FROM sent where hash=?', args) self.conn.commit() - return \ No newline at end of file + return diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 82dbece0..5003d73b 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -4,14 +4,9 @@ "display_header" : true, "minimum_block_pow": 5, "minimum_send_pow": 5, - - "minimum_block_pow": 5, - "minimum_send_pow": 5, - - "direct_connect" : { - "respond" : true, - "execute_callbacks" : true - } + "socket_servers": false, + "security_level": 0, + "public_key": "" }, "www" : { @@ -52,7 +47,7 @@ "verbosity" : "default", "file": { - "output": false, + "output": true, "path": "data/output.log" }, @@ -63,7 +58,7 @@ }, "tor" : { - "v3onions" : false + "v3onions" : true }, "i2p" : { @@ -86,7 +81,7 @@ }, "timers" : { - "lookup_blocks" : 25, - "get_blocks" : 30 + "lookupBlocks" : 25, + "getBlocks" : 30 } } diff --git a/RUN-LINUX.sh b/run-linux similarity index 100% rename from RUN-LINUX.sh rename to run-linux diff --git a/RUN-WINDOWS.bat b/run-windows.bat similarity index 100% rename from RUN-WINDOWS.bat rename to run-windows.bat