From a4d6dc5fa5827e19b4bbb9eceb4ceaae59d9eefb Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 7 May 2019 12:56:20 -0500 Subject: [PATCH] bug fixes and refactoring --- README.md | 12 +-- onionr/api.py | 86 ++----------------- onionr/communicatorutils/onionrdaemontools.py | 7 +- onionr/etc/onionrvalues.py | 3 +- onionr/httpapi/miscpublicapi/__init__.py | 6 ++ onionr/httpapi/miscpublicapi/announce.py | 63 ++++++++++++++ onionr/httpapi/miscpublicapi/getblocks.py | 50 +++++++++++ onionr/httpapi/miscpublicapi/upload.py | 43 ++++++++++ onionr/onionr.py | 2 + onionr/static-data/www/private/index.html | 12 +-- 10 files changed, 189 insertions(+), 95 deletions(-) create mode 100644 onionr/httpapi/miscpublicapi/__init__.py create mode 100644 onionr/httpapi/miscpublicapi/announce.py create mode 100644 onionr/httpapi/miscpublicapi/getblocks.py create mode 100644 onionr/httpapi/miscpublicapi/upload.py diff --git a/README.md b/README.md index b63fe020..2fb180e0 100755 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

- Anonymous P2P storage network 🕵️ + Anonymous P2P communication network 🕵️

(***pre-alpha & experimental, not well tested or easy to use yet***) @@ -19,7 +19,7 @@ # About -Onionr is a decentralized, peer-to-peer data storage network, designed to be anonymous and resistant to (meta)data analysis and spam/disruption. +Onionr is a decentralized, peer-to-peer communication and storage network, designed to be anonymous and resistant to (meta)data analysis and spam/disruption. Onionr stores data in independent packages referred to as 'blocks'. The blocks are synced to all other nodes in the network. Blocks and user IDs cannot be easily proven to have been created by a particular user. Even if there is enough evidence to believe that a specific user created a block, nodes still operate behind Tor or I2P and as such cannot be trivially unmasked. @@ -29,7 +29,7 @@ Onionr can be used for mail, as a social network, instant messenger, file sharin The whitepaper (subject to change prior to first alpha release) is available [here](docs/whitepaper.md). -![Tor stinks slide image](docs/tor-stinks-02.png) +![node web illustration](docs/onionr-web.png) ## Main Features @@ -54,7 +54,7 @@ Friend/contact manager Encrypted, metadata-masking mail application screenshot -Encrypted, metadata-masking mail application. Perhaps the first distributed mail system to have basic forward secrecy. +Encrypted, metadata-masking mail application. One of the first distributed mail systems to have basic forward secrecy. # Documentation @@ -115,4 +115,6 @@ The 'open source badge' is by Maik Ellerbrock and is licensed under a Creative C The Onionr logo was created by [Anhar Ismail](https://github.com/anharismail) under the [Creative Commons Attribution 4.0 International License](https://creativecommons.org/licenses/by/4.0/). -If you modify and redistribute our code ("forking"), please use a different logo and project name to avoid confusion. Please do not use our logo in a way that makes it seem like we endorse you without permission. \ No newline at end of file +If you modify and redistribute our code ("forking"), please use a different logo and project name to avoid confusion. Please do not use our logo in a way that makes it seem like we endorse you without permission. + +![Tor stinks slide image](docs/tor-stinks-02.png) diff --git a/onionr/api.py b/onionr/api.py index bf42348a..11197a0a 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -26,7 +26,7 @@ import core from onionrblockapi import Block import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config import httpapi -from httpapi import friendsapi, simplecache, profilesapi, configapi +from httpapi import friendsapi, simplecache, profilesapi, configapi, miscpublicapi from onionrservices import httpheaders import onionr @@ -111,36 +111,12 @@ class PublicAPI: @app.route('/getblocklist') def getBlockList(): - # Provide a list of our blocks, with a date offset - dateAdjust = request.args.get('date') - bList = clientAPI._core.getBlockList(dateRec=dateAdjust) - if config.get('general.hide_created_blocks', True): - for b in self.hideBlocks: - if b in bList: - # Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block - bList.remove(b) - return Response('\n'.join(bList)) + return httpapi.miscpublicapi.public_block_list(clientAPI, self, request) @app.route('/getdata/') def getBlockData(name): # Share data for a block if we have it - resp = '' - data = name - if clientAPI._utils.validateHash(data): - if not config.get('general.hide_created_blocks', True) or data not in self.hideBlocks: - if data in clientAPI._core.getBlockList(): - block = clientAPI.getBlockData(data, raw=True) - try: - block = block.encode() - except AttributeError: - abort(404) - block = clientAPI._core._utils.strToBytes(block) - resp = block - #resp = base64.b64encode(block).decode() - if len(resp) == 0: - abort(404) - resp = "" - return Response(resp, mimetype='application/octet-stream') + return httpapi.miscpublicapi.public_get_block_data(clientAPI, self, name) @app.route('/www/') def wwwPublic(path): @@ -163,40 +139,7 @@ class PublicAPI: @app.route('/announce', methods=['post']) def acceptAnnounce(): - resp = 'failure' - powHash = '' - randomData = '' - newNode = '' - ourAdder = clientAPI._core.hsAddress.encode() - try: - newNode = request.form['node'].encode() - except KeyError: - logger.warn('No node specified for upload') - pass - else: - try: - randomData = request.form['random'] - randomData = base64.b64decode(randomData) - except KeyError: - logger.warn('No random data specified for upload') - else: - nodes = newNode + clientAPI._core.hsAddress.encode() - nodes = clientAPI._core._crypto.blake2bHash(nodes) - powHash = clientAPI._core._crypto.blake2bHash(randomData + nodes) - try: - powHash = powHash.decode() - except AttributeError: - pass - if powHash.startswith('00000'): - newNode = clientAPI._core._utils.bytesToStr(newNode) - if clientAPI._core._utils.validateID(newNode) and not newNode in clientAPI._core.onionrInst.communicatorInst.newPeers: - clientAPI._core.onionrInst.communicatorInst.newPeers.append(newNode) - resp = 'Success' - else: - logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash) - resp = Response(resp) - if resp == 'failure': - return resp, 406 + resp = httpapi.miscpublicapi.announce(clientAPI, request) return resp @app.route('/upload', methods=['post']) @@ -204,26 +147,7 @@ class PublicAPI: '''Accept file uploads. In the future this will be done more often than on creation to speed up block sync ''' - resp = 'failure' - try: - data = request.form['block'] - except KeyError: - logger.warn('No block specified for upload') - pass - else: - if sys.getsizeof(data) < 100000000: - try: - if blockimporter.importBlockFromData(data, clientAPI._core): - resp = 'success' - else: - logger.warn('Error encountered importing uploaded block') - except onionrexceptions.BlacklistedBlock: - logger.debug('uploaded block is blacklisted') - pass - if resp == 'failure': - abort(400) - resp = Response(resp) - return resp + return httpapi.miscpublicapi.upload(clientAPI, request) # Set instances, then startup our public api server clientAPI.setPublicAPIInstance(self) diff --git a/onionr/communicatorutils/onionrdaemontools.py b/onionr/communicatorutils/onionrdaemontools.py index 85c97747..54ee2703 100755 --- a/onionr/communicatorutils/onionrdaemontools.py +++ b/onionr/communicatorutils/onionrdaemontools.py @@ -23,6 +23,8 @@ import base64, sqlite3, os from dependencies import secrets from utils import netutils from onionrusers import onionrusers +from etc import onionrvalues +ov = onionrvalues.OnionrValues() class DaemonTools: ''' @@ -64,7 +66,8 @@ class DaemonTools: if ourID != 1: #TODO: Extend existingRand for i2p existingRand = self.daemon._core.getAddressInfo(peer, 'powValue') - if type(existingRand) is type(None): + # Reset existingRand if it no longer meets the minimum POW + if type(existingRand) is type(None) or not existingRand.endswith(b'0' * ov.announce_pow): existingRand = '' if peer in self.announceCache: @@ -73,7 +76,7 @@ class DaemonTools: data['random'] = existingRand else: self.announceProgress[peer] = True - proof = onionrproofs.DataPOW(combinedNodes, forceDifficulty=5) + proof = onionrproofs.DataPOW(combinedNodes, forceDifficulty=ov.announce_pow) del self.announceProgress[peer] try: data['random'] = base64.b64encode(proof.waitForResult()[1]) diff --git a/onionr/etc/onionrvalues.py b/onionr/etc/onionrvalues.py index 9f636147..f55f5ac4 100755 --- a/onionr/etc/onionrvalues.py +++ b/onionr/etc/onionrvalues.py @@ -22,4 +22,5 @@ class OnionrValues: def __init__(self): self.passwordLength = 20 self.blockMetadataLengths = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'pow': 1000, 'encryptType': 4, 'expire': 14} #TODO properly refine values to minimum needed - self.default_expire = 2592000 \ No newline at end of file + self.default_expire = 2592000 + self.announce_pow = 5 \ No newline at end of file diff --git a/onionr/httpapi/miscpublicapi/__init__.py b/onionr/httpapi/miscpublicapi/__init__.py new file mode 100644 index 00000000..f03d3808 --- /dev/null +++ b/onionr/httpapi/miscpublicapi/__init__.py @@ -0,0 +1,6 @@ +from . import announce, upload, getblocks + +announce = announce.handle_announce # endpoint handler for accepting peer announcements +upload = upload.accept_upload # endpoint handler for accepting public uploads +public_block_list = getblocks.get_public_block_list # endpoint handler for getting block lists +public_get_block_data = getblocks.get_block_data # endpoint handler for responding to peers requests for block data \ No newline at end of file diff --git a/onionr/httpapi/miscpublicapi/announce.py b/onionr/httpapi/miscpublicapi/announce.py new file mode 100644 index 00000000..a6d9513f --- /dev/null +++ b/onionr/httpapi/miscpublicapi/announce.py @@ -0,0 +1,63 @@ +''' + Onionr - P2P Microblogging Platform & Social network + + Handle announcements to the public API server +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import base64 +from flask import Response +import logger +from etc import onionrvalues +def handle_announce(clientAPI, request): + ''' + accept announcement posts, validating POW + clientAPI should be an instance of the clientAPI server running, request is a instance of a flask request + ''' + resp = 'failure' + powHash = '' + randomData = '' + newNode = '' + ourAdder = clientAPI._core.hsAddress.encode() + try: + newNode = request.form['node'].encode() + except KeyError: + logger.warn('No node specified for upload') + pass + else: + try: + randomData = request.form['random'] + randomData = base64.b64decode(randomData) + except KeyError: + logger.warn('No random data specified for upload') + else: + nodes = newNode + clientAPI._core.hsAddress.encode() + nodes = clientAPI._core._crypto.blake2bHash(nodes) + powHash = clientAPI._core._crypto.blake2bHash(randomData + nodes) + try: + powHash = powHash.decode() + except AttributeError: + pass + if powHash.startswith('0' * onionrvalues.OnionrValues().announce_pow): + newNode = clientAPI._core._utils.bytesToStr(newNode) + if clientAPI._core._utils.validateID(newNode) and not newNode in clientAPI._core.onionrInst.communicatorInst.newPeers: + clientAPI._core.onionrInst.communicatorInst.newPeers.append(newNode) + resp = 'Success' + else: + logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash) + resp = Response(resp) + if resp == 'failure': + return resp, 406 + return resp \ No newline at end of file diff --git a/onionr/httpapi/miscpublicapi/getblocks.py b/onionr/httpapi/miscpublicapi/getblocks.py new file mode 100644 index 00000000..1ab8f949 --- /dev/null +++ b/onionr/httpapi/miscpublicapi/getblocks.py @@ -0,0 +1,50 @@ +''' + Onionr - P2P Microblogging Platform & Social network + + Public endpoints to get block data and lists +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +from flask import Response, abort +import config +def get_public_block_list(clientAPI, publicAPI, request): + # Provide a list of our blocks, with a date offset + dateAdjust = request.args.get('date') + bList = clientAPI._core.getBlockList(dateRec=dateAdjust) + if config.get('general.hide_created_blocks', True): + for b in publicAPI.hideBlocks: + if b in bList: + # Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block + bList.remove(b) + return Response('\n'.join(bList)) + +def get_block_data(clientAPI, publicAPI, data): + '''data is the block hash in hex''' + resp = '' + if clientAPI._utils.validateHash(data): + if not config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks: + if data in clientAPI._core.getBlockList(): + block = clientAPI.getBlockData(data, raw=True) + try: + block = block.encode() # Encode in case data is binary + except AttributeError: + abort(404) + block = clientAPI._core._utils.strToBytes(block) + resp = block + if len(resp) == 0: + abort(404) + resp = "" + # Has to be octet stream, otherwise binary data fails hash check + return Response(resp, mimetype='application/octet-stream') \ No newline at end of file diff --git a/onionr/httpapi/miscpublicapi/upload.py b/onionr/httpapi/miscpublicapi/upload.py new file mode 100644 index 00000000..0561ed08 --- /dev/null +++ b/onionr/httpapi/miscpublicapi/upload.py @@ -0,0 +1,43 @@ +''' + Onionr - P2P Microblogging Platform & Social network + + Accept block uploads to the public API server +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import sys +from flask import Response, abort +import blockimporter, onionrexceptions, logger +def accept_upload(clientAPI, request): + resp = 'failure' + try: + data = request.form['block'] + except KeyError: + logger.warn('No block specified for upload') + pass + else: + if sys.getsizeof(data) < 100000000: + try: + if blockimporter.importBlockFromData(data, clientAPI._core): + resp = 'success' + else: + logger.warn('Error encountered importing uploaded block') + except onionrexceptions.BlacklistedBlock: + logger.debug('uploaded block is blacklisted') + pass + if resp == 'failure': + abort(400) + resp = Response(resp) + return resp \ No newline at end of file diff --git a/onionr/onionr.py b/onionr/onionr.py index 532c87d5..1c19c6d5 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -401,6 +401,8 @@ class Onionr: ''' Starts the Onionr daemon ''' + if config.get('general.dev_mode', False): + override = True commands.daemonlaunch.start(self, input, override) def setClientAPIInst(self, inst): diff --git a/onionr/static-data/www/private/index.html b/onionr/static-data/www/private/index.html index 32c4c497..7b1aa1e5 100644 --- a/onionr/static-data/www/private/index.html +++ b/onionr/static-data/www/private/index.html @@ -29,12 +29,12 @@

Mail - Friend Manager - Boards

- Edit Configuration -
-

Warning: Some values can be dangerous to change. Use caution.

-
- - + Edit Configuration +
+

Warning: Some values can be dangerous to change. Use caution.

+
+ +

Stats