bug fixes and refactoring

README.md

@@ -5,7 +5,7 @@
 </p>
 
 <p align="center">
-    Anonymous P2P storage network 🕵️
+    Anonymous P2P communication network 🕵️
 </p>
 
 (***pre-alpha & experimental, not well tested or easy to use yet***)
@@ -19,7 +19,7 @@
 
 # About
 
-Onionr is a decentralized, peer-to-peer data storage network, designed to be anonymous and resistant to (meta)data analysis and spam/disruption.
+Onionr is a decentralized, peer-to-peer communication and storage network, designed to be anonymous and resistant to (meta)data analysis and spam/disruption.
 
 Onionr stores data in independent packages referred to as 'blocks'. The blocks are synced to all other nodes in the network. Blocks and user IDs cannot be easily proven to have been created by a particular user. Even if there is enough evidence to believe that a specific user created a block, nodes still operate behind Tor or I2P and as such cannot be trivially unmasked.
 
@@ -29,7 +29,7 @@ Onionr can be used for mail, as a social network, instant messenger, file sharin
 
 The whitepaper (subject to change prior to first alpha release) is available [here](docs/whitepaper.md).
 
-![Tor stinks slide image](docs/tor-stinks-02.png)
+![node web illustration](docs/onionr-web.png)
 
 ## Main Features
 
@@ -54,7 +54,7 @@ Friend/contact manager
 
 <img alt='Encrypted, metadata-masking mail application screenshot' src='docs/onionr-3.png' width=600>
 
-Encrypted, metadata-masking mail application. Perhaps the first distributed mail system to have basic forward secrecy.
+Encrypted, metadata-masking mail application. One of the first distributed mail systems to have basic forward secrecy.
 
 # Documentation
 
@@ -115,4 +115,6 @@ The 'open source badge' is by Maik Ellerbrock and is licensed under a Creative C
 
 The Onionr logo was created by [Anhar Ismail](https://github.com/anharismail) under the [Creative Commons Attribution 4.0 International License](https://creativecommons.org/licenses/by/4.0/). 
 
-If you modify and redistribute our code ("forking"), please use a different logo and project name to avoid confusion. Please do not use our logo in a way that makes it seem like we endorse you without permission.
+If you modify and redistribute our code ("forking"), please use a different logo and project name to avoid confusion. Please do not use our logo in a way that makes it seem like we endorse you without permission.
+
+![Tor stinks slide image](docs/tor-stinks-02.png)

onionr/api.py

@@ -26,7 +26,7 @@ import core
 from onionrblockapi import Block
 import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config
 import httpapi
-from httpapi import friendsapi, simplecache, profilesapi, configapi
+from httpapi import friendsapi, simplecache, profilesapi, configapi, miscpublicapi
 from onionrservices import httpheaders
 import onionr
 
@@ -111,36 +111,12 @@ class PublicAPI:
 
         @app.route('/getblocklist')
         def getBlockList():
-            # Provide a list of our blocks, with a date offset
-            dateAdjust = request.args.get('date')
-            bList = clientAPI._core.getBlockList(dateRec=dateAdjust)
-            if config.get('general.hide_created_blocks', True):
-                for b in self.hideBlocks:
-                    if b in bList:
-                        # Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block
-                        bList.remove(b)
-            return Response('\n'.join(bList))
+            return httpapi.miscpublicapi.public_block_list(clientAPI, self, request)
 
         @app.route('/getdata/<name>')
         def getBlockData(name):
             # Share data for a block if we have it
-            resp = ''
-            data = name
-            if clientAPI._utils.validateHash(data):
-                if not config.get('general.hide_created_blocks', True) or data not in self.hideBlocks:
-                    if data in clientAPI._core.getBlockList():
-                        block = clientAPI.getBlockData(data, raw=True)
-                        try:
-                            block = block.encode()
-                        except AttributeError:
-                            abort(404)
-                        block = clientAPI._core._utils.strToBytes(block)
-                        resp = block
-                        #resp = base64.b64encode(block).decode()
-            if len(resp) == 0:
-                abort(404)
-                resp = ""
-            return Response(resp, mimetype='application/octet-stream')
+            return httpapi.miscpublicapi.public_get_block_data(clientAPI, self, name)
 
         @app.route('/www/<path:path>')
         def wwwPublic(path):
@@ -163,40 +139,7 @@ class PublicAPI:
         
         @app.route('/announce', methods=['post'])
         def acceptAnnounce():
-            resp = 'failure'
-            powHash = ''
-            randomData = ''
-            newNode = ''
-            ourAdder = clientAPI._core.hsAddress.encode()
-            try:
-                newNode = request.form['node'].encode()
-            except KeyError:
-                logger.warn('No node specified for upload')
-                pass
-            else:
-                try:
-                    randomData = request.form['random']
-                    randomData = base64.b64decode(randomData)
-                except KeyError:
-                    logger.warn('No random data specified for upload')
-                else:
-                    nodes = newNode + clientAPI._core.hsAddress.encode()
-                    nodes = clientAPI._core._crypto.blake2bHash(nodes)
-                    powHash = clientAPI._core._crypto.blake2bHash(randomData + nodes)
-                    try:
-                        powHash = powHash.decode()
-                    except AttributeError:
-                        pass
-                    if powHash.startswith('00000'):
-                        newNode = clientAPI._core._utils.bytesToStr(newNode)
-                        if clientAPI._core._utils.validateID(newNode) and not newNode in clientAPI._core.onionrInst.communicatorInst.newPeers:
-                            clientAPI._core.onionrInst.communicatorInst.newPeers.append(newNode)
-                            resp = 'Success'
-                    else:
-                        logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash)
-            resp = Response(resp)
-            if resp == 'failure':
-                return resp, 406
+            resp = httpapi.miscpublicapi.announce(clientAPI, request)
             return resp
 
         @app.route('/upload', methods=['post'])
@@ -204,26 +147,7 @@ class PublicAPI:
             '''Accept file uploads. In the future this will be done more often than on creation 
             to speed up block sync
             '''
-            resp = 'failure'
-            try:
-                data = request.form['block']
-            except KeyError:
-                logger.warn('No block specified for upload')
-                pass
-            else:
-                if sys.getsizeof(data) < 100000000:
-                    try:
-                        if blockimporter.importBlockFromData(data, clientAPI._core):
-                            resp = 'success'
-                        else:
-                            logger.warn('Error encountered importing uploaded block')
-                    except onionrexceptions.BlacklistedBlock:
-                        logger.debug('uploaded block is blacklisted')
-                        pass
-            if resp == 'failure':
-                abort(400)
-            resp = Response(resp)
-            return resp
+            return httpapi.miscpublicapi.upload(clientAPI, request)
 
         # Set instances, then startup our public api server
         clientAPI.setPublicAPIInstance(self)

onionr/communicatorutils/onionrdaemontools.py

@@ -23,6 +23,8 @@ import base64, sqlite3, os
 from dependencies import secrets
 from utils import netutils
 from onionrusers import onionrusers
+from etc import onionrvalues
+ov = onionrvalues.OnionrValues()
 
 class DaemonTools:
     '''
@@ -64,7 +66,8 @@ class DaemonTools:
                 if ourID != 1:
                     #TODO: Extend existingRand for i2p
                     existingRand = self.daemon._core.getAddressInfo(peer, 'powValue')
-                    if type(existingRand) is type(None):
+                    # Reset existingRand if it no longer meets the minimum POW
+                    if type(existingRand) is type(None) or not existingRand.endswith(b'0' * ov.announce_pow):
                         existingRand = ''
 
                 if peer in self.announceCache:
@@ -73,7 +76,7 @@ class DaemonTools:
                     data['random'] = existingRand
                 else:
                     self.announceProgress[peer] = True
-                    proof = onionrproofs.DataPOW(combinedNodes, forceDifficulty=5)
+                    proof = onionrproofs.DataPOW(combinedNodes, forceDifficulty=ov.announce_pow)
                     del self.announceProgress[peer]
                     try:
                         data['random'] = base64.b64encode(proof.waitForResult()[1])

onionr/etc/onionrvalues.py

@@ -22,4 +22,5 @@ class OnionrValues:
     def __init__(self):
         self.passwordLength = 20
         self.blockMetadataLengths = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'pow': 1000, 'encryptType': 4, 'expire': 14} #TODO properly refine values to minimum needed
-        self.default_expire = 2592000
+        self.default_expire = 2592000
+        self.announce_pow = 5

onionr/httpapi/miscpublicapi/__init__.py

@@ -0,0 +1,6 @@
+from . import announce, upload, getblocks
+
+announce = announce.handle_announce # endpoint handler for accepting peer announcements
+upload = upload.accept_upload # endpoint handler for accepting public uploads
+public_block_list = getblocks.get_public_block_list # endpoint handler for getting block lists
+public_get_block_data = getblocks.get_block_data # endpoint handler for responding to peers requests for block data

onionr/httpapi/miscpublicapi/announce.py

@@ -0,0 +1,63 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network
+
+    Handle announcements to the public API server
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import base64
+from flask import Response
+import logger
+from etc import onionrvalues
+def handle_announce(clientAPI, request):
+    '''
+    accept announcement posts, validating POW
+    clientAPI should be an instance of the clientAPI server running, request is a instance of a flask request
+    '''
+    resp = 'failure'
+    powHash = ''
+    randomData = ''
+    newNode = ''
+    ourAdder = clientAPI._core.hsAddress.encode()
+    try:
+        newNode = request.form['node'].encode()
+    except KeyError:
+        logger.warn('No node specified for upload')
+        pass
+    else:
+        try:
+            randomData = request.form['random']
+            randomData = base64.b64decode(randomData)
+        except KeyError:
+            logger.warn('No random data specified for upload')
+        else:
+            nodes = newNode + clientAPI._core.hsAddress.encode()
+            nodes = clientAPI._core._crypto.blake2bHash(nodes)
+            powHash = clientAPI._core._crypto.blake2bHash(randomData + nodes)
+            try:
+                powHash = powHash.decode()
+            except AttributeError:
+                pass
+            if powHash.startswith('0' * onionrvalues.OnionrValues().announce_pow):
+                newNode = clientAPI._core._utils.bytesToStr(newNode)
+                if clientAPI._core._utils.validateID(newNode) and not newNode in clientAPI._core.onionrInst.communicatorInst.newPeers:
+                    clientAPI._core.onionrInst.communicatorInst.newPeers.append(newNode)
+                    resp = 'Success'
+            else:
+                logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash)
+    resp = Response(resp)
+    if resp == 'failure':
+        return resp, 406
+    return resp

onionr/httpapi/miscpublicapi/getblocks.py

@@ -0,0 +1,50 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network
+
+    Public endpoints to get block data and lists
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from flask import Response, abort
+import config
+def get_public_block_list(clientAPI, publicAPI, request):
+    # Provide a list of our blocks, with a date offset
+    dateAdjust = request.args.get('date')
+    bList = clientAPI._core.getBlockList(dateRec=dateAdjust)
+    if config.get('general.hide_created_blocks', True):
+        for b in publicAPI.hideBlocks:
+            if b in bList:
+                # Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block
+                bList.remove(b)
+    return Response('\n'.join(bList))
+
+def get_block_data(clientAPI, publicAPI, data):
+    '''data is the block hash in hex'''
+    resp = ''
+    if clientAPI._utils.validateHash(data):
+        if not config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks:
+            if data in clientAPI._core.getBlockList():
+                block = clientAPI.getBlockData(data, raw=True)
+                try:
+                    block = block.encode() # Encode in case data is binary
+                except AttributeError:
+                    abort(404)
+                block = clientAPI._core._utils.strToBytes(block)
+                resp = block
+    if len(resp) == 0:
+        abort(404)
+        resp = ""
+    # Has to be octet stream, otherwise binary data fails hash check
+    return Response(resp, mimetype='application/octet-stream')

onionr/httpapi/miscpublicapi/upload.py

@@ -0,0 +1,43 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network
+
+    Accept block uploads to the public API server
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sys
+from flask import Response, abort
+import blockimporter, onionrexceptions, logger
+def accept_upload(clientAPI, request):
+    resp = 'failure'
+    try:
+        data = request.form['block']
+    except KeyError:
+        logger.warn('No block specified for upload')
+        pass
+    else:
+        if sys.getsizeof(data) < 100000000:
+            try:
+                if blockimporter.importBlockFromData(data, clientAPI._core):
+                    resp = 'success'
+                else:
+                    logger.warn('Error encountered importing uploaded block')
+            except onionrexceptions.BlacklistedBlock:
+                logger.debug('uploaded block is blacklisted')
+                pass
+    if resp == 'failure':
+        abort(400)
+    resp = Response(resp)
+    return resp

onionr/onionr.py

@@ -401,6 +401,8 @@ class Onionr:
         '''
             Starts the Onionr daemon
         '''
+        if config.get('general.dev_mode', False):
+            override = True
         commands.daemonlaunch.start(self, input, override)
 
     def setClientAPIInst(self, inst):

onionr/static-data/www/private/index.html

@@ -29,12 +29,12 @@
             <br><br><a class='idLink' href='/mail/'>Mail</a> - <a class='idLink' href='/friends/'>Friend Manager</a> - <a class='idLink' href='/board/'>Boards</a>
             <br><br><hr>
             <details class='configArea'>
-                    <summary><b>Edit Configuration</b></summary>
-                    <br>
-                    <p><em>Warning: </em><b>Some values can be dangerous to change. Use caution.</b></p>
-                    <br>
-                    <textarea class='configEditor'></textarea>
-                    <button class='saveConfig successBtn'>Save Config</button>
+                <summary><b>Edit Configuration</b></summary>
+                <br>
+                <p><em>Warning: </em><b>Some values can be dangerous to change. Use caution.</b></p>
+                <br>
+                <textarea class='configEditor'></textarea>
+                <button class='saveConfig successBtn'>Save Config</button>
             </details>
             <hr>
             <h2>Stats</h2>