Merge branch 'node-pow' into 'master'

Reverse block insertion and other bug fixes

See merge request beardog/Onionr!4

onionr/api.py

@@ -24,7 +24,7 @@ from gevent.wsgi import WSGIServer
 import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config
 from core import Core
 from onionrblockapi import Block
-import onionrutils, onionrcrypto
+import onionrutils, onionrcrypto, blockimporter
 
 class API:
     '''
@@ -141,9 +141,6 @@ class API:
                 resp = Response('Goodbye')
             elif action == 'ping':
                 resp = Response('pong')
-            elif action == 'stats':
-                resp = Response('me_irl')
-                raise Exception
             elif action == 'site':
                 block = data
                 siteData = self._core.getData(data)
@@ -175,6 +172,24 @@ class API:
                 resp = Response("")
             return resp
 
+        @app.route('/public/upload/', methods=['POST'])
+        def blockUpload():
+            self.validateHost('public')
+            resp = 'failure'
+            try:
+                data = request.form['block']
+            except KeyError:
+                logger.warn('No block specified for upload')
+                pass
+            else:
+                if sys.getsizeof(data) < 100000000:
+                    if blockimporter.importBlockFromData(data, self._core):
+                        resp = 'success'
+                    else:
+                        logger.warn('Error encountered importing uploaded block')
+
+            resp = Response(resp)
+            return resp
         @app.route('/public/')
         def public_handler():
             # Public means it is publicly network accessible
@@ -198,6 +213,7 @@ class API:
                 resp = Response('\n'.join(self._core.getBlockList()))
             elif action == 'directMessage':
                 resp = Response(self._core.handle_direct_connection(data))
+
             elif action == 'announce':
                 if data != '':
                     # TODO: require POW for this

onionr/blockimporter.py

@@ -0,0 +1,40 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network
+
+    Import block data and save it
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import core, onionrexceptions, logger
+def importBlockFromData(content, coreInst):
+    retData = False
+    if not isinstance(coreInst, core.Core):
+        raise Exception("coreInst must be an Onionr core instance")
+
+    try:
+        content = content.encode()
+    except AttributeError:
+        pass
+
+    metas = coreInst._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata
+    metadata = metas[0]
+    if coreInst._utils.validateMetadata(metadata): # check if metadata is valid
+        if coreInst._crypto.verifyPow(content): # check if POW is enough/correct
+            logger.info('Block passed proof, saving.')
+            blockHash = coreInst.setData(content)
+            blockHash = coreInst.addToBlockDB(blockHash, dataSaved=True)
+            coreInst._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
+            retData = True
+    return retData

onionr/communicator2.py

@@ -36,6 +36,8 @@ class OnionrCommunicatorDaemon:
         # intalize NIST beacon salt and time
         self.nistSaltTimestamp = 0
         self.powSalt = 0
+        
+        self.blockToUpload = ''
 
         # loop time.sleep delay in seconds
         self.delay = 1
@@ -84,7 +86,7 @@ class OnionrCommunicatorDaemon:
         OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True)
 
         # set loop to execute instantly to load up peer pool (replaced old pool init wait)
-        peerPoolTimer.count = (peerPoolTimer.frequency - 1) 
+        peerPoolTimer.count = (peerPoolTimer.frequency - 1)
 
         # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking
         try:
@@ -101,7 +103,7 @@ class OnionrCommunicatorDaemon:
         logger.info('Goodbye.')
         self._core._utils.localCommand('shutdown')
         time.sleep(0.5)
-    
+
     def lookupKeys(self):
         '''Lookup new keys'''
         logger.debug('Looking up new keys...')
@@ -111,7 +113,6 @@ class OnionrCommunicatorDaemon:
             peer = self.pickOnlinePeer()
             newKeys = self.peerAction(peer, action='kex')
             self._core._utils.mergeKeys(newKeys)
-
         self.decrementThreadCount('lookupKeys')
         return
 
@@ -196,7 +197,7 @@ class OnionrCommunicatorDaemon:
                         pass
                     logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash)
                 self.blockQueue.remove(blockHash) # remove from block queue both if success or false
-                self.currentDownloading.remove(blockHash)
+            self.currentDownloading.remove(blockHash)
         self.decrementThreadCount('getBlocks')
         return
 
@@ -339,10 +340,32 @@ class OnionrCommunicatorDaemon:
                 for i in self.timers:
                     if i.timerFunction.__name__ == 'lookupKeys':
                         i.count = (i.frequency - 1)
+            elif cmd[0] == 'uploadBlock':
+                self.blockToUpload = cmd[1]
+                threading.Thread(target=self.uploadBlock).start()
             else:
                 logger.info('Recieved daemonQueue command:' + cmd[0])
         self.decrementThreadCount('daemonCommands')
 
+    def uploadBlock(self):
+        triedPeers = []
+        if not self._core._utils.validateHash(self.blockToUpload):
+            logger.warn('Requested to upload invalid block')
+            return
+        for i in range(max(len(self.onlinePeers), 2)):
+            peer = self.pickOnlinePeer()
+            if peer in triedPeers:
+                continue
+            triedPeers.append(peer)
+            url = 'http://' + peer + '/public/upload/'
+            data = {'block': block.Block(self.blockToUpload).getRaw()}
+            if peer.endswith('.onion'):
+                proxyType = 'tor'
+            elif peer.endswith('.i2p'):
+                proxyType = 'i2p'
+            logger.info("Uploading block")
+            self._core._utils.doPostRequest(url, data=data, proxyType=proxyType)
+
     def announce(self, peer):
         '''Announce to peers our address'''
         announceCount = 0

onionr/core.py

@@ -41,10 +41,12 @@ class Core:
             self.blockDataLocation = 'data/blocks/'
             self.addressDB = 'data/address.db'
             self.hsAdder = ''
-
             self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
             self.bootstrapList = []
             self.requirements = onionrvalues.OnionrValues()
+            self.torPort = torPort
+
+            self.usageFile = 'data/disk-usage.txt'
 
             if not os.path.exists('data/'):
                 os.mkdir('data/')
@@ -578,25 +580,6 @@ class Core:
         conn.close()
         return
 
-    def handle_direct_connection(self, data):
-        '''
-            Handles direct messages
-        '''
-        try:
-            data = json.loads(data)
-
-            # TODO: Determine the sender, verify, etc
-            if ('callback' in data) and (data['callback'] is True):
-                # then this is a response to the message we sent earlier
-                self.daemonQueueAdd('checkCallbacks', json.dumps(data))
-            else:
-                # then we should handle it and respond accordingly
-                self.daemonQueueAdd('incomingDirectConnection', json.dumps(data))
-        except Exception as e:
-            logger.warn('Failed to handle incoming direct message: %s' % str(e))
-
-        return
-
     def getBlockList(self, unsaved = False): # TODO: Use unsaved??
         '''
             Get list of our blocks
@@ -757,6 +740,7 @@ class Core:
             retData = self.setData(payload)
             self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
             self.setBlockType(retData, meta['type'])
+            self.daemonQueueAdd('uploadBlock', retData)
 
         if retData != False:
             events.event('insertBlock', onionr = None, threaded = False)

onionr/onionr.py

@@ -199,7 +199,7 @@ class Onionr:
             'connect': self.addAddress,
             'kex': self.doKEX,
 
-            'getpassword': self.getWebPassword
+            'getpassword': self.printWebPassword
         }
 
         self.cmdhelp = {
@@ -258,6 +258,9 @@ class Onionr:
 
     def getWebPassword(self):
         return config.get('client.hmac')
+        
+    def printWebPassword(self):
+        print(self.getWebPassword())
 
     def getHelp(self):
         return self.cmdhelp

onionr/onionrexceptions.py

@@ -42,6 +42,10 @@ class InvalidHexHash(Exception):
     '''When a string is not a valid hex string of appropriate length for a hash value'''
     pass
 
+class InvalidProof(Exception):
+    '''When a proof is invalid or inadequate'''
+    pass
+
 # network level exceptions
 class MissingPort(Exception):
     pass

onionr/onionrproofs.py

@@ -22,16 +22,19 @@ import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, lo
 import core
 
 class DataPOW:
-    def __init__(self, data, threadCount = 5):
+    def __init__(self, data, forceDifficulty=0, threadCount = 5):
         self.foundHash = False
         self.difficulty = 0
         self.data = data
         self.threadCount = threadCount
 
-        dataLen = sys.getsizeof(data)
+        if forceDifficulty == 0:
-        self.difficulty = math.floor(dataLen / 1000000)
+            dataLen = sys.getsizeof(data)
-        if self.difficulty <= 2:
+            self.difficulty = math.floor(dataLen / 1000000)
-            self.difficulty = 4
+            if self.difficulty <= 2:
+                self.difficulty = 4
+        else:
+            self.difficulty = forceDifficulty
 
         try:
             self.data = self.data.encode()

onionr/onionrutils.py

@@ -54,21 +54,12 @@ class OnionrUtils:
         except Exception as error:
             logger.error('Failed to fetch time bypass token.', error=error)
 
-    def sendPM(self, pubkey, message):
+    def getRoundedEpoch(self, roundS=60):
         '''
-            High level function to encrypt a message to a peer and insert it as a block
+            Returns the epoch, rounded down to given seconds (Default 60)
-        '''
-
-        self._core.insertBlock(message, header='pm', sign=True, encryptType='asym', asymPeer=pubkey)
-
-        return
-
-    def getCurrentHourEpoch(self):
-        '''
-            Returns the current epoch, rounded down to the hour
         '''
         epoch = self.getEpoch()
-        return epoch - (epoch % 3600)
+        return epoch - (epoch % roundS)
 
     def incrementAddressSuccess(self, address):
         '''
@@ -134,9 +125,10 @@ class OnionrUtils:
             if newAdderList != False:
                 for adder in newAdderList.split(','):
                     if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress():
-                        if self._core.addAddress(adder):
+                        if adder[:4] == '0000':
-                            logger.info('Added %s to db.' % adder, timestamp = True)
+                            if self._core.addAddress(adder):
-                            retVal = True
+                                logger.info('Added %s to db.' % adder, timestamp = True)
+                                retVal = True
                     else:
                         pass
                         #logger.debug('%s is either our address or already in our DB' % adder)
@@ -210,19 +202,26 @@ class OnionrUtils:
             
         '''
         meta = {}
+        metadata = {}
+        data = blockData
         try:
             blockData = blockData.encode()
         except AttributeError:
             pass
-        metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
+        
-        data = blockData[blockData.find(b'\n'):].decode()
+        try:
+            metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
+        except json.decoder.JSONDecodeError:
+            pass
+        else:
+            data = blockData[blockData.find(b'\n'):].decode()
 
-        if not metadata['encryptType'] in ('asym', 'sym'):
+            if not metadata['encryptType'] in ('asym', 'sym'):
-            try:
+                try:
-                meta = json.loads(metadata['meta'])
+                    meta = json.loads(metadata['meta'])
-            except KeyError:
+                except KeyError:
-                pass
+                    pass
-        meta = metadata['meta']       
+            meta = metadata['meta']       
         return (metadata, meta, data)
 
     def checkPort(self, port, host=''):
@@ -525,6 +524,30 @@ class OnionrUtils:
         '''returns epoch'''
         return math.floor(time.time())
 
+    def doPostRequest(self, url, data={}, port=0, proxyType='tor'):
+        '''
+        Do a POST request through a local tor or i2p instance
+        '''
+        if proxyType == 'tor':
+            if port == 0:
+                port = self._core.torPort
+            proxies = {'http': 'socks5://127.0.0.1:' + str(port), 'https': 'socks5://127.0.0.1:' + str(port)}
+        elif proxyType == 'i2p':
+            proxies = {'http': 'http://127.0.0.1:4444'}
+        else:
+            return
+        headers = {'user-agent': 'PyOnionr'}
+        try:
+            proxies = {'http': 'socks5h://127.0.0.1:' + str(port), 'https': 'socks5h://127.0.0.1:' + str(port)}
+            r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
+            retData = r.text
+        except KeyboardInterrupt:
+            raise KeyboardInterrupt
+        except requests.exceptions.RequestException as e:
+            logger.debug('Error: %s' % str(e))
+            retData = False
+        return retData
+
     def doGetRequest(self, url, port=0, proxyType='tor'):
         '''
         Do a get request through a local tor or i2p instance
@@ -549,7 +572,7 @@ class OnionrUtils:
             retData = False
         return retData
 
-    def getNistBeaconSalt(self, torPort=0):
+    def getNistBeaconSalt(self, torPort=0, rounding=3600):
         '''
             Get the token for the current hour from the NIST randomness beacon
         '''
@@ -559,7 +582,7 @@ class OnionrUtils:
             except IndexError:
                 raise onionrexceptions.MissingPort('Missing Tor socks port')
         retData = ''
-        curTime = self._core._utils.getCurrentHourEpoch
+        curTime = self.getRoundedEpoch(rounding)
         self.nistSaltTimestamp = curTime
         data = self.doGetRequest('https://beacon.nist.gov/rest/record/' + str(curTime), port=torPort)
         dataXML = minidom.parseString(data, forbid_dtd=True, forbid_entities=True, forbid_external=True)

onionr/static-data/default_config.json

@@ -33,7 +33,7 @@
      },
 
     "allocations":{
-        "disk": 1000000000,
+        "disk": 9000000000,
         "netTotal": 1000000000,
         "blockCache" : 5000000,
         "blockCacheTotal" : 50000000