work on exchanging data

docs/api.md

@@ -0,0 +1,26 @@
+HTTP API
+------------------------------------------------
+/client/ (Private info, not publicly accessible)
+
+-   hello
+    - hello world
+-   shutdown
+    - exit onionr
+-   stats
+    - show node stats
+
+/public/
+
+-   firstConnect
+    - initialize with peer
+-   ping
+    - pong
+-   setHMAC
+    - set a created symmetric key 
+-   getDBHash
+    - get the hash of the current hash database state
+-   getPGP
+    - export node's PGP public key
+-   getData
+    - get a data block
+-------------------------------------------------

onionr/api.py

@@ -50,7 +50,7 @@ class API:
         self.debug = debug
         self._privateDelayTime = 3
         self._core = Core()
-        self._utils = onionrutils.OnionrUtils()
+        self._utils = onionrutils.OnionrUtils(self._core)
         app = flask.Flask(__name__)
         bindPort = int(self.config['CLIENT']['PORT'])
         self.bindPort = bindPort
@@ -127,6 +127,10 @@ class API:
                 resp = Response("pong!")
             elif action == 'setHMAC':
                 pass
+            elif action == 'getDBHash':
+                resp = Response(self._utils.getBlockDBHash())
+            elif action == 'getBlockHashes':
+                resp = Response(self._core.getBlockList())
             elif action == 'getPGP':
                 resp = Response(self._utils.exportMyPubkey())
             # setData should be something the communicator initiates, not this api

onionr/communicator.py

@@ -29,7 +29,7 @@ class OnionrCommunicate:
         '''
         self._core = core.Core()
         blockProcessTimer = 0
-        blockProccesAmount = 5
+        blockProcessAmount = 5
         if debug:
             print('Communicator debugging enabled')
         torID = open('data/hs/hostname').read()
@@ -48,9 +48,9 @@ class OnionrCommunicate:
             # Process blocks based on a timer
             blockProcessTimer += 1
             if blockProcessTimer == blockProcessAmount:
+                self.lookupBlocks()
                 self._core.processBlocks()
                 blockProcessTimer = 0
-
             if debug:
                 print('Communicator daemon heartbeat')
             if command != False:
@@ -79,6 +79,26 @@ class OnionrCommunicate:
         '''This function sends the proof result to a peer previously fetched with getPeerProof'''
         return
 
+    def lookupBlocks(self):
+        '''Lookup blocks and merge new ones'''
+        peerList = self._core.listPeers()
+        blocks = ''
+        for i in peerList:
+            lastDB = self._core.getPeerInfo(i, 'blockDBHash')
+            currentDB = self.performGet('getDBHash', i)
+            if lastDB != currentDB:
+                blocks += self.performGet('getBlockHashes', i)
+        blockList = blocks.split('\n')
+        for i in blockList:
+            if not self._core.validateHash(i):
+                # skip hash if it isn't valid
+                continue
+            else:
+                print('adding', i, 'to hash database')
+                self._core.addToBlockDB(i)
+
+        return
+    
     def performGet(self, action, peer, data=None, type='tor'):
         '''performs a request to a peer through Tor or i2p (currently only tor)'''
         if not peer.endswith('.onion') and not peer.endswith('.onion/'):

onionr/core.py

@@ -84,6 +84,7 @@ class Core:
         name text,
         pgpKey text,
         hmacKey text,
+        blockDBHash text,
         forwardKey text,
         dateSeen not null,
         bytesStored int,
@@ -108,7 +109,8 @@ class Core:
             hash text not null,
             dateReceived int,
             decrypted int,
-            dataFound int
+            dataFound int,
+            dataSaved int
         );
         ''')
         conn.commit()
@@ -229,9 +231,68 @@ class Core:
         key = base64.b64encode(os.urandom(32))
         return key
 
+    def listPeers(self):
+        '''Return a list of peers
+        '''
+        conn = sqlite3.connect(self.peerDB)
+        c = conn.cursor()
+        peers = c.execute('SELECT * FROM peers;')
+        peerList = []
+        for i in peers:
+            peerList.append(i[0])
+        conn.close()
+        return peerList
+
     def processBlocks(self):
         '''
         Work with the block database and download any missing blocks
         This is meant to be called from the communicator daemon on its timer.
         '''
+        conn = sqlite3.connect(self.blockDB)
+        c = conn.cursor()
+        for i in blocks:
+            pass
+        conn.close()
         return
+    def getPeerInfo(self, peer, info):
+        '''
+        get info about a peer
+
+        id text             0
+        name text,          1
+        pgpKey text,        2
+        hmacKey text,       3
+        blockDBHash text,   4
+        forwardKey text,    5
+        dateSeen not null,  7
+        bytesStored int,    8
+        trust int           9
+        '''
+        # Lookup something about a peer from their database entry
+        conn = sqlite3.connect(self.peerDB)
+        c = conn.cursor()
+        command = (peer,)
+        infoNumbers = {'id': 0, 'name': 1, 'pgpKey': 2, 'hmacKey': 3, 'blockDBHash': 4, 'forwardKey': 5, 'dateSeen': 6, 'bytesStored': 7, 'trust': 8}
+        info = infoNumbers[info]
+        iterCount = 0
+        retVal = ''
+        for row in c.execute('SELECT * from peers where id=?;', command):
+            for i in row:
+                if iterCount == info:
+                    retVal = i
+                    break
+                else:
+                    iterCount += 1
+        conn.close()
+        return retVal
+
+    def getBlockList(self):
+        '''get list of our blocks'''
+        conn = sqlite3.connect(self.blockDB)
+        c = conn.cursor()
+        retData = ''
+        for row in c.execute('SELECT hash FROM hashes;'):
+            for i in row:
+                retData += i
+        return retData
+        

onionr/onionr.py

@@ -45,7 +45,7 @@ class Onionr:
         colors = Colors()
 
         self.onionrCore = core.Core()
-        self.onionrUtils = OnionrUtils()
+        self.onionrUtils = OnionrUtils(self.onionrCore)
 
         # Get configuration and Handle commands
         
@@ -63,6 +63,7 @@ class Onionr:
         else:
             if not os.path.exists('data/'):
                 os.mkdir('data/')
+                os.mkdir('data/blocks/')
         
         if not os.path.exists('data/peers.db'):
             self.onionrCore.createPeerDB()
@@ -128,7 +129,7 @@ class Onionr:
             net.startTor()
             print(colors.GREEN + 'Started Tor .onion service: ' + colors.UNDERLINE + net.myID + colors.RESET)
             time.sleep(1)
-            subprocess.Popen(["./communicator.py", "run", net.socksPort])
+            subprocess.Popen(["./communicator.py", "run", str(net.socksPort)])
             print('Started communicator')
         api.API(self.config, self.debug)
         return
@@ -149,4 +150,5 @@ class Onionr:
         '''Show help for Onionr'''
         return
         
+
 Onionr()

onionr/onionrutils.py

@@ -18,11 +18,18 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 # Misc functions that do not fit in the main api, but are useful
-import getpass, sys, requests, configparser, os, socket, gnupg
+import getpass, sys, requests, configparser, os, socket, gnupg, hashlib
-class OnionrUtils():
+if sys.version_info < (3, 6):
+    try:
+        import sha3
+    except ModuleNotFoundError:
+        sys.stderr.write('On Python 3 versions prior to 3.6.x, you need the sha3 module')
+        sys.exit(1)
+class OnionrUtils:
     '''Various useful functions'''
-    def __init__(self):
+    def __init__(self, coreInstance):
         self.fingerprintFile = 'data/own-fingerprint.txt'
+        self._core = coreInstance
         return
     def printErr(self, text='an error occured'):
         '''Print an error message to stderr with a new line'''
@@ -74,3 +81,24 @@ class OnionrUtils():
             fingerprint = f.read()
         ascii_armored_public_keys = gpg.export_keys(fingerprint)
         return ascii_armored_public_keys
+
+    def getBlockDBHash(self):
+        '''Return a sha3_256 hash of the blocks DB'''
+        with open(self._core.blockDB, 'rb') as data:
+            data = data.read()
+        hasher = hashlib.sha3_256()
+        hasher.update(data)
+        dataHash = hasher.hexdigest()
+        return dataHash
+
+    def validateHash(self, data, length=64):
+        '''validate if a string is a valid hex formatted hash'''
+        retVal = True
+        if len(data) != length:
+            retVal = False
+        else:
+            try:
+                int(data, 16)
+            except ValueError:
+                retVal = False
+        return retVal