renamed communicator, bug fixes and added work on onionfragment.py

onionr/communicator.py

@@ -567,6 +567,7 @@ class OnionrCommunicatorDaemon:
         # when inserting a block, we try to upload it to a few peers to add some deniability
         triedPeers = []
         finishedUploads = []
+        self.blocksToUpload = self._core._crypto.randomShuffle(self.blocksToUpload)
         if len(self.blocksToUpload) != 0:
             for bl in self.blocksToUpload:
                 if not self._core._utils.validateHash(bl):

onionr/core.py

@@ -99,6 +99,7 @@ class Core:
                 logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation)
 
             self._utils = onionrutils.OnionrUtils(self)
+            self.blockCache = onionrstorage.BlockCache()
             # Initialize the crypto object
             self._crypto = onionrcrypto.OnionrCrypto(self)
             self._blacklist = onionrblacklist.OnionrBlackList(self)

onionr/onionr.py

@@ -33,7 +33,7 @@ import onionrutils
 import netcontroller
 from netcontroller import NetController
 from onionrblockapi import Block
-import onionrproofs, onionrexceptions, onionrusers, communicator2
+import onionrproofs, onionrexceptions, onionrusers, communicator
 
 try:
     from urllib3.contrib.socks import SOCKSProxyManager
@@ -710,7 +710,6 @@ class Onionr:
         '''
             Starts the Onionr communication daemon
         '''
-        communicatorDaemon = './communicator2.py'
 
         # remove runcheck if it exists
         if os.path.isfile('data/.runcheck'):
@@ -750,10 +749,8 @@ class Onionr:
         logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey))
         time.sleep(1)
 
-        # TODO: make runable on windows
-        #communicatorProc = subprocess.Popen([communicatorDaemon, 'run', str(net.socksPort)])
         self.onionrCore.torPort = net.socksPort
-        communicatorThread = Thread(target=communicator2.startCommunicator, args=(self, str(net.socksPort)))
+        communicatorThread = Thread(target=communicator.startCommunicator, args=(self, str(net.socksPort)))
         communicatorThread.start()
         
         while self.communicatorInst is None:
@@ -940,7 +937,8 @@ class Onionr:
                 logger.error('Block hash is invalid')
                 return
 
-            Block.mergeChain(bHash, fileName)
+            with open(fileName, 'wb') as myFile:
+                myFile.write(base64.b64decode(Block(bHash, core=self.onionrCore).bcontent))
         return
 
     def addWebpage(self):
@@ -963,12 +961,9 @@ class Onionr:
                 return
             logger.info('Adding file... this might take a long time.')
             try:
-                if singleBlock:
                 with open(filename, 'rb') as singleFile:
                     blockhash = self.onionrCore.insertBlock(base64.b64encode(singleFile.read()), header=blockType)
-                else:
+                logger.info('File %s saved in block %s' % (filename, blockhash))
-                    blockhash = Block.createChain(file = filename)
-                logger.info('File %s saved in block %s.' % (filename, blockhash))
             except:
                 logger.error('Failed to save file in block.', timestamp = False)
         else:

onionr/onionrblockapi.py

@@ -1,7 +1,7 @@
 '''
     Onionr - P2P Anonymous Storage Network
 
-    This class contains the OnionrBlocks class which is a class for working with Onionr blocks
+    This file contains the OnionrBlocks class which is a class for working with Onionr blocks
 '''
 '''
     This program is free software: you can redistribute it and/or modify
@@ -56,18 +56,7 @@ class Block:
         if self.getCore() is None:
             self.core = onionrcore.Core()
 
-        # update the blocks' contents if it exists
+        self.update()
-        if not self.getHash() is None:
-            if not self.core._utils.validateHash(self.hash):
-                logger.debug('Block hash %s is invalid.' % self.getHash())
-                raise onionrexceptions.InvalidHexHash('Block hash is invalid.')
-            elif not self.update():
-                logger.debug('Failed to open block %s.' % self.getHash())
-        else:
-            pass
-            #logger.debug('Did not update block.')
-
-    # logic
 
     def decrypt(self, anonymous = True, encodedData = True):
         '''
@@ -140,13 +129,15 @@ class Block:
             Outputs:
             - (bool): indicates whether or not the operation was successful
         '''
-
         try:
             # import from string
             blockdata = data
 
             # import from file
             if blockdata is None:
+                blockdata = onionrstorage.getData(self.core, self.getHash()).decode()
+                '''
+                
                 filelocation = file
 
                 readfile = True
@@ -169,6 +160,7 @@ class Block:
                     #blockdata = f.read().decode()
 
                 self.blockFile = filelocation
+                '''
             else:
                 self.blockFile = None
             # parse block
@@ -588,7 +580,7 @@ class Block:
 
         return list()
 
-    def mergeChain(child, file = None, maximumFollows = 32, core = None):
+    def mergeChain(child, file = None, maximumFollows = 1000, core = None):
         '''
             Follows a child Block to its root parent Block, merging content
 
@@ -635,7 +627,7 @@ class Block:
 
             blocks.append(block.getHash())
 
-        buffer = ''
+        buffer = b''
 
         # combine block contents
         for hash in blocks:
@@ -644,101 +636,17 @@ class Block:
             contents = base64.b64decode(contents.encode())
 
             if file is None:
-                buffer += contents.decode()
+                try:
+                    buffer += contents.encode()
+                except AttributeError:
+                    buffer += contents
             else:
                 file.write(contents)
+        if file is not None:
             file.close()
 
         return (None if not file is None else buffer)
 
-    def createChain(data = None, chunksize = 99800, file = None, type = 'chunk', sign = True, encrypt = False, verbose = False):
-        '''
-            Creates a chain of blocks to store larger amounts of data
-
-            The chunksize is set to 99800 because it provides the least amount of PoW for the most amount of data.
-
-            Inputs:
-            - data (*): if `file` is None, the data to be stored in blocks
-            - file (file/str): the filename or file object to read from (or None to read `data` instead)
-            - chunksize (int): the number of bytes per block chunk
-            - type (str): the type header for each of the blocks
-            - sign (bool): whether or not to sign each block
-            - encrypt (str): the public key to encrypt to, or False to disable encryption
-            - verbose (bool): whether or not to return a tuple containing more info
-
-            Outputs:
-            - if `verbose`:
-              - (tuple):
-                - (str): the child block hash
-                - (list): all block hashes associated with storing the file
-            - if not `verbose`:
-              - (str): the child block hash
-        '''
-
-        blocks = list()
-
-        # initial datatype checks
-        if data is None and file is None:
-            return blocks
-        elif not (file is None or (isinstance(file, str) and os.path.exists(file))):
-            return blocks
-        elif isinstance(file, str):
-            file = open(file, 'rb')
-        if not isinstance(data, str):
-            data = str(data)
-
-        if not file is None:
-            filesize = os.stat(file.name).st_size
-            offset = filesize % chunksize
-            maxtimes = int(filesize / chunksize)
-
-            for times in range(0, maxtimes + 1):
-                # read chunksize bytes from the file (end -> beginning)
-                if times < maxtimes:
-                    file.seek(- ((times + 1) * chunksize), 2)
-                    content = file.read(chunksize)
-                else:
-                    file.seek(0, 0)
-                    content = file.read(offset)
-
-                # encode it- python is really bad at handling certain bytes that
-                # are often present in binaries.
-                content = base64.b64encode(content).decode()
-
-                # if it is the end of the file, exit
-                if not content:
-                    break
-
-                # create block
-                block = Block()
-                block.setType(type)
-                block.setContent(content)
-                block.setParent((blocks[-1] if len(blocks) != 0 else None))
-                hash = block.save(sign = sign)
-
-                # remember the hash in cache
-                blocks.append(hash)
-        elif not data is None:
-            for content in reversed([data[n:n + chunksize] for n in range(0, len(data), chunksize)]):
-                # encode chunk with base64
-                content = base64.b64encode(content.encode()).decode()
-
-                # create block
-                block = Block()
-                block.setType(type)
-                block.setContent(content)
-                block.setParent((blocks[-1] if len(blocks) != 0 else None))
-                hash = block.save(sign = sign)
-
-                # remember the hash in cache
-                blocks.append(hash)
-
-        # return different things depending on verbosity
-        if verbose:
-            return (blocks[-1], blocks)
-        file.close()
-        return blocks[-1]
-
     def exists(bHash):
         '''
             Checks if a block is saved to file or not
@@ -799,7 +707,7 @@ class Block:
         if block.getHash() in Block.getCache() and not override:
             return False
 
-        # dump old cached blocks if the size exeeds the maximum
+        # dump old cached blocks if the size exceeds the maximum
         if sys.getsizeof(Block.blockCacheOrder) >= config.get('allocations.block_cache_total', 50000000): # 50MB default cache size
             del Block.blockCache[blockCacheOrder.pop(0)]
 

onionr/onionrdaemontools.py

@@ -34,7 +34,7 @@ class DaemonTools:
         '''Announce our node to our peers'''
         retData = False
         announceFail = False
-        if config.get('general.security_level') == 0:
+        if self.daemon._core.config('general.security_level') == 0:
             # Announce to random online peers
             for i in self.daemon.onlinePeers:
                 if not i in self.announceCache:

onionr/onionrexceptions.py

@@ -53,6 +53,9 @@ class BlacklistedBlock(Exception):
 class DataExists(Exception):
     pass
 
+class NoDataAvailable(Exception):
+    pass
+
 class InvalidHexHash(Exception):
     '''When a string is not a valid hex string of appropriate length for a hash value'''
     pass

onionr/onionrfragment.py

@@ -0,0 +1,73 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file contains the OnionrFragment class which implements the fragment system
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+# onionr:10ch+10ch+10chgdecryptionkey
+import core, sys, binascii, os
+
+FRAGMENT_SIZE = 0.25
+TRUNCATE_LENGTH = 30
+
+class OnionrFragment:
+    def __init__(self, uri=None):
+        uri = uri.replace('onionr:', '')
+        count = 0
+        blocks = []
+        appendData = ''
+        key = ''
+        for x in uri:
+            if x == 'k':
+                key = uri[uri.index('k') + 1:]
+            appendData += x
+            if count == TRUNCATE_LENGTH:
+                blocks.append(appendData)
+                appendData = ''
+                count = 0
+            count += 1
+        self.key = key
+        self.blocks = blocks
+        return
+
+    @staticmethod
+    def generateFragments(data=None, coreInst=None):
+        if coreInst is None:
+            coreInst = core.Core()
+
+        key = os.urandom(32)
+        data = coreInst._crypto.symmetricEncrypt(data, key).decode()
+        blocks = []
+        blockData = b""
+        uri = "onionr:"
+        total = sys.getsizeof(data)
+        for x in data:
+            blockData += x.encode()
+            if round(len(blockData) / len(data), 3) > FRAGMENT_SIZE:
+                blocks.append(core.Core().insertBlock(blockData))
+                blockData = b""
+
+        for bl in blocks:
+            uri += bl[:TRUNCATE_LENGTH]
+        uri += "k"
+        uri += binascii.hexlify(key).decode()
+        return (uri, key)
+
+if __name__ == '__main__':
+    uri = OnionrFragment.generateFragments("test")[0]
+    print(uri)
+    OnionrFragment(uri)

onionr/onionrstorage.py

@@ -21,6 +21,13 @@ import core, sys, sqlite3, os, dbcreator
 
 DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option
 
+class BlockCache:
+    def __init__(self):
+        self.blocks = {}
+    def cleanCache(self):
+        while sys.getsizeof(self.blocks) > 100000000:
+            self.blocks.pop(list(self.blocks.keys())[0])
+
 def dbCreate(coreInst):
     try:
         dbcreator.DBCreator(coreInst).createBlockDataDB()
@@ -62,6 +69,7 @@ def store(coreInst, data, blockHash=''):
     else:
         with open('%s/%s.dat' % (coreInst.blockDataLocation, blockHash), 'wb') as blockFile:
             blockFile.write(data)
+    coreInst.blockCache.cleanCache()
 
 def getData(coreInst, bHash):
     assert isinstance(coreInst, core.Core)