From e60503771edfd6fc52ebe48e404f8ffd99a3008d Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 28 Jan 2019 00:06:20 -0600 Subject: [PATCH] renamed communicator, bug fixes and added work on onionfragment.py --- onionr/{communicator2.py => communicator.py} | 1 + onionr/core.py | 1 + onionr/onionr.py | 19 ++- onionr/onionrblockapi.py | 122 +++---------------- onionr/onionrdaemontools.py | 2 +- onionr/onionrexceptions.py | 3 + onionr/onionrfragment.py | 73 +++++++++++ onionr/onionrstorage.py | 8 ++ 8 files changed, 109 insertions(+), 120 deletions(-) rename onionr/{communicator2.py => communicator.py} (99%) create mode 100644 onionr/onionrfragment.py diff --git a/onionr/communicator2.py b/onionr/communicator.py similarity index 99% rename from onionr/communicator2.py rename to onionr/communicator.py index 81e73f29..8ba63c32 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator.py @@ -567,6 +567,7 @@ class OnionrCommunicatorDaemon: # when inserting a block, we try to upload it to a few peers to add some deniability triedPeers = [] finishedUploads = [] + self.blocksToUpload = self._core._crypto.randomShuffle(self.blocksToUpload) if len(self.blocksToUpload) != 0: for bl in self.blocksToUpload: if not self._core._utils.validateHash(bl): diff --git a/onionr/core.py b/onionr/core.py index 07762236..b14eee1f 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -99,6 +99,7 @@ class Core: logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation) self._utils = onionrutils.OnionrUtils(self) + self.blockCache = onionrstorage.BlockCache() # Initialize the crypto object self._crypto = onionrcrypto.OnionrCrypto(self) self._blacklist = onionrblacklist.OnionrBlackList(self) diff --git a/onionr/onionr.py b/onionr/onionr.py index 4f054a3b..2733500a 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -33,7 +33,7 @@ import onionrutils import netcontroller from netcontroller import NetController from onionrblockapi import Block -import onionrproofs, onionrexceptions, onionrusers, communicator2 +import onionrproofs, onionrexceptions, onionrusers, communicator try: from urllib3.contrib.socks import SOCKSProxyManager @@ -710,7 +710,6 @@ class Onionr: ''' Starts the Onionr communication daemon ''' - communicatorDaemon = './communicator2.py' # remove runcheck if it exists if os.path.isfile('data/.runcheck'): @@ -750,10 +749,8 @@ class Onionr: logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey)) time.sleep(1) - # TODO: make runable on windows - #communicatorProc = subprocess.Popen([communicatorDaemon, 'run', str(net.socksPort)]) self.onionrCore.torPort = net.socksPort - communicatorThread = Thread(target=communicator2.startCommunicator, args=(self, str(net.socksPort))) + communicatorThread = Thread(target=communicator.startCommunicator, args=(self, str(net.socksPort))) communicatorThread.start() while self.communicatorInst is None: @@ -940,7 +937,8 @@ class Onionr: logger.error('Block hash is invalid') return - Block.mergeChain(bHash, fileName) + with open(fileName, 'wb') as myFile: + myFile.write(base64.b64decode(Block(bHash, core=self.onionrCore).bcontent)) return def addWebpage(self): @@ -963,12 +961,9 @@ class Onionr: return logger.info('Adding file... this might take a long time.') try: - if singleBlock: - with open(filename, 'rb') as singleFile: - blockhash = self.onionrCore.insertBlock(base64.b64encode(singleFile.read()), header=blockType) - else: - blockhash = Block.createChain(file = filename) - logger.info('File %s saved in block %s.' % (filename, blockhash)) + with open(filename, 'rb') as singleFile: + blockhash = self.onionrCore.insertBlock(base64.b64encode(singleFile.read()), header=blockType) + logger.info('File %s saved in block %s' % (filename, blockhash)) except: logger.error('Failed to save file in block.', timestamp = False) else: diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index f60e3b0f..c8f12a54 100644 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -1,7 +1,7 @@ ''' Onionr - P2P Anonymous Storage Network - This class contains the OnionrBlocks class which is a class for working with Onionr blocks + This file contains the OnionrBlocks class which is a class for working with Onionr blocks ''' ''' This program is free software: you can redistribute it and/or modify @@ -56,18 +56,7 @@ class Block: if self.getCore() is None: self.core = onionrcore.Core() - # update the blocks' contents if it exists - if not self.getHash() is None: - if not self.core._utils.validateHash(self.hash): - logger.debug('Block hash %s is invalid.' % self.getHash()) - raise onionrexceptions.InvalidHexHash('Block hash is invalid.') - elif not self.update(): - logger.debug('Failed to open block %s.' % self.getHash()) - else: - pass - #logger.debug('Did not update block.') - - # logic + self.update() def decrypt(self, anonymous = True, encodedData = True): ''' @@ -140,13 +129,15 @@ class Block: Outputs: - (bool): indicates whether or not the operation was successful ''' - try: # import from string blockdata = data # import from file if blockdata is None: + blockdata = onionrstorage.getData(self.core, self.getHash()).decode() + ''' + filelocation = file readfile = True @@ -169,6 +160,7 @@ class Block: #blockdata = f.read().decode() self.blockFile = filelocation + ''' else: self.blockFile = None # parse block @@ -588,7 +580,7 @@ class Block: return list() - def mergeChain(child, file = None, maximumFollows = 32, core = None): + def mergeChain(child, file = None, maximumFollows = 1000, core = None): ''' Follows a child Block to its root parent Block, merging content @@ -635,7 +627,7 @@ class Block: blocks.append(block.getHash()) - buffer = '' + buffer = b'' # combine block contents for hash in blocks: @@ -644,101 +636,17 @@ class Block: contents = base64.b64decode(contents.encode()) if file is None: - buffer += contents.decode() + try: + buffer += contents.encode() + except AttributeError: + buffer += contents else: file.write(contents) - file.close() + if file is not None: + file.close() return (None if not file is None else buffer) - def createChain(data = None, chunksize = 99800, file = None, type = 'chunk', sign = True, encrypt = False, verbose = False): - ''' - Creates a chain of blocks to store larger amounts of data - - The chunksize is set to 99800 because it provides the least amount of PoW for the most amount of data. - - Inputs: - - data (*): if `file` is None, the data to be stored in blocks - - file (file/str): the filename or file object to read from (or None to read `data` instead) - - chunksize (int): the number of bytes per block chunk - - type (str): the type header for each of the blocks - - sign (bool): whether or not to sign each block - - encrypt (str): the public key to encrypt to, or False to disable encryption - - verbose (bool): whether or not to return a tuple containing more info - - Outputs: - - if `verbose`: - - (tuple): - - (str): the child block hash - - (list): all block hashes associated with storing the file - - if not `verbose`: - - (str): the child block hash - ''' - - blocks = list() - - # initial datatype checks - if data is None and file is None: - return blocks - elif not (file is None or (isinstance(file, str) and os.path.exists(file))): - return blocks - elif isinstance(file, str): - file = open(file, 'rb') - if not isinstance(data, str): - data = str(data) - - if not file is None: - filesize = os.stat(file.name).st_size - offset = filesize % chunksize - maxtimes = int(filesize / chunksize) - - for times in range(0, maxtimes + 1): - # read chunksize bytes from the file (end -> beginning) - if times < maxtimes: - file.seek(- ((times + 1) * chunksize), 2) - content = file.read(chunksize) - else: - file.seek(0, 0) - content = file.read(offset) - - # encode it- python is really bad at handling certain bytes that - # are often present in binaries. - content = base64.b64encode(content).decode() - - # if it is the end of the file, exit - if not content: - break - - # create block - block = Block() - block.setType(type) - block.setContent(content) - block.setParent((blocks[-1] if len(blocks) != 0 else None)) - hash = block.save(sign = sign) - - # remember the hash in cache - blocks.append(hash) - elif not data is None: - for content in reversed([data[n:n + chunksize] for n in range(0, len(data), chunksize)]): - # encode chunk with base64 - content = base64.b64encode(content.encode()).decode() - - # create block - block = Block() - block.setType(type) - block.setContent(content) - block.setParent((blocks[-1] if len(blocks) != 0 else None)) - hash = block.save(sign = sign) - - # remember the hash in cache - blocks.append(hash) - - # return different things depending on verbosity - if verbose: - return (blocks[-1], blocks) - file.close() - return blocks[-1] - def exists(bHash): ''' Checks if a block is saved to file or not @@ -799,7 +707,7 @@ class Block: if block.getHash() in Block.getCache() and not override: return False - # dump old cached blocks if the size exeeds the maximum + # dump old cached blocks if the size exceeds the maximum if sys.getsizeof(Block.blockCacheOrder) >= config.get('allocations.block_cache_total', 50000000): # 50MB default cache size del Block.blockCache[blockCacheOrder.pop(0)] diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index 01fce0b5..7e7c8ec7 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -34,7 +34,7 @@ class DaemonTools: '''Announce our node to our peers''' retData = False announceFail = False - if config.get('general.security_level') == 0: + if self.daemon._core.config('general.security_level') == 0: # Announce to random online peers for i in self.daemon.onlinePeers: if not i in self.announceCache: diff --git a/onionr/onionrexceptions.py b/onionr/onionrexceptions.py index a0c468a3..6763a172 100644 --- a/onionr/onionrexceptions.py +++ b/onionr/onionrexceptions.py @@ -53,6 +53,9 @@ class BlacklistedBlock(Exception): class DataExists(Exception): pass +class NoDataAvailable(Exception): + pass + class InvalidHexHash(Exception): '''When a string is not a valid hex string of appropriate length for a hash value''' pass diff --git a/onionr/onionrfragment.py b/onionr/onionrfragment.py new file mode 100644 index 00000000..c8386465 --- /dev/null +++ b/onionr/onionrfragment.py @@ -0,0 +1,73 @@ +''' + Onionr - P2P Anonymous Storage Network + + This file contains the OnionrFragment class which implements the fragment system +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' + +# onionr:10ch+10ch+10chgdecryptionkey +import core, sys, binascii, os + +FRAGMENT_SIZE = 0.25 +TRUNCATE_LENGTH = 30 + +class OnionrFragment: + def __init__(self, uri=None): + uri = uri.replace('onionr:', '') + count = 0 + blocks = [] + appendData = '' + key = '' + for x in uri: + if x == 'k': + key = uri[uri.index('k') + 1:] + appendData += x + if count == TRUNCATE_LENGTH: + blocks.append(appendData) + appendData = '' + count = 0 + count += 1 + self.key = key + self.blocks = blocks + return + + @staticmethod + def generateFragments(data=None, coreInst=None): + if coreInst is None: + coreInst = core.Core() + + key = os.urandom(32) + data = coreInst._crypto.symmetricEncrypt(data, key).decode() + blocks = [] + blockData = b"" + uri = "onionr:" + total = sys.getsizeof(data) + for x in data: + blockData += x.encode() + if round(len(blockData) / len(data), 3) > FRAGMENT_SIZE: + blocks.append(core.Core().insertBlock(blockData)) + blockData = b"" + + for bl in blocks: + uri += bl[:TRUNCATE_LENGTH] + uri += "k" + uri += binascii.hexlify(key).decode() + return (uri, key) + +if __name__ == '__main__': + uri = OnionrFragment.generateFragments("test")[0] + print(uri) + OnionrFragment(uri) \ No newline at end of file diff --git a/onionr/onionrstorage.py b/onionr/onionrstorage.py index 39dfd323..8e2aae41 100644 --- a/onionr/onionrstorage.py +++ b/onionr/onionrstorage.py @@ -21,6 +21,13 @@ import core, sys, sqlite3, os, dbcreator DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option +class BlockCache: + def __init__(self): + self.blocks = {} + def cleanCache(self): + while sys.getsizeof(self.blocks) > 100000000: + self.blocks.pop(list(self.blocks.keys())[0]) + def dbCreate(coreInst): try: dbcreator.DBCreator(coreInst).createBlockDataDB() @@ -62,6 +69,7 @@ def store(coreInst, data, blockHash=''): else: with open('%s/%s.dat' % (coreInst.blockDataLocation, blockHash), 'wb') as blockFile: blockFile.write(data) + coreInst.blockCache.cleanCache() def getData(coreInst, bHash): assert isinstance(coreInst, core.Core)