diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 5d045a69..8e10e8b1 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -102,7 +102,7 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True) OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1) OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1) - OnionrCommunicatorTimers(self, self.detectAPICrash, 5, maxThreads=1) + OnionrCommunicatorTimers(self, self.detectAPICrash, 30, maxThreads=1) deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1) netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) diff --git a/onionr/core.py b/onionr/core.py index 1e698a4c..2a349f68 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -680,7 +680,10 @@ class Core: Inserts a block into the network encryptType must be specified to encrypt a block ''' - + allocationReachedMessage = 'Cannot insert block, disk allocation reached.' + if self._utils.storageCounter.isFull(): + logger.error(allocationReachedMessage) + return False retData = False # check nonce dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data)) @@ -774,13 +777,18 @@ class Core: proof = onionrproofs.POW(metadata, data) payload = proof.waitForResult() if payload != False: - retData = self.setData(payload) - # Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult - self._utils.localCommand('waitforshare/' + retData) - self.addToBlockDB(retData, selfInsert=True, dataSaved=True) - #self.setBlockType(retData, meta['type']) - self._utils.processBlockMetadata(retData) - self.daemonQueueAdd('uploadBlock', retData) + try: + retData = self.setData(payload) + except onionrexceptions.DiskAllocationReached: + logger.error(allocationReachedMessage) + retData = False + else: + # Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult + self._utils.localCommand('waitforshare/' + retData) + self.addToBlockDB(retData, selfInsert=True, dataSaved=True) + #self.setBlockType(retData, meta['type']) + self._utils.processBlockMetadata(retData) + self.daemonQueueAdd('uploadBlock', retData) if retData != False: events.event('insertBlock', onionr = None, threaded = False) diff --git a/onionr/onionr.py b/onionr/onionr.py index aadceb56..e108a3c7 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -719,7 +719,6 @@ class Onionr: ''' Starts the Onionr communication daemon ''' - communicatorDaemon = './communicator2.py' # remove runcheck if it exists diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index e6506faf..93bc3201 100644 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -245,8 +245,8 @@ class Block: blockFile.write(self.getRaw().encode()) else: self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire()) - - self.update() + if self.hash != False: + self.update() return self.getHash() else: diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index 04c821cd..0daf6e3e 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -269,7 +269,8 @@ class OnionrCrypto: except AttributeError: pass - difficulty = math.floor(dataLen / 1000000) + difficulty = onionrproofs.getDifficultyForNewBlock(blockContent, ourBlock=False) + if difficulty < int(config.get('general.minimum_block_pow')): difficulty = int(config.get('general.minimum_block_pow')) mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode() diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py index 9665a4cb..8e3da0a4 100644 --- a/onionr/onionrproofs.py +++ b/onionr/onionrproofs.py @@ -19,7 +19,55 @@ ''' import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys, base64, json -import core, config +import core, onionrutils, config +import onionrblockapi + +def getDifficultyModifier(coreOrUtilsInst=None): + '''Accepts a core or utils instance returns + the difficulty modifier for block storage based + on a variety of factors, currently only disk use. + ''' + classInst = coreOrUtilsInst + retData = 0 + if isinstance(classInst, core.Core): + useFunc = classInst._utils.storageCounter.getPercent + elif isinstance(classInst, onionrutils.OnionrUtils): + useFunc = classInst.storageCounter.getPercent + else: + useFunc = core.Core()._utils.storageCounter.getPercent + + percentUse = useFunc() + + if percentUse >= 0.50: + retData += 1 + elif percentUse >= 0.75: + retData += 2 + elif percentUse >= 0.95: + retData += 3 + + return retData + +def getDifficultyForNewBlock(data, ourBlock=True): + ''' + Get difficulty for block. Accepts size in integer, Block instance, or str/bytes full block contents + ''' + retData = 0 + dataSize = 0 + if isinstance(data, onionrblockapi.Block): + dataSize = len(data.getRaw().encode('utf-8')) + elif isinstance(data, str): + dataSize = len(data.encode('utf-8')) + elif isinstance(data, int): + dataSize = data + else: + raise ValueError('not Block, str, or int') + if ourBlock: + minDifficulty = config.get('general.minimum_send_pow') + else: + minDifficulty = config.get('general.minimum_block_pow') + + retData = max(minDifficulty, math.floor(dataSize / 1000000)) + getDifficultyModifier() + return retData def getHashDifficulty(h): ''' @@ -55,6 +103,7 @@ class DataPOW: self.difficulty = 0 self.data = data self.threadCount = threadCount + self.rounds = 0 config.reload() if forceDifficulty == 0: @@ -96,6 +145,7 @@ class DataPOW: while self.hashing: rand = nacl.utils.random() token = nacl.hash.blake2b(rand + self.data).decode() + self.rounds += 1 #print(token) if self.puzzle == token[0:self.difficulty]: self.hashing = False @@ -106,6 +156,7 @@ class DataPOW: endTime = math.floor(time.time()) if self.reporting: logger.debug('Found token after %s seconds: %s' % (endTime - startTime, token), timestamp=True) + logger.debug('Round count: %s' % (self.rounds,)) self.result = (token, rand) def shutdown(self): @@ -146,18 +197,28 @@ class DataPOW: return result class POW: - def __init__(self, metadata, data, threadCount = 5): + def __init__(self, metadata, data, threadCount = 5, forceDifficulty=0, coreInst=None): self.foundHash = False self.difficulty = 0 self.data = data self.metadata = metadata self.threadCount = threadCount - dataLen = len(data) + len(json.dumps(metadata)) - self.difficulty = math.floor(dataLen / 1000000) - if self.difficulty <= 2: - self.difficulty = int(config.get('general.minimum_block_pow')) + try: + assert isinstance(coreInst, core.Core) + except AssertionError: + myCore = core.Core() + else: + myCore = coreInst + dataLen = len(data) + len(json.dumps(metadata)) + + if forceDifficulty > 0: + self.difficulty = forceDifficulty + else: + # Calculate difficulty. Dumb for now, may use good algorithm in the future. + self.difficulty = getDifficultyForNewBlock(dataLen) + try: self.data = self.data.encode() except AttributeError: @@ -167,8 +228,7 @@ class POW: self.mainHash = '0' * 64 self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))] - - myCore = core.Core() + for i in range(max(1, threadCount)): t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,myCore)) t.start() diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index c6cfdfb9..afd61866 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -155,18 +155,21 @@ class OnionrUtils: ''' Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers. ''' - config.reload() self.getTimeBypassToken() # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless. hostname = '' + maxWait = 5 + waited = 0 while hostname == '': try: with open(self._core.privateApiHostFile, 'r') as host: hostname = host.read() except FileNotFoundError: - print('wat') time.sleep(1) + waited += 1 + if waited == maxWait: + return False if data != '': data = '&data=' + urllib.parse.quote_plus(data) payload = 'http://%s:%s/%s%s' % (hostname, config.get('client.client.port'), command, data) diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 13436852..524ee185 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -1,14 +1,13 @@ { "general" : { "dev_mode" : true, - "display_header" : true, - "minimum_block_pow": 5, - "minimum_send_pow": 5, + "display_header" : false, + "minimum_block_pow": 4, + "minimum_send_pow": 4, "socket_servers": false, "security_level": 0, "max_block_age": 2678400, - "public_key": "", - "use_new_api_server": false + "public_key": "" }, "www" : { @@ -70,7 +69,7 @@ }, "allocations" : { - "disk" : 10000000000, + "disk" : 2000, "net_total" : 1000000000, "blockCache" : 5000000, "blockCacheTotal" : 50000000 diff --git a/onionr/storagecounter.py b/onionr/storagecounter.py index 4468dacc..863145f9 100644 --- a/onionr/storagecounter.py +++ b/onionr/storagecounter.py @@ -43,6 +43,11 @@ class StorageCounter: except FileNotFoundError: pass return retData + + def getPercent(self): + '''Return percent (decimal/float) of disk space we're using''' + amount = self.getAmount() + return round(amount / self._core.config.get('allocations.disk'), 2) def addBytes(self, amount): '''Record that we are now using more disk space, unless doing so would exceed configured max'''