From e34c08b0368e14c545e7e5c9008fc1b1d3bcc141 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 23 Aug 2018 12:48:49 -0500 Subject: [PATCH] sync improvements, bug fixes, config changes --- onionr/communicator2.py | 22 +++++++--------------- onionr/core.py | 14 ++++++++++++-- onionr/onionr.py | 10 ++++++++++ onionr/onionrdaemontools.py | 10 ++++++++-- onionr/onionrexceptions.py | 2 +- onionr/onionrutils.py | 4 ---- onionr/static-data/default_config.json | 2 +- onionr/storagecounter.py | 2 +- 8 files changed, 40 insertions(+), 26 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index da610f4b..057defed 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -80,10 +80,6 @@ class OnionrCommunicatorDaemon: if debug or developmentMode: OnionrCommunicatorTimers(self, self.heartbeat, 10) - # Print nice header thing :) - if config.get('general.display_header', True) and not self.shutdown: - self.header() - # Set timers, function reference, seconds # requiresPeer True means the timer function won't fire if we have no connected peers # TODO: make some of these timer counts configurable @@ -93,6 +89,7 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.lookupBlocks, 7, requiresPeer=True, maxThreads=1) OnionrCommunicatorTimers(self, self.getBlocks, 10, requiresPeer=True) OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58) + OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 650) OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True) OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True) netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) @@ -152,6 +149,8 @@ class OnionrCommunicatorDaemon: triedPeers = [] # list of peers we've tried this time around for i in range(tryAmount): # check if disk allocation is used + if not self.isOnline: + break if self._core._utils.storageCounter.isFull(): logger.warn('Not looking up new blocks due to maximum amount of allowed disk space used') break @@ -188,8 +187,8 @@ class OnionrCommunicatorDaemon: def getBlocks(self): '''download new blocks in queue''' for blockHash in self.blockQueue: - if self.shutdown: - # Exit loop if shutting down + if self.shutdown or not self.isOnline: + # Exit loop if shutting down or offline break # Do not download blocks being downloaded or that are already saved (edge cases) if blockHash in self.currentDownloading: @@ -221,11 +220,11 @@ class OnionrCommunicatorDaemon: #meta = metas[1] if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce if self._core._crypto.verifyPow(content): # check if POW is enough/correct - logger.info('Block passed proof, saving.') + logger.info('Block passed proof, attemping save.') try: self._core.setData(content) except onionrexceptions.DiskAllocationReached: - logger.error("Reached disk allocation allowance, cannot save additional blocks.") + logger.error("Reached disk allocation allowance, cannot save this block.") else: self._core.addToBlockDB(blockHash, dataSaved=True) self._core._utils.processBlockMetadata(blockHash) # caches block metadata values to block database @@ -489,13 +488,6 @@ class OnionrCommunicatorDaemon: self.shutdown = True self.decrementThreadCount('detectAPICrash') - def header(self, message = logger.colors.fg.pink + logger.colors.bold + 'Onionr' + logger.colors.reset + logger.colors.fg.pink + ' has started.'): - if os.path.exists('static-data/header.txt'): - with open('static-data/header.txt', 'rb') as file: - # only to stdout, not file or log or anything - sys.stderr.write(file.read().decode().replace('P', logger.colors.fg.pink).replace('W', logger.colors.reset + logger.colors.bold).replace('G', logger.colors.fg.green).replace('\n', logger.colors.reset + '\n').replace('B', logger.colors.bold).replace('V', onionr.ONIONR_VERSION)) - logger.info(logger.colors.fg.lightgreen + '-> ' + str(message) + logger.colors.reset + logger.colors.fg.lightgreen + ' <-\n') - class OnionrCommunicatorTimers: def __init__(self, daemonInstance, timerFunction, frequency, makeThread=True, threadAmount=1, maxThreads=5, requiresPeer=False): self.timerFunction = timerFunction diff --git a/onionr/core.py b/onionr/core.py index 14f4c3ba..13c89c9e 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -183,8 +183,16 @@ class Core: c.execute('Delete from hashes where hash=?;', t) conn.commit() conn.close() + blockFile = 'data/blocks/' + block + '.dat' + dataSize = 0 try: - os.remove('data/blocks/' + block + '.dat') + ''' Get size of data when loaded as an object/var, rather than on disk, + to avoid conflict with getsizeof when saving blocks + ''' + with open(blockFile, 'r') as data: + dataSize = sys.getsizeof(data.read()) + self._utils.storageCounter.removeBytes(dataSize) + os.remove(blockFile) except FileNotFoundError: pass @@ -280,6 +288,8 @@ class Core: c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = '" + dataHash + "';") conn.commit() conn.close() + with open(self.dataNonceFile, 'a') as nonceFile: + nonceFile.write(dataHash + '\n') else: raise onionrexceptions.DiskAllocationReached @@ -544,7 +554,7 @@ class Core: if unsaved: execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();' else: - execute = 'SELECT hash FROM hashes ORDER BY dateReceived DESC;' + execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;' rows = list() for row in c.execute(execute): for i in row: diff --git a/onionr/onionr.py b/onionr/onionr.py index 1736c3f9..52b12e48 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -588,6 +588,9 @@ class Onionr: time.sleep(1) #TODO make runable on windows subprocess.Popen([communicatorDaemon, "run", str(net.socksPort)]) + # Print nice header thing :) + if config.get('general.display_header', True): + self.header() logger.debug('Started communicator') events.event('daemon_start', onionr = self) try: @@ -759,5 +762,12 @@ class Onionr: print('Opening %s ...' % url) webbrowser.open(url, new = 1, autoraise = True) + def header(self, message = logger.colors.fg.pink + logger.colors.bold + 'Onionr' + logger.colors.reset + logger.colors.fg.pink + ' has started.'): + if os.path.exists('static-data/header.txt'): + with open('static-data/header.txt', 'rb') as file: + # only to stdout, not file or log or anything + sys.stderr.write(file.read().decode().replace('P', logger.colors.fg.pink).replace('W', logger.colors.reset + logger.colors.bold).replace('G', logger.colors.fg.green).replace('\n', logger.colors.reset + '\n').replace('B', logger.colors.bold).replace('V', ONIONR_VERSION)) + logger.info(logger.colors.fg.lightgreen + '-> ' + str(message) + logger.colors.reset + logger.colors.fg.lightgreen + ' <-\n') + if __name__ == "__main__": Onionr() diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index 36264600..f269f028 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -61,5 +61,11 @@ class DaemonTools: if not self.daemon._core._utils.checkNetwork(): logger.warn('Network check failed, are you connected to the internet?') self.daemon.isOnline = False - - self.daemon.decrementThreadCount('netCheck') \ No newline at end of file + self.daemon.decrementThreadCount('netCheck') + + def cleanOldBlocks(self): + '''Delete old blocks if our disk allocation is full/near full''' + if self.daemon._core._utils.storageCounter.isFull(): + + + self.daemon.decrementThreadCount('cleanOldBlocks') \ No newline at end of file diff --git a/onionr/onionrexceptions.py b/onionr/onionrexceptions.py index 040bc9be..8044508f 100644 --- a/onionr/onionrexceptions.py +++ b/onionr/onionrexceptions.py @@ -61,5 +61,5 @@ class InvalidAddress(Exception): # file exceptions -class DiskAllocationReached: +class DiskAllocationReached(Exception): pass \ No newline at end of file diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index b48160a3..d1383cab 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -384,10 +384,6 @@ class OnionrUtils: pass else: retData = True - if retData: - # Executes if data not seen - with open(self._core.dataNonceFile, 'a') as nonceFile: - nonceFile.write(nonce + '\n') else: logger.warn('In call to utils.validateMetadata, metadata must be JSON string or a dictionary object') diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 86d44499..4b53f391 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -51,7 +51,7 @@ }, "allocations":{ - "disk": 9000000000, + "disk": 800, "netTotal": 1000000000, "blockCache": 5000000, "blockCacheTotal": 50000000 diff --git a/onionr/storagecounter.py b/onionr/storagecounter.py index 481d7821..a5c11f21 100644 --- a/onionr/storagecounter.py +++ b/onionr/storagecounter.py @@ -27,7 +27,7 @@ class StorageCounter: def isFull(self): retData = False - if self._core.config.get('allocations.disk') <= self.getAmount(): + if self._core.config.get('allocations.disk') <= (self.getAmount() + 100): retData = True return retData