sync improvements, bug fixes, config changes

onionr/communicator2.py

@@ -80,10 +80,6 @@ class OnionrCommunicatorDaemon:
         if debug or developmentMode:
             OnionrCommunicatorTimers(self, self.heartbeat, 10)
 
-        # Print nice header thing :)
-        if config.get('general.display_header', True) and not self.shutdown:
-            self.header()
-
         # Set timers, function reference, seconds
         # requiresPeer True means the timer function won't fire if we have no connected peers
         # TODO: make some of these timer counts configurable
@@ -93,6 +89,7 @@ class OnionrCommunicatorDaemon:
         OnionrCommunicatorTimers(self, self.lookupBlocks, 7, requiresPeer=True, maxThreads=1)
         OnionrCommunicatorTimers(self, self.getBlocks, 10, requiresPeer=True)
         OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58)
+        OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 650)
         OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True)
         OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True)
         netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600)
@@ -152,6 +149,8 @@ class OnionrCommunicatorDaemon:
         triedPeers = [] # list of peers we've tried this time around
         for i in range(tryAmount):
             # check if disk allocation is used
+            if not self.isOnline:
+                break
             if self._core._utils.storageCounter.isFull():
                 logger.warn('Not looking up new blocks due to maximum amount of allowed disk space used')
                 break
@@ -188,8 +187,8 @@ class OnionrCommunicatorDaemon:
     def getBlocks(self):
         '''download new blocks in queue'''
         for blockHash in self.blockQueue:
-            if self.shutdown:
+            if self.shutdown or not self.isOnline:
-                # Exit loop if shutting down
+                # Exit loop if shutting down or offline
                 break
             # Do not download blocks being downloaded or that are already saved (edge cases)
             if blockHash in self.currentDownloading:
@@ -221,11 +220,11 @@ class OnionrCommunicatorDaemon:
                     #meta = metas[1]
                     if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce
                         if self._core._crypto.verifyPow(content): # check if POW is enough/correct
-                            logger.info('Block passed proof, saving.')
+                            logger.info('Block passed proof, attemping save.')
                             try:
                                 self._core.setData(content)
                             except onionrexceptions.DiskAllocationReached:
-                                logger.error("Reached disk allocation allowance, cannot save additional blocks.")
+                                logger.error("Reached disk allocation allowance, cannot save this block.")
                             else:
                                 self._core.addToBlockDB(blockHash, dataSaved=True)
                                 self._core._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
@@ -489,13 +488,6 @@ class OnionrCommunicatorDaemon:
                 self.shutdown = True
         self.decrementThreadCount('detectAPICrash')
 
-    def header(self, message = logger.colors.fg.pink + logger.colors.bold + 'Onionr' + logger.colors.reset + logger.colors.fg.pink + ' has started.'):
-        if os.path.exists('static-data/header.txt'):
-            with open('static-data/header.txt', 'rb') as file:
-                # only to stdout, not file or log or anything
-                sys.stderr.write(file.read().decode().replace('P', logger.colors.fg.pink).replace('W', logger.colors.reset + logger.colors.bold).replace('G', logger.colors.fg.green).replace('\n', logger.colors.reset + '\n').replace('B', logger.colors.bold).replace('V', onionr.ONIONR_VERSION))
-                logger.info(logger.colors.fg.lightgreen + '-> ' + str(message) + logger.colors.reset + logger.colors.fg.lightgreen + ' <-\n')
-
 class OnionrCommunicatorTimers:
     def __init__(self, daemonInstance, timerFunction, frequency, makeThread=True, threadAmount=1, maxThreads=5, requiresPeer=False):
         self.timerFunction = timerFunction

onionr/core.py

@@ -183,8 +183,16 @@ class Core:
             c.execute('Delete from hashes where hash=?;', t)
             conn.commit()
             conn.close()
+            blockFile = 'data/blocks/' + block + '.dat'
+            dataSize = 0
             try:
-                os.remove('data/blocks/' + block + '.dat')
+                ''' Get size of data when loaded as an object/var, rather than on disk, 
+                    to avoid conflict with getsizeof when saving blocks
+                '''
+                with open(blockFile, 'r') as data:
+                    dataSize = sys.getsizeof(data.read())
+                self._utils.storageCounter.removeBytes(dataSize)
+                os.remove(blockFile)
             except FileNotFoundError:
                 pass
 
@@ -280,6 +288,8 @@ class Core:
                 c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = '" + dataHash + "';")
                 conn.commit()
                 conn.close()
+                with open(self.dataNonceFile, 'a') as nonceFile:
+                    nonceFile.write(dataHash + '\n')
             else:
                 raise onionrexceptions.DiskAllocationReached
 
@@ -544,7 +554,7 @@ class Core:
         if unsaved:
             execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
         else:
-            execute = 'SELECT hash FROM hashes ORDER BY dateReceived DESC;'
+            execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;'
         rows = list()
         for row in c.execute(execute):
             for i in row:

onionr/onionr.py

@@ -588,6 +588,9 @@ class Onionr:
                 time.sleep(1)
                 #TODO make runable on windows
                 subprocess.Popen([communicatorDaemon, "run", str(net.socksPort)])
+                # Print nice header thing :)
+                if config.get('general.display_header', True):
+                    self.header()
                 logger.debug('Started communicator')
                 events.event('daemon_start', onionr = self)
                 try:
@@ -759,5 +762,12 @@ class Onionr:
         print('Opening %s ...' % url)
         webbrowser.open(url, new = 1, autoraise = True)
 
+    def header(self, message = logger.colors.fg.pink + logger.colors.bold + 'Onionr' + logger.colors.reset + logger.colors.fg.pink + ' has started.'):
+        if os.path.exists('static-data/header.txt'):
+            with open('static-data/header.txt', 'rb') as file:
+                # only to stdout, not file or log or anything
+                sys.stderr.write(file.read().decode().replace('P', logger.colors.fg.pink).replace('W', logger.colors.reset + logger.colors.bold).replace('G', logger.colors.fg.green).replace('\n', logger.colors.reset + '\n').replace('B', logger.colors.bold).replace('V', ONIONR_VERSION))
+                logger.info(logger.colors.fg.lightgreen + '-> ' + str(message) + logger.colors.reset + logger.colors.fg.lightgreen + ' <-\n')
+
 if __name__ == "__main__":
     Onionr()

onionr/onionrdaemontools.py

@@ -61,5 +61,11 @@ class DaemonTools:
             if not self.daemon._core._utils.checkNetwork():
                 logger.warn('Network check failed, are you connected to the internet?')
                 self.daemon.isOnline = False
-
+        self.daemon.decrementThreadCount('netCheck')
-        self.daemon.decrementThreadCount('netCheck')
+    
+    def cleanOldBlocks(self):
+        '''Delete old blocks if our disk allocation is full/near full'''
+        if self.daemon._core._utils.storageCounter.isFull():
+            
+        
+        self.daemon.decrementThreadCount('cleanOldBlocks')

onionr/onionrexceptions.py

@@ -61,5 +61,5 @@ class InvalidAddress(Exception):
 
 # file exceptions
 
-class DiskAllocationReached:
+class DiskAllocationReached(Exception):
     pass

onionr/onionrutils.py

@@ -384,10 +384,6 @@ class OnionrUtils:
                     pass
                 else:
                     retData = True
-                if retData:
-                    # Executes if data not seen
-                    with open(self._core.dataNonceFile, 'a') as nonceFile:
-                        nonceFile.write(nonce + '\n')
         else:
             logger.warn('In call to utils.validateMetadata, metadata must be JSON string or a dictionary object')
 

onionr/static-data/default_config.json

@@ -51,7 +51,7 @@
      },
 
     "allocations":{
-        "disk": 9000000000,
+        "disk": 800,
         "netTotal": 1000000000,
         "blockCache": 5000000,
         "blockCacheTotal": 50000000

onionr/storagecounter.py

@@ -27,7 +27,7 @@ class StorageCounter:
 
     def isFull(self):
         retData = False
-        if self._core.config.get('allocations.disk') <= self.getAmount():
+        if self._core.config.get('allocations.disk') <= (self.getAmount() + 100):
             retData = True
         return retData