uploads now have sessions

onionr/communicatorutils/downloadblocks/__init__.py

@@ -30,7 +30,8 @@ def download_blocks_from_communicator(comm_inst):
     blacklist = onionrblacklist.OnionrBlackList()
     storage_counter = storagecounter.StorageCounter()
     LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
-    count = 0
+    count: int = 0
+    metadata_validation_result: bool = False
     # Iterate the block queue in the communicator
     for blockHash in list(comm_inst.blockQueue):
         count += 1
@@ -80,7 +81,11 @@ def download_blocks_from_communicator(comm_inst):
                 #content = content.decode() # decode here because sha3Hash needs bytes above
                 metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
                 metadata = metas[0]
-                if validatemetadata.validate_metadata(metadata, metas[2]): # check if metadata is valid, and verify nonce
+                try:
+                    metadata_validation_result = validatemetadata.validate_metadata(metadata, metas[2])
+                except onionrexceptions.DataExists:
+                    metadata_validation_result = False
+                if metadata_validation_result: # check if metadata is valid, and verify nonce
                     if onionrcrypto.cryptoutils.verify_POW(content): # check if POW is enough/correct
                         logger.info('Attempting to save block %s...' % blockHash[:12])
                         try:

onionr/communicatorutils/uploadblocks/__init__.py

@@ -74,6 +74,7 @@ def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
                         session.success()
                         session.peer_exists[peer] = True
                     elif resp == 'exists':
+                        session.success()
                         session.peer_exists[peer] = True
                     else:
                         session.fail()

onionr/communicatorutils/uploadblocks/sessionmanager.py

@@ -72,4 +72,14 @@ class BlockUploadSessionManager:
                 sessions_to_delete.append(session)
         for session in sessions_to_delete:
             self.sessions.remove(session)
+            # TODO cleanup to one round of search
+            # Remove the blocks from the sessions, upload list, and waitforshare list
+            try:
+                comm_inst.blocksToUpload.remove(reconstructhash.reconstruct_hash(session.block_hash))
+            except ValueError:
+                pass
+            try:
+                comm_inst.blocksToUpload.remove(session.block_hash)
+            except ValueError:
+                pass
             localcommand.local_command('waitforshare/{session.block_hash}')

onionr/onionrutils/validatemetadata.py

@@ -93,13 +93,14 @@ def validate_metadata(metadata, block_data) -> bool:
             try:
                 with open(filepaths.data_nonce_file, 'r') as nonceFile:
                     if nonce in nonceFile.read():
-                        ret_data = False # we've seen that nonce before, so we can't pass metadata
+                        # we've seen that nonce before, so we can't pass metadata
                         raise onionrexceptions.DataExists
             except FileNotFoundError:
                 ret_data = True
             except onionrexceptions.DataExists:
-                # do not set ret_data to True, because nonce has been seen before
-                pass
+                # do not set ret_data to True, because data has been seen before
+                logger.warn(f'{nonce} seen before')
+                raise onionrexceptions.DataExists
             else:
                 ret_data = True
     else: