From 7db3e208677127e0581dfc23a5fb1b446aaaeba4 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 19 Sep 2019 18:34:16 -0500 Subject: [PATCH] uploads now have sessions --- onionr/communicatorutils/downloadblocks/__init__.py | 9 +++++++-- onionr/communicatorutils/uploadblocks/__init__.py | 1 + .../communicatorutils/uploadblocks/sessionmanager.py | 10 ++++++++++ onionr/onionrutils/validatemetadata.py | 7 ++++--- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/onionr/communicatorutils/downloadblocks/__init__.py b/onionr/communicatorutils/downloadblocks/__init__.py index 7abf4c9e..cf6b5998 100755 --- a/onionr/communicatorutils/downloadblocks/__init__.py +++ b/onionr/communicatorutils/downloadblocks/__init__.py @@ -30,7 +30,8 @@ def download_blocks_from_communicator(comm_inst): blacklist = onionrblacklist.OnionrBlackList() storage_counter = storagecounter.StorageCounter() LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter - count = 0 + count: int = 0 + metadata_validation_result: bool = False # Iterate the block queue in the communicator for blockHash in list(comm_inst.blockQueue): count += 1 @@ -80,7 +81,11 @@ def download_blocks_from_communicator(comm_inst): #content = content.decode() # decode here because sha3Hash needs bytes above metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata metadata = metas[0] - if validatemetadata.validate_metadata(metadata, metas[2]): # check if metadata is valid, and verify nonce + try: + metadata_validation_result = validatemetadata.validate_metadata(metadata, metas[2]) + except onionrexceptions.DataExists: + metadata_validation_result = False + if metadata_validation_result: # check if metadata is valid, and verify nonce if onionrcrypto.cryptoutils.verify_POW(content): # check if POW is enough/correct logger.info('Attempting to save block %s...' % blockHash[:12]) try: diff --git a/onionr/communicatorutils/uploadblocks/__init__.py b/onionr/communicatorutils/uploadblocks/__init__.py index 27f205a3..fd776573 100755 --- a/onionr/communicatorutils/uploadblocks/__init__.py +++ b/onionr/communicatorutils/uploadblocks/__init__.py @@ -74,6 +74,7 @@ def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon): session.success() session.peer_exists[peer] = True elif resp == 'exists': + session.success() session.peer_exists[peer] = True else: session.fail() diff --git a/onionr/communicatorutils/uploadblocks/sessionmanager.py b/onionr/communicatorutils/uploadblocks/sessionmanager.py index 4e8f58d2..36fa5f49 100644 --- a/onionr/communicatorutils/uploadblocks/sessionmanager.py +++ b/onionr/communicatorutils/uploadblocks/sessionmanager.py @@ -72,4 +72,14 @@ class BlockUploadSessionManager: sessions_to_delete.append(session) for session in sessions_to_delete: self.sessions.remove(session) + # TODO cleanup to one round of search + # Remove the blocks from the sessions, upload list, and waitforshare list + try: + comm_inst.blocksToUpload.remove(reconstructhash.reconstruct_hash(session.block_hash)) + except ValueError: + pass + try: + comm_inst.blocksToUpload.remove(session.block_hash) + except ValueError: + pass localcommand.local_command('waitforshare/{session.block_hash}') \ No newline at end of file diff --git a/onionr/onionrutils/validatemetadata.py b/onionr/onionrutils/validatemetadata.py index ddb74455..405bf799 100644 --- a/onionr/onionrutils/validatemetadata.py +++ b/onionr/onionrutils/validatemetadata.py @@ -93,13 +93,14 @@ def validate_metadata(metadata, block_data) -> bool: try: with open(filepaths.data_nonce_file, 'r') as nonceFile: if nonce in nonceFile.read(): - ret_data = False # we've seen that nonce before, so we can't pass metadata + # we've seen that nonce before, so we can't pass metadata raise onionrexceptions.DataExists except FileNotFoundError: ret_data = True except onionrexceptions.DataExists: - # do not set ret_data to True, because nonce has been seen before - pass + # do not set ret_data to True, because data has been seen before + logger.warn(f'{nonce} seen before') + raise onionrexceptions.DataExists else: ret_data = True else: