From 3ef29077e78e84fbfdf46e247bb3b85ad3481721 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 10 Oct 2020 23:23:22 +0000 Subject: [PATCH] plaintext block fixes with sharing, fixed peer lookup --- src/communicatorutils/downloadblocks/__init__.py | 3 +++ src/communicatorutils/lookupadders.py | 1 + src/communicatorutils/uploadblocks/mixmate/__init__.py | 2 +- src/httpapi/miscpublicapi/upload.py | 3 +++ src/onionrblocks/deleteplaintext.py | 7 ++++--- src/onionrblocks/onionrblockapi.py | 2 -- src/onionrexceptions.py | 3 +++ src/onionrstorage/removeblock.py | 8 ++++++-- src/onionrutils/validatemetadata.py | 6 ++---- 9 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/communicatorutils/downloadblocks/__init__.py b/src/communicatorutils/downloadblocks/__init__.py index 9b564847..3060364c 100755 --- a/src/communicatorutils/downloadblocks/__init__.py +++ b/src/communicatorutils/downloadblocks/__init__.py @@ -111,6 +111,9 @@ def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"): try: metadata_validation_result = \ validatemetadata.validate_metadata(metadata, metas[2]) + except onionrexceptions.PlaintextNotSupported: + logger.debug(f"Not saving {blockHash} due to plaintext not enabled") + removeFromQueue = True except onionrexceptions.DataExists: metadata_validation_result = False if metadata_validation_result: # check if metadata is valid, and verify nonce diff --git a/src/communicatorutils/lookupadders.py b/src/communicatorutils/lookupadders.py index 738cce08..ae03ad39 100755 --- a/src/communicatorutils/lookupadders.py +++ b/src/communicatorutils/lookupadders.py @@ -64,3 +64,4 @@ def lookup_new_peer_transports_with_communicator(shared_state): except ValueError: pass kv.get('newPeers').extend(newPeers) + shared_state.get_by_string("OnionrCommunicatorDaemon").decrementThreadCount('clean_old_blocks') diff --git a/src/communicatorutils/uploadblocks/mixmate/__init__.py b/src/communicatorutils/uploadblocks/mixmate/__init__.py index 446ac44a..617fa163 100644 --- a/src/communicatorutils/uploadblocks/mixmate/__init__.py +++ b/src/communicatorutils/uploadblocks/mixmate/__init__.py @@ -38,7 +38,7 @@ def block_mixer(upload_list: List[onionrtypes.BlockHash], to the said block list """ bl = onionrblockapi.Block(block_to_mix) - + print(bl.raw) if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE: raise ValueError if block_to_mix: diff --git a/src/httpapi/miscpublicapi/upload.py b/src/httpapi/miscpublicapi/upload.py index 8054952c..da2609ad 100755 --- a/src/httpapi/miscpublicapi/upload.py +++ b/src/httpapi/miscpublicapi/upload.py @@ -65,6 +65,9 @@ def accept_upload(request): resp = 'proof' except onionrexceptions.DataExists: resp = 'exists' + except onionrexceptions.PlaintextNotSupported: + logger.debug("attempted plaintext upload to us: {b_hash}") + resp = 'failure' if resp == 'failure': abort(400) elif resp == 'proof': diff --git a/src/onionrblocks/deleteplaintext.py b/src/onionrblocks/deleteplaintext.py index 9789935e..427283be 100644 --- a/src/onionrblocks/deleteplaintext.py +++ b/src/onionrblocks/deleteplaintext.py @@ -6,6 +6,7 @@ from coredb import blockmetadb from onionrstorage.removeblock import remove_block import onionrstorage from .onionrblockapi import Block +import onionrexceptions """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -28,7 +29,7 @@ def delete_plaintext_no_blacklist(): block_list = blockmetadb.get_block_list() for block in block_list: - block = Block(hash=block) + block = Block(hash=block, decrypt=False) if not block.isEncrypted: - remove_block(block.hash) - onionrstorage.deleteBlock(block.hash) + remove_block(block.hash) # delete metadata entry + onionrstorage.deleteBlock(block.hash) # delete block data diff --git a/src/onionrblocks/onionrblockapi.py b/src/onionrblocks/onionrblockapi.py index 27bcac8f..753c08c6 100755 --- a/src/onionrblocks/onionrblockapi.py +++ b/src/onionrblocks/onionrblockapi.py @@ -3,7 +3,6 @@ OnionrBlocks class for abstraction of blocks """ import binascii -import os import datetime import onionrstorage @@ -17,7 +16,6 @@ from onionrusers import onionrusers from onionrutils import stringvalidators, epoch from coredb import blockmetadb from onionrutils import bytesconverter -from onionrstorage import removeblock import onionrblocks from onionrcrypto import encryption, cryptoutils as cryptoutils, signing """ diff --git a/src/onionrexceptions.py b/src/onionrexceptions.py index af5b6a9a..0f30c7b1 100755 --- a/src/onionrexceptions.py +++ b/src/onionrexceptions.py @@ -48,6 +48,9 @@ class SignatureError(Exception): # block exceptions +class PlaintextNotSupported(Exception): + pass + class ReplayAttack(Exception): pass diff --git a/src/onionrstorage/removeblock.py b/src/onionrstorage/removeblock.py index 26a3c0b6..6b9452ea 100644 --- a/src/onionrstorage/removeblock.py +++ b/src/onionrstorage/removeblock.py @@ -35,7 +35,10 @@ def remove_block(block): **You may want blacklist.addToDB(blockHash) """ if stringvalidators.validate_hash(block): - dataSize = sys.getsizeof(onionrstorage.getData(block)) + try: + data_size = sys.getsizeof(onionrstorage.getData(block)) + except onionrexceptions.NoDataAvailable: + data_size = 0 conn = sqlite3.connect( dbfiles.block_meta_db, timeout=DATABASE_LOCK_TIMEOUT) c = conn.cursor() @@ -43,6 +46,7 @@ def remove_block(block): c.execute('Delete from hashes where hash=?;', t) conn.commit() conn.close() - storage_counter.remove_bytes(dataSize) + if data_size: + storage_counter.remove_bytes(data_size) else: raise onionrexceptions.InvalidHexHash diff --git a/src/onionrutils/validatemetadata.py b/src/onionrutils/validatemetadata.py index 11970bdc..2f221a06 100644 --- a/src/onionrutils/validatemetadata.py +++ b/src/onionrutils/validatemetadata.py @@ -76,8 +76,6 @@ def validate_metadata(metadata, block_data) -> bool: elif i == 'encryptType': try: if not metadata[i] in ('asym', 'sym', ''): raise ValueError - if not config.get('general.store_plaintext_blocks', True): - break except ValueError: logger.warn('Invalid encryption mode') break @@ -100,9 +98,9 @@ def validate_metadata(metadata, block_data) -> bool: if not config.get('general.store_plaintext_blocks', True): try: if not metadata['encryptType']: - raise onionrexceptions.DataExists + raise onionrexceptions.PlaintextNotSupported except KeyError: - raise onionrexceptions.DataExists + raise onionrexceptions.PlaintextNotSupported try: metadata['time'] except KeyError: