diff --git a/src/communicatorutils/downloadblocks/__init__.py b/src/communicatorutils/downloadblocks/__init__.py index 08cf9a1e..26b38e52 100755 --- a/src/communicatorutils/downloadblocks/__init__.py +++ b/src/communicatorutils/downloadblocks/__init__.py @@ -25,6 +25,7 @@ import onionrcrypto import onionrstorage from onionrblocks import onionrblacklist from onionrblocks import storagecounter +from onionrproofs import vdf from . import shoulddownload """ This program is free software: you can redistribute it and/or modify @@ -100,12 +101,7 @@ def download_blocks_from_communicator(shared_state: "TooMany"): except AttributeError: pass - realHash = onionrcrypto.hashers.sha3_hash(content) - try: - realHash = realHash.decode() # bytes on some versions for some reason - except AttributeError: - pass - if realHash == blockHash: + if vdf.verify_vdf(blockHash, content): #content = content.decode() # decode here because sha3Hash needs bytes above metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata metadata = metas[0] @@ -118,30 +114,10 @@ def download_blocks_from_communicator(shared_state: "TooMany"): except onionrexceptions.DataExists: metadata_validation_result = False if metadata_validation_result: # check if metadata is valid, and verify nonce - if onionrcrypto.cryptoutils.verify_POW(content): # check if POW is enough/correct - logger.info('Attempting to save block %s...' % blockHash[:12]) - try: - onionrstorage.set_data(content) - except onionrexceptions.DataExists: - logger.warn('Data is already set for %s ' % (blockHash,)) - except onionrexceptions.DiskAllocationReached: - logger.error('Reached disk allocation allowance, cannot save block %s.' % (blockHash,)) - removeFromQueue = False - else: - blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db - blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database - spawn( - local_command, - f'/daemon-event/upload_event', - post=True, - is_json=True, - post_data={'block': blockHash} - ) - else: - logger.warn('POW failed for block %s.' % (blockHash,)) + save_block(blockHash, data) else: - if blacklist.inBlacklist(realHash): - logger.warn('Block %s is blacklisted.' % (realHash,)) + if blacklist.inBlacklist(blockHash): + logger.warn(f'Block {blockHash} is blacklisted.') else: logger.warn('Metadata for block %s is invalid.' % (blockHash,)) blacklist.addToDB(blockHash) @@ -154,13 +130,6 @@ def download_blocks_from_communicator(shared_state: "TooMany"): pass # Punish peer for sharing invalid block (not always malicious, but is bad regardless) onionrpeers.PeerProfiles(peerUsed).addScore(-50) - if tempHash != 'ed55e34cb828232d6c14da0479709bfa10a0923dca2b380496e6b2ed4f7a0253': - # Dumb hack for 404 response from peer. Don't log it if 404 since its likely not malicious or a critical error. - logger.warn( - 'Block hash validation failed for ' + - blockHash + ' got ' + tempHash) - else: - removeFromQueue = False # Don't remove from queue if 404 if removeFromQueue: try: del kv.get('blockQueue')[blockHash] # remove from block queue both if success or false diff --git a/src/db/__init__.py b/src/db/__init__.py new file mode 100644 index 00000000..ef5daed9 --- /dev/null +++ b/src/db/__init__.py @@ -0,0 +1,37 @@ +import dbm +import time +import os + +timeout = 120 + + +def _do_timeout(func, *args): + ts = 0 + res = None + while True: + try: + res = func(*args) + except dbm.error: + if not ts: + ts = time.time() + continue + if time.time() - ts > timeout: + raise TimeoutError() + time.sleep(0.1) + else: + return res + + +def set(db_path, key, value): + def _set(key, value): + with dbm.open(db_path, "c") as my_db: + my_db[key] = value + _do_timeout(_set, key, value) + + +def get(db_path, key): + def _get(key): + with dbm.open(db_path, "c") as my_db: + return my_db[key] + return _do_timeout(_get, key) + diff --git a/src/filepaths/__init__.py b/src/filepaths/__init__.py index ba205383..5160b791 100644 --- a/src/filepaths/__init__.py +++ b/src/filepaths/__init__.py @@ -36,3 +36,5 @@ log_file = home + 'onionr.log' ephemeral_services_file = home + 'ephemeral-services.list' restarting_indicator = home + "is-restarting" + +block_database = home + "blocks.db" diff --git a/src/onionrblocks/storagecounter.py b/src/onionrblocks/storagecounter.py index ef35f64f..7ff0fe9a 100755 --- a/src/onionrblocks/storagecounter.py +++ b/src/onionrblocks/storagecounter.py @@ -84,7 +84,7 @@ class StorageCounter: new_amount = amount + self.amount ret_data = new_amount if new_amount > config.get('allocations.disk', 2000000000): - ret_data = False + ret_data = 0 else: self._update(new_amount) return ret_data diff --git a/src/onionrproofs/__init__.py b/src/onionrproofs/__init__.py index e0f4a956..17730e01 100755 --- a/src/onionrproofs/__init__.py +++ b/src/onionrproofs/__init__.py @@ -12,6 +12,7 @@ from onionrutils import bytesconverter from onionrcrypto import hashers from .blocknoncestart import BLOCK_NONCE_START_INT +from .vdf import create_vdf """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/onionrproofs/vdf.py b/src/onionrproofs/vdf.py index 4e1c7d20..845d9b67 100644 --- a/src/onionrproofs/vdf.py +++ b/src/onionrproofs/vdf.py @@ -7,9 +7,37 @@ def _wrap_vdf_create(queue, block_data_bytes, rounds): queue.put(mimcvdf.vdf_create(block_data_bytes, rounds)) -def do_vdf(block_data_bytes): +def _wrap_vdf_verify(queue, block_data_bytes, block_hash_hex, rounds): + queue.put(mimcvdf.vdf_verify(block_data_bytes, block_hash_hex, rounds)) + + + +def rounds_for_bytes(int: byte_count): + return byte_count * 1000 + + + +def create_vdf(block_data_bytes): + rounds = rounds_for_bytes(block_data_bytes) queue = multiprocessing.Queue() - vdf_proc = multiprocessing.Process(target=_wrap_vdf_create, args=(queue, block_data_bytes, 1000)) + vdf_proc = multiprocessing.Process( + target=_wrap_vdf_create, + args=(queue, block_data_bytes, rounds)) vdf_proc.start() vdf_proc.join() - return queue.get() \ No newline at end of file + return queue.get() + + +def verify_vdf(block_hash_hex, block_data_bytes): + rounds = rounds_for_bytes(block_data_bytes) + if rounds < 10 ** 6: + # >million rounds it starts to take long enough to warrant a subprocess + queue = multiprocessing.Queue() + vdf_proc = multiprocessing.Process( + target=_wrap_vdf_verify, + args=(queue, block_data_bytes, block_hash_hex, rounds)) + vdf_proc.start() + vdf_proc.join() + return queue.get() + return mimcvdf.vdf_verify(block_data_bytes, block_hash_hex, rounds) + diff --git a/src/onionrstorage/setdata.py b/src/onionrstorage/setdata.py index 456276f5..502ec086 100644 --- a/src/onionrstorage/setdata.py +++ b/src/onionrstorage/setdata.py @@ -31,9 +31,8 @@ from onionrtypes import BlockHash storage_counter = storagecounter.StorageCounter() -def set_data(data) -> BlockHash: +def set_data(data): """Set the data assciated with a hash.""" - data = data dataSize = sys.getsizeof(data) nonce_hash = crypto.hashers.sha3_hash( bytesconverter.str_to_bytes( @@ -50,7 +49,7 @@ def set_data(data) -> BlockHash: try: onionrstorage.getData(dataHash) except onionrexceptions.NoDataAvailable: - if storage_counter.add_bytes(dataSize) is not False: + if storage_counter.add_bytes(dataSize): onionrstorage.store(data, block_hash=dataHash) conn = sqlite3.connect( dbfiles.block_meta_db, timeout=DATABASE_LOCK_TIMEOUT)