diff --git a/src/blockdb/__init__.py b/src/blockdb/__init__.py index a27c113a..67968cd8 100644 --- a/src/blockdb/__init__.py +++ b/src/blockdb/__init__.py @@ -5,36 +5,19 @@ from onionrblocks import Block import db from .dbpath import block_db_path - +from .blockcleaner import clean_block_database +from .getblocks import get_blocks_after_timestamp, get_blocks_by_type +from .deleteblock import delete_block block_storage_observers: List[Callable] = [] + + def add_block_to_db(block: Block): # Raises db.DuplicateKey if dupe db.set_if_new(block_db_path, block.id, block.raw) -def get_blocks_by_type(block_type: str) -> "Generator[Block]": - block_db = db.get_db_obj(block_db_path, 'u') - for block_hash in db.list_keys(block_db_path): - block = Block(block_hash, block_db[block_hash], auto_verify=False) - if block.type == block_type: - yield block - - -def get_blocks_after_timestamp( - timestamp: int, block_type: str = '') -> "Generator[Block]": - block_db = db.get_db_obj(block_db_path, 'u') - - for block_hash in db.list_keys(block_db_path): - block = Block(block_hash, block_db[block_hash], auto_verify=False) - if block.timestamp > timestamp: - if block_type: - if block_type == block.type: - yield block - else: - yield block - def has_block(block_hash): return block_hash in db.list_keys(block_db_path) diff --git a/src/blockdb/blockcleaner.py b/src/blockdb/blockcleaner.py new file mode 100644 index 00000000..b7fa0867 --- /dev/null +++ b/src/blockdb/blockcleaner.py @@ -0,0 +1,25 @@ +from typing import Set + +from onionrblocks import Block + +import logger + +from .deleteblock import delete_block +from .getblocks import get_blocks_after_timestamp + + + + +def clean_block_database(): + """Delete expired blocks from block db""" + remove_set: Set[bytes] = set() + block: Block + + for block in get_blocks_after_timestamp(0): + try: + Block(block.id, block.raw, auto_verify=True) + except ValueError: # block expired + remove_set.add(block) + + logger.info(f"Cleaning {len(remove_set)} blocks", terminal=True) + [i for i in map(delete_block, remove_set)] diff --git a/src/blockdb/deleteblock.py b/src/blockdb/deleteblock.py new file mode 100644 index 00000000..652f08ad --- /dev/null +++ b/src/blockdb/deleteblock.py @@ -0,0 +1,10 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from onionrblocks import Block + +import db + +from .dbpath import block_db_path + +def delete_block(block: 'Block'): db.delete(block_db_path, block.id) \ No newline at end of file diff --git a/src/blockdb/getblocks.py b/src/blockdb/getblocks.py new file mode 100644 index 00000000..a780838c --- /dev/null +++ b/src/blockdb/getblocks.py @@ -0,0 +1,28 @@ +from typing import Generator + +import db + +from onionrblocks import Block + +from .dbpath import block_db_path + +def get_blocks_by_type(block_type: str) -> "Generator[Block]": + block_db = db.get_db_obj(block_db_path, 'u') + for block_hash in db.list_keys(block_db_path): + block = Block(block_hash, block_db[block_hash], auto_verify=False) + if block.type == block_type: + yield block + + +def get_blocks_after_timestamp( + timestamp: int, block_type: str = '') -> "Generator[Block]": + block_db = db.get_db_obj(block_db_path, 'u') + + for block_hash in db.list_keys(block_db_path): + block = Block(block_hash, block_db[block_hash], auto_verify=False) + if block.timestamp > timestamp: + if block_type: + if block_type == block.type: + yield block + else: + yield block \ No newline at end of file diff --git a/src/db/__init__.py b/src/db/__init__.py index 1bb763dc..2860cd75 100644 --- a/src/db/__init__.py +++ b/src/db/__init__.py @@ -24,6 +24,16 @@ def _do_timeout(func, *args): return res +def delete(db_path, key): + def _delete(key): + with dbm.open(db_path, "c") as my_db: + del my_db[key] + try: + _do_timeout(_delete, key) + except KeyError: + pass + + def set_if_new(db_path, key, value) -> bool: def _set(key, value): with dbm.open(db_path, "c") as my_db: diff --git a/src/onionrcommands/daemonlaunch/__init__.py b/src/onionrcommands/daemonlaunch/__init__.py index a8ad51be..cc89c974 100755 --- a/src/onionrcommands/daemonlaunch/__init__.py +++ b/src/onionrcommands/daemonlaunch/__init__.py @@ -28,6 +28,8 @@ import filepaths import onionrvalues from onionrutils import cleanup from onionrcrypto import getourkeypair +from onionrthreads import add_onionr_thread +from blockdb.blockcleaner import clean_block_database import runtests from .. import version from .killdaemon import kill_daemon # noqa @@ -100,7 +102,6 @@ def daemon(): # Run time tests are not normally run shared_state.get(runtests.OnionrRunTestManager) - shared_state.share_object() # share the parent object to the threads show_logo() @@ -112,11 +113,15 @@ def daemon(): f"Onionr daemon is running under pid {os.getpid()}", terminal=True) events.event('init', threaded=False) events.event('daemon_start') + + add_onionr_thread( + clean_block_database, 60, 'clean_block_database', initial_sleep=0) + Thread( target=gossip.start_gossip_threads, daemon=True, name='start_gossip_threads').start() - + try: apiservers.private_api.start() events.event('shutdown', threaded=False) diff --git a/tests/test_blockdb_cleaner.py b/tests/test_blockdb_cleaner.py new file mode 100644 index 00000000..31d0f68d --- /dev/null +++ b/tests/test_blockdb_cleaner.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +import sys, os +import time +import dbm +sys.path.append(".") +sys.path.append("src/") +import uuid +TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' +print("Test directory:", TEST_DIR) +os.environ["ONIONR_HOME"] = TEST_DIR +import unittest + +from utils import createdirs +createdirs.create_dirs() + + +import onionrblocks + +import blockdb + + +def _delete_db(): + try: + os.remove(blockdb.block_db_path) + except FileNotFoundError: + pass + +class TestBlockDBCleaner(unittest.TestCase): + def test_clean_block_with_others(self): + _delete_db() + test_bl_data = b'lcRKabk1Vs(7l19baZUZ34Q\ntest' + test_id = b'2374c534b8535a5bf35693448596c634dbea9d78a39f1519dfbcc47e8fcb25f7'.zfill(128) + + #onionrblocks.create_anonvdf_block() + blockdb.add_block_to_db(onionrblocks.blockcreator.create_anonvdf_block(b"test data", b"dat", 2420)) + blockdb.add_block_to_db(onionrblocks.Block(test_id, test_bl_data, auto_verify=False)) + self.assertEqual(len(list(blockdb.get_blocks_after_timestamp(0))), 2) + + blockdb.clean_block_database() + + self.assertEqual(len(list(blockdb.get_blocks_after_timestamp(0))), 1) + def test_clean_block_database(self): + _delete_db() + test_bl_data = b'lcRKabk1Vs(7l19baZUZ34Q\ntest' + test_id = b'2374c534b8535a5bf35693448596c634dbea9d78a39f1519dfbcc47e8fcb25f7'.zfill(128) + + #onionrblocks.create_anonvdf_block() + blockdb.add_block_to_db(onionrblocks.Block(test_id, test_bl_data, auto_verify=False)) + self.assertEqual(len(list(blockdb.get_blocks_after_timestamp(0))), 1) + + blockdb.clean_block_database() + + self.assertEqual(len(list(blockdb.get_blocks_after_timestamp(0))), 0) + +unittest.main()