Added block database cleaner

src/blockdb/__init__.py

@@ -5,36 +5,19 @@ from onionrblocks import Block
 import db
 
 from .dbpath import block_db_path
-
+from .blockcleaner import clean_block_database
+from .getblocks import get_blocks_after_timestamp, get_blocks_by_type
+from .deleteblock import delete_block
 
 block_storage_observers: List[Callable] = []
 
 
+
+
 def add_block_to_db(block: Block):
     # Raises db.DuplicateKey if dupe
     db.set_if_new(block_db_path, block.id, block.raw)
 
-def get_blocks_by_type(block_type: str) -> "Generator[Block]":
-    block_db = db.get_db_obj(block_db_path, 'u')
-    for block_hash in db.list_keys(block_db_path):
-        block = Block(block_hash, block_db[block_hash], auto_verify=False)
-        if block.type == block_type:
-            yield block
-
-
-def get_blocks_after_timestamp(
-        timestamp: int, block_type: str = '') -> "Generator[Block]":
-    block_db = db.get_db_obj(block_db_path, 'u')
-
-    for block_hash in db.list_keys(block_db_path):
-        block = Block(block_hash, block_db[block_hash], auto_verify=False)
-        if block.timestamp > timestamp:
-            if block_type:
-                if block_type == block.type:
-                    yield block
-            else:
-                yield block
-
 
 def has_block(block_hash):
     return block_hash in db.list_keys(block_db_path)

src/blockdb/blockcleaner.py

@@ -0,0 +1,25 @@
+from typing import Set
+
+from onionrblocks import Block
+
+import logger
+
+from .deleteblock import delete_block
+from .getblocks import get_blocks_after_timestamp
+
+
+
+
+def clean_block_database():
+    """Delete expired blocks from block db"""
+    remove_set: Set[bytes] = set()
+    block: Block
+
+    for block in get_blocks_after_timestamp(0):
+        try:
+            Block(block.id, block.raw, auto_verify=True)
+        except ValueError:  # block expired
+            remove_set.add(block)
+    
+    logger.info(f"Cleaning {len(remove_set)} blocks", terminal=True)
+    [i for i in map(delete_block, remove_set)]

src/blockdb/deleteblock.py

@@ -0,0 +1,10 @@
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from onionrblocks import Block
+
+import db
+
+from .dbpath import block_db_path
+
+def delete_block(block: 'Block'):  db.delete(block_db_path, block.id)

src/blockdb/getblocks.py

@@ -0,0 +1,28 @@
+from typing import Generator
+
+import db
+
+from onionrblocks import Block
+
+from .dbpath import block_db_path
+
+def get_blocks_by_type(block_type: str) -> "Generator[Block]":
+    block_db = db.get_db_obj(block_db_path, 'u')
+    for block_hash in db.list_keys(block_db_path):
+        block = Block(block_hash, block_db[block_hash], auto_verify=False)
+        if block.type == block_type:
+            yield block
+
+
+def get_blocks_after_timestamp(
+        timestamp: int, block_type: str = '') -> "Generator[Block]":
+    block_db = db.get_db_obj(block_db_path, 'u')
+
+    for block_hash in db.list_keys(block_db_path):
+        block = Block(block_hash, block_db[block_hash], auto_verify=False)
+        if block.timestamp > timestamp:
+            if block_type:
+                if block_type == block.type:
+                    yield block
+            else:
+                yield block

src/db/__init__.py

@@ -24,6 +24,16 @@ def _do_timeout(func, *args):
             return res
 
 
+def delete(db_path, key):
+    def _delete(key):
+        with dbm.open(db_path, "c") as my_db:
+            del my_db[key]
+    try:
+        _do_timeout(_delete, key)
+    except KeyError:
+        pass
+
+
 def set_if_new(db_path, key, value) -> bool:
     def _set(key, value):
         with dbm.open(db_path, "c") as my_db:

src/onionrcommands/daemonlaunch/__init__.py

@@ -28,6 +28,8 @@ import filepaths
 import onionrvalues
 from onionrutils import cleanup
 from onionrcrypto import getourkeypair
+from onionrthreads import add_onionr_thread
+from blockdb.blockcleaner import clean_block_database
 import runtests
 from .. import version
 from .killdaemon import kill_daemon  # noqa
@@ -100,7 +102,6 @@ def daemon():
     # Run time tests are not normally run
     shared_state.get(runtests.OnionrRunTestManager)
 
-
     shared_state.share_object()  # share the parent object to the threads
 
     show_logo()
@@ -112,11 +113,15 @@ def daemon():
         f"Onionr daemon is running under pid {os.getpid()}", terminal=True)
     events.event('init', threaded=False)
     events.event('daemon_start')
+
+    add_onionr_thread(
+        clean_block_database, 60, 'clean_block_database', initial_sleep=0)
+
     Thread(
         target=gossip.start_gossip_threads, 
         daemon=True, 
         name='start_gossip_threads').start()
-
+    
     try:
         apiservers.private_api.start()
         events.event('shutdown', threaded=False)

tests/test_blockdb_cleaner.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+import sys, os
+import time
+import dbm
+sys.path.append(".")
+sys.path.append("src/")
+import uuid
+TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
+print("Test directory:", TEST_DIR)
+os.environ["ONIONR_HOME"] = TEST_DIR
+import unittest
+
+from utils import createdirs
+createdirs.create_dirs()
+
+
+import onionrblocks
+
+import blockdb
+
+
+def _delete_db():
+    try:
+        os.remove(blockdb.block_db_path)
+    except FileNotFoundError:
+        pass
+
+class TestBlockDBCleaner(unittest.TestCase):
+    def test_clean_block_with_others(self):
+        _delete_db()
+        test_bl_data = b'lcRKabk1Vs(7l19baZUZ34Q\ntest'
+        test_id = b'2374c534b8535a5bf35693448596c634dbea9d78a39f1519dfbcc47e8fcb25f7'.zfill(128)
+        
+        #onionrblocks.create_anonvdf_block()
+        blockdb.add_block_to_db(onionrblocks.blockcreator.create_anonvdf_block(b"test data", b"dat", 2420))
+        blockdb.add_block_to_db(onionrblocks.Block(test_id, test_bl_data, auto_verify=False))
+        self.assertEqual(len(list(blockdb.get_blocks_after_timestamp(0))), 2)
+
+        blockdb.clean_block_database()
+
+        self.assertEqual(len(list(blockdb.get_blocks_after_timestamp(0))), 1)
+    def test_clean_block_database(self):
+        _delete_db()
+        test_bl_data = b'lcRKabk1Vs(7l19baZUZ34Q\ntest'
+        test_id = b'2374c534b8535a5bf35693448596c634dbea9d78a39f1519dfbcc47e8fcb25f7'.zfill(128)
+        
+        #onionrblocks.create_anonvdf_block()
+        blockdb.add_block_to_db(onionrblocks.Block(test_id, test_bl_data, auto_verify=False))
+        self.assertEqual(len(list(blockdb.get_blocks_after_timestamp(0))), 1)
+
+        blockdb.clean_block_database()
+
+        self.assertEqual(len(list(blockdb.get_blocks_after_timestamp(0))), 0)
+
+unittest.main()