properly clean block list in blockio

src/blockio/__init__.py

@@ -4,7 +4,7 @@ Wrap safedb for storing and fetching blocks
 """
 from .store import store_block
 from .load import load_block, list_blocks_by_type
-from .cleanexpired import clean_expired_blocks
+from .clean import clean_expired_blocks, clean_block_list_entries
 """
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

src/blockio/clean/__init__.py

@@ -0,0 +1,2 @@
+from .cleanexpired import clean_expired_blocks
+from .cleanblocklistentries import clean_block_list_entries

src/blockio/clean/cleanblocklistentries.py

@@ -0,0 +1,33 @@
+"""Onionr - Private P2P Communication.
+
+Delete block type lists that are empty
+"""
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from safedb import SafeDB
+"""
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+
+def clean_block_list_entries(db: 'SafeDB'):
+    key = db.db_conn.firstkey()
+    delete_keys = []
+    while key:
+        if key.startswith(b'bl-'):
+            if not db.get(key):
+                delete_keys.append(key)
+        key = db.db_conn.nextkey(key)
+    for key in delete_keys:
+        del db.db_conn[key]

src/blockio/clean/cleanexpired.py

@@ -9,7 +9,6 @@ from onionrblocks.generators.anonvdf import AnonVDFGenerator
 from onionrblocks.exceptions import BlockExpired
 
 if TYPE_CHECKING:
-    from kasten.types import BlockChecksumBytes
     from safedb import SafeDB
 """
 This program is free software: you can redistribute it and/or modify
@@ -30,6 +29,7 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 def clean_expired_blocks(db: 'SafeDB'):
     key = db.db_conn.firstkey()
     delete_list = set()
+    # Scan all database keys and check kasten objs if they are a hash
     while key:
         try:
             if key.startswith(b'bl-') or key.startswith(b'enc'):
@@ -37,7 +37,12 @@ def clean_expired_blocks(db: 'SafeDB'):
                 continue
             Kasten(key, db.get(key), AnonVDFGenerator)
         except BlockExpired:
+            block_type = Kasten(
+                key, db.get(key),
+                None, auto_check_generator=False).get_data_type()
+            db.db_conn[f'bl-{block_type}'] = \
+                db.db_conn[f'bl-{block_type}'].replace(key, b'')
             delete_list.add(key)
         key = db.db_conn.nextkey(key)
     for key in delete_list:
-        del db[key]
+        del db.db_conn[key]

tests/runtime-result.txt

@@ -1 +1 @@
-1611429331
+1611612283

tests/test_blockio.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import sys, os
+
 sys.path.append(".")
 sys.path.append("src/")
 import uuid
@@ -8,7 +9,7 @@ print("Test directory:", TEST_DIR)
 os.environ["ONIONR_HOME"] = TEST_DIR
 import unittest
 import time
-from utils import identifyhome, createdirs, bettersleep
+from utils import identifyhome, createdirs
 from onionrsetup import setup_config, setup_default_plugins
 
 createdirs.create_dirs()
@@ -22,6 +23,7 @@ from utils import identifyhome
 
 import safedb
 import blockio
+from blockio.clean.cleanblocklistentries import clean_block_list_entries
 
 
 def _remove_db(path):
@@ -33,6 +35,17 @@ def _remove_db(path):
 
 class TestBlockIO(unittest.TestCase):
 
+    def test_clean_blocklist_entries(self):
+        db_file = identifyhome.identify_home() + 'test.db'
+        db = safedb.SafeDB(db_file)
+        bl = blockcreator.create_anonvdf_block(b"hello" + int(10).to_bytes(1, "big"), b"txt", 5)
+        blockio.store_block(bl, db)
+        db.db_conn[b'bl-txt'] = b''
+        clean_block_list_entries(db)
+        self.assertRaises(KeyError, db.get, 'bl-txt')
+        db.close()
+        _remove_db(db_file)
+
     def test_clean_expired(self):
 
         db_file = identifyhome.identify_home() + 'test.db'
@@ -40,12 +53,12 @@ class TestBlockIO(unittest.TestCase):
         for i in range(3):
             bl = blockcreator.create_anonvdf_block(b"hello" + int(i).to_bytes(1, "big"), b"txt", 5)
             blockio.store_block(bl, db)
-        print("done gening")
         blockio.clean_expired_blocks(db)
         time.sleep(1)
         self.assertEqual(len(list(blockio.list_blocks_by_type("txt", db))), 3)
-        time.sleep(4.1)
-        blockio.list_blocks_by_type("txt", db)
+        time.sleep(10.1)
+        blockio.clean_expired_blocks(db)
+        self.assertEqual(len(db.db_conn[b'bl-txt']), 0)
         db.close()
         _remove_db(db_file)
 

tests/test_safedb.py

@@ -44,7 +44,7 @@ class TestSafeDB(unittest.TestCase):
             db['enc'] = b'1'
         db = safedb.SafeDB(db_path, protected=True)
         db.close()
-        self.assertRaises(ValueError, safedb.SafeDB, db_path, protected=False)
+        self.assertRaises(safedb.DBProtectionOpeningModeError, safedb.SafeDB, db_path, protected=False)
 
     def test_db_open_unprotected(self):
         _remove_db()
@@ -52,7 +52,7 @@ class TestSafeDB(unittest.TestCase):
             db['enc'] = b'0'
         db = safedb.SafeDB(db_path, protected=False)
         db.close()
-        self.assertRaises(ValueError, safedb.SafeDB, db_path, protected=True)
+        self.assertRaises(safedb.DBProtectionOpeningModeError, safedb.SafeDB, db_path, protected=True)
 
     def test_db_put_unprotected(self):
         _remove_db()
@@ -60,8 +60,6 @@ class TestSafeDB(unittest.TestCase):
         db.put("test", b"Test")
         db.close()
         with dbm.open(db_path, 'c') as db:
-            self.assertEqual(db['test', b"Test"])
-
-
+            self.assertEqual(db['test'], b"Test")
 
 unittest.main()