diff --git a/src/blockio/__init__.py b/src/blockio/__init__.py
index fa550df7..97900e09 100644
--- a/src/blockio/__init__.py
+++ b/src/blockio/__init__.py
@@ -4,7 +4,7 @@ Wrap safedb for storing and fetching blocks
"""
from .store import store_block
from .load import load_block, list_blocks_by_type
-from .cleanexpired import clean_expired_blocks
+from .clean import clean_expired_blocks, clean_block_list_entries
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
diff --git a/src/blockio/clean/__init__.py b/src/blockio/clean/__init__.py
new file mode 100644
index 00000000..aaf6d6d1
--- /dev/null
+++ b/src/blockio/clean/__init__.py
@@ -0,0 +1,2 @@
+from .cleanexpired import clean_expired_blocks
+from .cleanblocklistentries import clean_block_list_entries
\ No newline at end of file
diff --git a/src/blockio/clean/cleanblocklistentries.py b/src/blockio/clean/cleanblocklistentries.py
new file mode 100644
index 00000000..dd53fb96
--- /dev/null
+++ b/src/blockio/clean/cleanblocklistentries.py
@@ -0,0 +1,33 @@
+"""Onionr - Private P2P Communication.
+
+Delete block type lists that are empty
+"""
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+ from safedb import SafeDB
+"""
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+"""
+
+
+def clean_block_list_entries(db: 'SafeDB'):
+ key = db.db_conn.firstkey()
+ delete_keys = []
+ while key:
+ if key.startswith(b'bl-'):
+ if not db.get(key):
+ delete_keys.append(key)
+ key = db.db_conn.nextkey(key)
+ for key in delete_keys:
+ del db.db_conn[key]
diff --git a/src/blockio/cleanexpired.py b/src/blockio/clean/cleanexpired.py
similarity index 79%
rename from src/blockio/cleanexpired.py
rename to src/blockio/clean/cleanexpired.py
index badb4a92..bba2cc58 100644
--- a/src/blockio/cleanexpired.py
+++ b/src/blockio/clean/cleanexpired.py
@@ -9,7 +9,6 @@ from onionrblocks.generators.anonvdf import AnonVDFGenerator
from onionrblocks.exceptions import BlockExpired
if TYPE_CHECKING:
- from kasten.types import BlockChecksumBytes
from safedb import SafeDB
"""
This program is free software: you can redistribute it and/or modify
@@ -30,6 +29,7 @@ along with this program. If not, see .
def clean_expired_blocks(db: 'SafeDB'):
key = db.db_conn.firstkey()
delete_list = set()
+ # Scan all database keys and check kasten objs if they are a hash
while key:
try:
if key.startswith(b'bl-') or key.startswith(b'enc'):
@@ -37,7 +37,12 @@ def clean_expired_blocks(db: 'SafeDB'):
continue
Kasten(key, db.get(key), AnonVDFGenerator)
except BlockExpired:
+ block_type = Kasten(
+ key, db.get(key),
+ None, auto_check_generator=False).get_data_type()
+ db.db_conn[f'bl-{block_type}'] = \
+ db.db_conn[f'bl-{block_type}'].replace(key, b'')
delete_list.add(key)
key = db.db_conn.nextkey(key)
for key in delete_list:
- del db[key]
+ del db.db_conn[key]
diff --git a/tests/runtime-result.txt b/tests/runtime-result.txt
index 5b6a44e2..ceb63a9e 100644
--- a/tests/runtime-result.txt
+++ b/tests/runtime-result.txt
@@ -1 +1 @@
-1611429331
\ No newline at end of file
+1611612283
\ No newline at end of file
diff --git a/tests/test_blockio.py b/tests/test_blockio.py
index 8ab4b17b..7461df87 100644
--- a/tests/test_blockio.py
+++ b/tests/test_blockio.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
import sys, os
+
sys.path.append(".")
sys.path.append("src/")
import uuid
@@ -8,7 +9,7 @@ print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
import unittest
import time
-from utils import identifyhome, createdirs, bettersleep
+from utils import identifyhome, createdirs
from onionrsetup import setup_config, setup_default_plugins
createdirs.create_dirs()
@@ -22,6 +23,7 @@ from utils import identifyhome
import safedb
import blockio
+from blockio.clean.cleanblocklistentries import clean_block_list_entries
def _remove_db(path):
@@ -33,6 +35,17 @@ def _remove_db(path):
class TestBlockIO(unittest.TestCase):
+ def test_clean_blocklist_entries(self):
+ db_file = identifyhome.identify_home() + 'test.db'
+ db = safedb.SafeDB(db_file)
+ bl = blockcreator.create_anonvdf_block(b"hello" + int(10).to_bytes(1, "big"), b"txt", 5)
+ blockio.store_block(bl, db)
+ db.db_conn[b'bl-txt'] = b''
+ clean_block_list_entries(db)
+ self.assertRaises(KeyError, db.get, 'bl-txt')
+ db.close()
+ _remove_db(db_file)
+
def test_clean_expired(self):
db_file = identifyhome.identify_home() + 'test.db'
@@ -40,12 +53,12 @@ class TestBlockIO(unittest.TestCase):
for i in range(3):
bl = blockcreator.create_anonvdf_block(b"hello" + int(i).to_bytes(1, "big"), b"txt", 5)
blockio.store_block(bl, db)
- print("done gening")
blockio.clean_expired_blocks(db)
time.sleep(1)
self.assertEqual(len(list(blockio.list_blocks_by_type("txt", db))), 3)
- time.sleep(4.1)
- blockio.list_blocks_by_type("txt", db)
+ time.sleep(10.1)
+ blockio.clean_expired_blocks(db)
+ self.assertEqual(len(db.db_conn[b'bl-txt']), 0)
db.close()
_remove_db(db_file)
diff --git a/tests/test_safedb.py b/tests/test_safedb.py
index 1e709bdf..3db4183b 100644
--- a/tests/test_safedb.py
+++ b/tests/test_safedb.py
@@ -44,7 +44,7 @@ class TestSafeDB(unittest.TestCase):
db['enc'] = b'1'
db = safedb.SafeDB(db_path, protected=True)
db.close()
- self.assertRaises(ValueError, safedb.SafeDB, db_path, protected=False)
+ self.assertRaises(safedb.DBProtectionOpeningModeError, safedb.SafeDB, db_path, protected=False)
def test_db_open_unprotected(self):
_remove_db()
@@ -52,7 +52,7 @@ class TestSafeDB(unittest.TestCase):
db['enc'] = b'0'
db = safedb.SafeDB(db_path, protected=False)
db.close()
- self.assertRaises(ValueError, safedb.SafeDB, db_path, protected=True)
+ self.assertRaises(safedb.DBProtectionOpeningModeError, safedb.SafeDB, db_path, protected=True)
def test_db_put_unprotected(self):
_remove_db()
@@ -60,8 +60,6 @@ class TestSafeDB(unittest.TestCase):
db.put("test", b"Test")
db.close()
with dbm.open(db_path, 'c') as db:
- self.assertEqual(db['test', b"Test"])
-
-
+ self.assertEqual(db['test'], b"Test")
unittest.main()