added block cleaning method for new database

requirements.in

@@ -15,4 +15,4 @@ filenuke==0.0.0
 watchdog==1.0.2
 ujson==4.0.1
 cffi==1.14.4
-onionrblocks==4.0.0
+onionrblocks==4.1.0

requirements.txt

@@ -202,9 +202,9 @@ niceware==0.2.1 \
     --hash=sha256:0f8b192f2a1e800e068474f6e208be9c7e2857664b33a96f4045340de4e5c69c \
     --hash=sha256:cf2dc0e1567d36d067c61b32fed0f1b9c4534ed511f9eeead4ba548d03b5c9eb
     # via -r requirements.in
-onionrblocks==4.0.0 \
+onionrblocks==4.1.0 \
-    --hash=sha256:7a616228b2100f7d54021b825a921035299c8862150bb94379f5108ba7e2af78 \
+    --hash=sha256:2f806d1a4cf332ffef8630ac3d362499854316d957945be0a090c7ff6917a6c4 \
-    --hash=sha256:a5d87779930800d6364cabba560d24f0e2d491cffb180a16271a37a551a8eac0
+    --hash=sha256:bfdfa90df6fcdaef44b9ff1bc8f7b645fc41d10cf46481a3158d6c77e3832507
     # via -r requirements.in
 psutil==5.8.0 \
     --hash=sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64 \

src/blockio/__init__.py

@@ -4,6 +4,7 @@ Wrap safedb for storing and fetching blocks
 """
 from .store import store_block
 from .load import load_block, list_blocks_by_type
+from .cleanexpired import clean_expired_blocks
 """
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

src/blockio/cleanexpired.py

@@ -0,0 +1,43 @@
+"""Onionr - Private P2P Communication.
+
+clean expired blocks
+"""
+from typing import TYPE_CHECKING
+
+from kasten import Kasten
+from onionrblocks.generators.anonvdf import AnonVDFGenerator
+from onionrblocks.exceptions import BlockExpired
+
+if TYPE_CHECKING:
+    from kasten.types import BlockChecksumBytes
+    from safedb import SafeDB
+"""
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+
+def clean_expired_blocks(db: 'SafeDB'):
+    key = db.db_conn.firstkey()
+    delete_list = set()
+    while key:
+        try:
+            if key.startswith(b'bl-') or key.startswith(b'enc'):
+                key = db.db_conn.nextkey(key)
+                continue
+            Kasten(key, db.get(key), AnonVDFGenerator)
+        except BlockExpired:
+            delete_list.add(key)
+        key = db.db_conn.nextkey(key)
+    for key in delete_list:
+        del db[key]

src/blockio/getgenerator.py

@@ -1,23 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Get the kasten generator from a block metadata section
-"""
-from onionrblocks.generators import anonvdf
-"""
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
-"""
-
-GENERATOR_STRING_MAPPING = {
-    0: anonvdf.AnonVDFGenerator
-}

src/blockio/store/__init__.py

@@ -41,4 +41,3 @@ def store_block(block: 'Kasten', safe_db: 'SafeDB'):
     block_list_for_type += block.id
     safe_db.put(f'bl-{block_type}', block_list_for_type)
 
-

src/safedb/__init__.py

@@ -68,5 +68,3 @@ class SafeDB:
                 pass
 
         self.protected = protected
-
-

tests/test_blockio.py

@@ -7,6 +7,7 @@ TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
 print("Test directory:", TEST_DIR)
 os.environ["ONIONR_HOME"] = TEST_DIR
 import unittest
+import time
 from utils import identifyhome, createdirs, bettersleep
 from onionrsetup import setup_config, setup_default_plugins
 
@@ -22,13 +23,32 @@ from utils import identifyhome
 import safedb
 import blockio
 
+
 def _remove_db(path):
     try:
         os.remove(path)
     except FileNotFoundError:
         pass
 
+
 class TestBlockIO(unittest.TestCase):
+
+    def test_clean_expired(self):
+
+        db_file = identifyhome.identify_home() + 'test.db'
+        db = safedb.SafeDB(db_file)
+        for i in range(3):
+            bl = blockcreator.create_anonvdf_block(b"hello" + int(i).to_bytes(1, "big"), b"txt", 5)
+            blockio.store_block(bl, db)
+        print("done gening")
+        blockio.clean_expired_blocks(db)
+        time.sleep(1)
+        self.assertEqual(len(list(blockio.list_blocks_by_type("txt", db))), 3)
+        time.sleep(4.1)
+        blockio.list_blocks_by_type("txt", db)
+        db.close()
+        _remove_db(db_file)
+
     def test_store_block(self):
         packed = kasten.generator.pack.pack(b"test", "tst")
         bl: kasten.Kasten = anonvdf.AnonVDFGenerator.generate(packed, rounds=1000)
@@ -39,9 +59,10 @@ class TestBlockIO(unittest.TestCase):
         _remove_db(db_file)
 
     def test_store_dupe(self):
+        db_file = identifyhome.identify_home() + 'test.db'
+        _remove_db(db_file)
         packed = kasten.generator.pack.pack(b"test", "tst")
         bl: kasten.Kasten = anonvdf.AnonVDFGenerator.generate(packed, rounds=1000)
-        db_file = identifyhome.identify_home() + 'test.db'
         db = safedb.SafeDB(db_file)
         blockio.store_block(bl, db)
         self.assertRaises(ValueError, blockio.store_block, bl, db)
@@ -50,16 +71,20 @@ class TestBlockIO(unittest.TestCase):
 
     def test_list_blocks(self):
         db_file = identifyhome.identify_home() + 'test.db'
+        _remove_db(db_file)
         db = safedb.SafeDB(db_file)
         expected_list = []
         for i in range(10):
             bl = blockcreator.create_anonvdf_block(b'test' + int(i).to_bytes(1, 'big'), 'txt', 60)
             blockio.store_block(bl, db)
             expected_list.append(bl.id)
+        #db.db_conn.sync()
         l = blockio.list_blocks_by_type('txt', db)
         self.assertEqual(len(list(l)), len(expected_list))
         for i in l:
             self.assertIn(bytes(i), expected_list)
+        db.close()
+        _remove_db(db_file)
 
 
 unittest.main()