From 708c5d2e711fff251874cfd32ac22e895e74b8c7 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 25 Jan 2021 06:28:18 +0000 Subject: [PATCH] added block cleaning method for new database --- requirements.in | 2 +- requirements.txt | 6 ++--- src/blockio/__init__.py | 1 + src/blockio/cleanexpired.py | 43 +++++++++++++++++++++++++++++++++++ src/blockio/getgenerator.py | 23 ------------------- src/blockio/store/__init__.py | 1 - src/safedb/__init__.py | 2 -- tests/test_blockio.py | 27 +++++++++++++++++++++- 8 files changed, 74 insertions(+), 31 deletions(-) create mode 100644 src/blockio/cleanexpired.py delete mode 100644 src/blockio/getgenerator.py diff --git a/requirements.in b/requirements.in index 229bd7b0..a9473789 100644 --- a/requirements.in +++ b/requirements.in @@ -15,4 +15,4 @@ filenuke==0.0.0 watchdog==1.0.2 ujson==4.0.1 cffi==1.14.4 -onionrblocks==4.0.0 +onionrblocks==4.1.0 diff --git a/requirements.txt b/requirements.txt index b925689e..dc64238f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -202,9 +202,9 @@ niceware==0.2.1 \ --hash=sha256:0f8b192f2a1e800e068474f6e208be9c7e2857664b33a96f4045340de4e5c69c \ --hash=sha256:cf2dc0e1567d36d067c61b32fed0f1b9c4534ed511f9eeead4ba548d03b5c9eb # via -r requirements.in -onionrblocks==4.0.0 \ - --hash=sha256:7a616228b2100f7d54021b825a921035299c8862150bb94379f5108ba7e2af78 \ - --hash=sha256:a5d87779930800d6364cabba560d24f0e2d491cffb180a16271a37a551a8eac0 +onionrblocks==4.1.0 \ + --hash=sha256:2f806d1a4cf332ffef8630ac3d362499854316d957945be0a090c7ff6917a6c4 \ + --hash=sha256:bfdfa90df6fcdaef44b9ff1bc8f7b645fc41d10cf46481a3158d6c77e3832507 # via -r requirements.in psutil==5.8.0 \ --hash=sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64 \ diff --git a/src/blockio/__init__.py b/src/blockio/__init__.py index 85dd0e64..fa550df7 100644 --- a/src/blockio/__init__.py +++ b/src/blockio/__init__.py @@ -4,6 +4,7 @@ Wrap safedb for storing and fetching blocks """ from .store import store_block from .load import load_block, list_blocks_by_type +from .cleanexpired import clean_expired_blocks """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/blockio/cleanexpired.py b/src/blockio/cleanexpired.py new file mode 100644 index 00000000..badb4a92 --- /dev/null +++ b/src/blockio/cleanexpired.py @@ -0,0 +1,43 @@ +"""Onionr - Private P2P Communication. + +clean expired blocks +""" +from typing import TYPE_CHECKING + +from kasten import Kasten +from onionrblocks.generators.anonvdf import AnonVDFGenerator +from onionrblocks.exceptions import BlockExpired + +if TYPE_CHECKING: + from kasten.types import BlockChecksumBytes + from safedb import SafeDB +""" +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + + +def clean_expired_blocks(db: 'SafeDB'): + key = db.db_conn.firstkey() + delete_list = set() + while key: + try: + if key.startswith(b'bl-') or key.startswith(b'enc'): + key = db.db_conn.nextkey(key) + continue + Kasten(key, db.get(key), AnonVDFGenerator) + except BlockExpired: + delete_list.add(key) + key = db.db_conn.nextkey(key) + for key in delete_list: + del db[key] diff --git a/src/blockio/getgenerator.py b/src/blockio/getgenerator.py deleted file mode 100644 index 92bfbb19..00000000 --- a/src/blockio/getgenerator.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Onionr - Private P2P Communication. - -Get the kasten generator from a block metadata section -""" -from onionrblocks.generators import anonvdf -""" -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -GENERATOR_STRING_MAPPING = { - 0: anonvdf.AnonVDFGenerator -} diff --git a/src/blockio/store/__init__.py b/src/blockio/store/__init__.py index 000c150d..42cd93cb 100644 --- a/src/blockio/store/__init__.py +++ b/src/blockio/store/__init__.py @@ -41,4 +41,3 @@ def store_block(block: 'Kasten', safe_db: 'SafeDB'): block_list_for_type += block.id safe_db.put(f'bl-{block_type}', block_list_for_type) - diff --git a/src/safedb/__init__.py b/src/safedb/__init__.py index 504e40b4..d1a1e1fa 100644 --- a/src/safedb/__init__.py +++ b/src/safedb/__init__.py @@ -68,5 +68,3 @@ class SafeDB: pass self.protected = protected - - diff --git a/tests/test_blockio.py b/tests/test_blockio.py index b67acf65..8ab4b17b 100644 --- a/tests/test_blockio.py +++ b/tests/test_blockio.py @@ -7,6 +7,7 @@ TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' print("Test directory:", TEST_DIR) os.environ["ONIONR_HOME"] = TEST_DIR import unittest +import time from utils import identifyhome, createdirs, bettersleep from onionrsetup import setup_config, setup_default_plugins @@ -22,13 +23,32 @@ from utils import identifyhome import safedb import blockio + def _remove_db(path): try: os.remove(path) except FileNotFoundError: pass + class TestBlockIO(unittest.TestCase): + + def test_clean_expired(self): + + db_file = identifyhome.identify_home() + 'test.db' + db = safedb.SafeDB(db_file) + for i in range(3): + bl = blockcreator.create_anonvdf_block(b"hello" + int(i).to_bytes(1, "big"), b"txt", 5) + blockio.store_block(bl, db) + print("done gening") + blockio.clean_expired_blocks(db) + time.sleep(1) + self.assertEqual(len(list(blockio.list_blocks_by_type("txt", db))), 3) + time.sleep(4.1) + blockio.list_blocks_by_type("txt", db) + db.close() + _remove_db(db_file) + def test_store_block(self): packed = kasten.generator.pack.pack(b"test", "tst") bl: kasten.Kasten = anonvdf.AnonVDFGenerator.generate(packed, rounds=1000) @@ -39,9 +59,10 @@ class TestBlockIO(unittest.TestCase): _remove_db(db_file) def test_store_dupe(self): + db_file = identifyhome.identify_home() + 'test.db' + _remove_db(db_file) packed = kasten.generator.pack.pack(b"test", "tst") bl: kasten.Kasten = anonvdf.AnonVDFGenerator.generate(packed, rounds=1000) - db_file = identifyhome.identify_home() + 'test.db' db = safedb.SafeDB(db_file) blockio.store_block(bl, db) self.assertRaises(ValueError, blockio.store_block, bl, db) @@ -50,16 +71,20 @@ class TestBlockIO(unittest.TestCase): def test_list_blocks(self): db_file = identifyhome.identify_home() + 'test.db' + _remove_db(db_file) db = safedb.SafeDB(db_file) expected_list = [] for i in range(10): bl = blockcreator.create_anonvdf_block(b'test' + int(i).to_bytes(1, 'big'), 'txt', 60) blockio.store_block(bl, db) expected_list.append(bl.id) + #db.db_conn.sync() l = blockio.list_blocks_by_type('txt', db) self.assertEqual(len(list(l)), len(expected_list)) for i in l: self.assertIn(bytes(i), expected_list) + db.close() + _remove_db(db_file) unittest.main()