added block cleaning method for new database

This commit is contained in:
Kevin Froman 2021-01-25 06:28:18 +00:00
parent 75234310ca
commit 708c5d2e71
8 changed files with 74 additions and 31 deletions

View File

@ -15,4 +15,4 @@ filenuke==0.0.0
watchdog==1.0.2
ujson==4.0.1
cffi==1.14.4
onionrblocks==4.0.0
onionrblocks==4.1.0

View File

@ -202,9 +202,9 @@ niceware==0.2.1 \
--hash=sha256:0f8b192f2a1e800e068474f6e208be9c7e2857664b33a96f4045340de4e5c69c \
--hash=sha256:cf2dc0e1567d36d067c61b32fed0f1b9c4534ed511f9eeead4ba548d03b5c9eb
# via -r requirements.in
onionrblocks==4.0.0 \
--hash=sha256:7a616228b2100f7d54021b825a921035299c8862150bb94379f5108ba7e2af78 \
--hash=sha256:a5d87779930800d6364cabba560d24f0e2d491cffb180a16271a37a551a8eac0
onionrblocks==4.1.0 \
--hash=sha256:2f806d1a4cf332ffef8630ac3d362499854316d957945be0a090c7ff6917a6c4 \
--hash=sha256:bfdfa90df6fcdaef44b9ff1bc8f7b645fc41d10cf46481a3158d6c77e3832507
# via -r requirements.in
psutil==5.8.0 \
--hash=sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64 \

View File

@ -4,6 +4,7 @@ Wrap safedb for storing and fetching blocks
"""
from .store import store_block
from .load import load_block, list_blocks_by_type
from .cleanexpired import clean_expired_blocks
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by

View File

@ -0,0 +1,43 @@
"""Onionr - Private P2P Communication.
clean expired blocks
"""
from typing import TYPE_CHECKING
from kasten import Kasten
from onionrblocks.generators.anonvdf import AnonVDFGenerator
from onionrblocks.exceptions import BlockExpired
if TYPE_CHECKING:
from kasten.types import BlockChecksumBytes
from safedb import SafeDB
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def clean_expired_blocks(db: 'SafeDB'):
key = db.db_conn.firstkey()
delete_list = set()
while key:
try:
if key.startswith(b'bl-') or key.startswith(b'enc'):
key = db.db_conn.nextkey(key)
continue
Kasten(key, db.get(key), AnonVDFGenerator)
except BlockExpired:
delete_list.add(key)
key = db.db_conn.nextkey(key)
for key in delete_list:
del db[key]

View File

@ -1,23 +0,0 @@
"""Onionr - Private P2P Communication.
Get the kasten generator from a block metadata section
"""
from onionrblocks.generators import anonvdf
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
GENERATOR_STRING_MAPPING = {
0: anonvdf.AnonVDFGenerator
}

View File

@ -41,4 +41,3 @@ def store_block(block: 'Kasten', safe_db: 'SafeDB'):
block_list_for_type += block.id
safe_db.put(f'bl-{block_type}', block_list_for_type)

View File

@ -68,5 +68,3 @@ class SafeDB:
pass
self.protected = protected

View File

@ -7,6 +7,7 @@ TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
import unittest
import time
from utils import identifyhome, createdirs, bettersleep
from onionrsetup import setup_config, setup_default_plugins
@ -22,13 +23,32 @@ from utils import identifyhome
import safedb
import blockio
def _remove_db(path):
try:
os.remove(path)
except FileNotFoundError:
pass
class TestBlockIO(unittest.TestCase):
def test_clean_expired(self):
db_file = identifyhome.identify_home() + 'test.db'
db = safedb.SafeDB(db_file)
for i in range(3):
bl = blockcreator.create_anonvdf_block(b"hello" + int(i).to_bytes(1, "big"), b"txt", 5)
blockio.store_block(bl, db)
print("done gening")
blockio.clean_expired_blocks(db)
time.sleep(1)
self.assertEqual(len(list(blockio.list_blocks_by_type("txt", db))), 3)
time.sleep(4.1)
blockio.list_blocks_by_type("txt", db)
db.close()
_remove_db(db_file)
def test_store_block(self):
packed = kasten.generator.pack.pack(b"test", "tst")
bl: kasten.Kasten = anonvdf.AnonVDFGenerator.generate(packed, rounds=1000)
@ -39,9 +59,10 @@ class TestBlockIO(unittest.TestCase):
_remove_db(db_file)
def test_store_dupe(self):
db_file = identifyhome.identify_home() + 'test.db'
_remove_db(db_file)
packed = kasten.generator.pack.pack(b"test", "tst")
bl: kasten.Kasten = anonvdf.AnonVDFGenerator.generate(packed, rounds=1000)
db_file = identifyhome.identify_home() + 'test.db'
db = safedb.SafeDB(db_file)
blockio.store_block(bl, db)
self.assertRaises(ValueError, blockio.store_block, bl, db)
@ -50,16 +71,20 @@ class TestBlockIO(unittest.TestCase):
def test_list_blocks(self):
db_file = identifyhome.identify_home() + 'test.db'
_remove_db(db_file)
db = safedb.SafeDB(db_file)
expected_list = []
for i in range(10):
bl = blockcreator.create_anonvdf_block(b'test' + int(i).to_bytes(1, 'big'), 'txt', 60)
blockio.store_block(bl, db)
expected_list.append(bl.id)
#db.db_conn.sync()
l = blockio.list_blocks_by_type('txt', db)
self.assertEqual(len(list(l)), len(expected_list))
for i in l:
self.assertIn(bytes(i), expected_list)
db.close()
_remove_db(db_file)
unittest.main()