From 085b90f84c28c9e041bd53bc689696a536dad750 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 8 Sep 2019 04:48:16 -0500 Subject: [PATCH] added storagecounter test and renamed setup --- onionr/__init__.py | 2 +- .../downloadblocks/__init__.py | 2 +- onionr/communicatorutils/housekeeping.py | 2 +- onionr/communicatorutils/lookupblocks.py | 2 +- onionr/logger/__init__.py | 10 +-- onionr/logger/settings.py | 2 +- onionr/onionrblocks/insert.py | 15 ++-- onionr/onionrexceptions.py | 16 ++--- onionr/onionrproofs.py | 2 +- onionr/{setup => onionrsetup}/__init__.py | 0 .../defaultpluginsetup.py | 0 onionr/{setup => onionrsetup}/setupconfig.py | 10 +-- onionr/onionrstorage/removeblock.py | 2 +- onionr/onionrstorage/setdata.py | 4 +- onionr/storagecounter.py | 70 ++++++++++--------- onionr/tests/test_keymanager.py | 2 +- onionr/tests/test_networkmerger.py | 2 +- onionr/tests/test_storagecounter.py | 45 ++++++++++++ requirements.in | 4 +- requirements.txt | 4 +- 20 files changed, 120 insertions(+), 76 deletions(-) rename onionr/{setup => onionrsetup}/__init__.py (100%) rename onionr/{setup => onionrsetup}/defaultpluginsetup.py (100%) rename onionr/{setup => onionrsetup}/setupconfig.py (91%) create mode 100644 onionr/tests/test_storagecounter.py diff --git a/onionr/__init__.py b/onionr/__init__.py index 61e5c579..c7439429 100755 --- a/onionr/__init__.py +++ b/onionr/__init__.py @@ -36,7 +36,7 @@ except ModuleNotFoundError: # Onionr imports from etc import onionrvalues # For different Onionr related constants such as versions -import setup +import onionrsetup as setup # Ensure we have at least the minimum python version if sys.version_info[0] == 2 or sys.version_info[1] < onionrvalues.MIN_PY_VERSION: diff --git a/onionr/communicatorutils/downloadblocks/__init__.py b/onionr/communicatorutils/downloadblocks/__init__.py index e47a3cc0..ab289bc7 100755 --- a/onionr/communicatorutils/downloadblocks/__init__.py +++ b/onionr/communicatorutils/downloadblocks/__init__.py @@ -46,7 +46,7 @@ def download_blocks_from_communicator(comm_inst): if not shoulddownload.should_download(comm_inst, blockHash): continue - if comm_inst.shutdown or not comm_inst.isOnline or storage_counter.isFull(): + if comm_inst.shutdown or not comm_inst.isOnline or storage_counter.is_full(): # Exit loop if shutting down or offline, or disk allocation reached break # Do not download blocks being downloaded diff --git a/onionr/communicatorutils/housekeeping.py b/onionr/communicatorutils/housekeeping.py index e4475bb5..135ad506 100755 --- a/onionr/communicatorutils/housekeeping.py +++ b/onionr/communicatorutils/housekeeping.py @@ -43,7 +43,7 @@ def clean_old_blocks(comm_inst): __remove_from_upload(comm_inst, bHash) logger.info('Deleted block: %s' % (bHash,)) - while comm_inst.storage_counter.isFull(): + while comm_inst.storage_counter.is_full(): oldest = blockmetadb.get_block_list()[0] blacklist.addToDB(oldest) removeblock.remove_block(oldest) diff --git a/onionr/communicatorutils/lookupblocks.py b/onionr/communicatorutils/lookupblocks.py index f3765b7a..3e1ade6a 100755 --- a/onionr/communicatorutils/lookupblocks.py +++ b/onionr/communicatorutils/lookupblocks.py @@ -40,7 +40,7 @@ def lookup_blocks_from_communicator(comm_inst): if not comm_inst.isOnline: break # check if disk allocation is used - if comm_inst.storage_counter.isFull(): + if comm_inst.storage_counter.is_full(): logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used') break peer = onlinepeers.pick_online_peer(comm_inst) # select random online peer diff --git a/onionr/logger/__init__.py b/onionr/logger/__init__.py index 9ded60d5..65426f41 100755 --- a/onionr/logger/__init__.py +++ b/onionr/logger/__init__.py @@ -28,33 +28,33 @@ raw = raw.raw confirm = confirm.confirm # debug: when there is info that could be useful for debugging purposes only -def debug(data, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_DEBUG): +def debug(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_DEBUG): if settings.get_level() <= level: log('/', data, timestamp = timestamp, prompt = prompt, terminal = terminal) if not error is None: debug('Error: ' + str(error) + parse_error()) # info: when there is something to notify the user of, such as the success of a process -def info(data, timestamp = False, prompt = True, terminal = False, level = settings.LEVEL_INFO): +def info(data: str, timestamp = False, prompt = True, terminal = False, level = settings.LEVEL_INFO): if settings.get_level() <= level: log('+', data, colors.fg.green, timestamp = timestamp, prompt = prompt, terminal = terminal) # warn: when there is a potential for something bad to happen -def warn(data, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_WARN): +def warn(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_WARN): if not error is None: debug('Error: ' + str(error) + parse_error()) if settings.get_level() <= level: log('!', data, colors.fg.orange, timestamp = timestamp, prompt = prompt, terminal = terminal) # error: when only one function, module, or process of the program encountered a problem and must stop -def error(data, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_ERROR): +def error(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_ERROR): if settings.get_level() <= level: log('-', data, colors.fg.red, timestamp = timestamp, fd = sys.stderr, prompt = prompt, terminal = terminal) if not error is None: debug('Error: ' + str(error) + parse_error()) # fatal: when the something so bad has happened that the program must stop -def fatal(data, error = None, timestamp=True, prompt = True, terminal = False, level = settings.LEVEL_FATAL): +def fatal(data: str, error = None, timestamp=True, prompt = True, terminal = False, level = settings.LEVEL_FATAL): if not error is None: debug('Error: ' + str(error) + parse_error(), terminal = terminal) if settings.get_level() <= level: diff --git a/onionr/logger/settings.py b/onionr/logger/settings.py index cc1a2967..0f02e19b 100644 --- a/onionr/logger/settings.py +++ b/onionr/logger/settings.py @@ -64,7 +64,7 @@ def set_level(level): global _level _level = level -def get_level(): +def get_level()->int: ''' Get the lowest log level currently being outputted ''' diff --git a/onionr/onionrblocks/insert.py b/onionr/onionrblocks/insert.py index c0c23599..e2404735 100644 --- a/onionr/onionrblocks/insert.py +++ b/onionr/onionrblocks/insert.py @@ -1,3 +1,4 @@ +from typing import Union import json from onionrutils import bytesconverter, epoch import storagecounter, filepaths, onionrstorage @@ -8,17 +9,21 @@ from onionrusers import onionrusers from onionrutils import localcommand, blockmetadata, stringvalidators import coredb import onionrproofs -def insert_block(data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None, disableForward=False): - ''' +import logger +def insert_block(data: Union[str, bytes], header: str ='txt', + sign: bool =False, encryptType:str ='', symKey:str ='', + asymPeer:str ='', meta:dict = {}, + expire:Union[int, None] =None, disableForward:bool =False)->Union[str,bool]: + """ Inserts a block into the network encryptType must be specified to encrypt a block - ''' + """ use_subprocess = powchoice.use_subprocess(config) storage_counter = storagecounter.StorageCounter() allocationReachedMessage = 'Cannot insert block, disk allocation reached.' - if storage_counter.isFull(): + if storage_counter.is_full(): logger.error(allocationReachedMessage) - return False + raise onionrexceptions.DiskAllocationReached retData = False if type(data) is None: diff --git a/onionr/onionrexceptions.py b/onionr/onionrexceptions.py index b510dee8..336e7c9f 100755 --- a/onionr/onionrexceptions.py +++ b/onionr/onionrexceptions.py @@ -1,9 +1,9 @@ -''' - Onionr - P2P Anonymous Storage Network +""" + Onionr - Private P2P Communication This file contains exceptions for onionr -''' -''' +""" +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -16,7 +16,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' +""" # general exceptions class NotFound(Exception): @@ -67,11 +67,11 @@ class NoDataAvailable(Exception): pass class InvalidHexHash(Exception): - '''When a string is not a valid hex string of appropriate length for a hash value''' + """When a string is not a valid hex string of appropriate length for a hash value""" pass class InvalidProof(Exception): - '''When a proof is invalid or inadequate''' + """When a proof is invalid or inadequate""" pass # network level exceptions @@ -100,4 +100,4 @@ class MissingAddress(Exception): # Contact exceptions class ContactDeleted(Exception): - pass \ No newline at end of file + pass diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py index fb6cafda..f476981c 100755 --- a/onionr/onionrproofs.py +++ b/onionr/onionrproofs.py @@ -27,7 +27,7 @@ def getDifficultyModifier(): on a variety of factors, currently only disk use. ''' retData = 0 - useFunc = storagecounter.StorageCounter().getPercent + useFunc = storagecounter.StorageCounter().get_percent percentUse = useFunc() diff --git a/onionr/setup/__init__.py b/onionr/onionrsetup/__init__.py similarity index 100% rename from onionr/setup/__init__.py rename to onionr/onionrsetup/__init__.py diff --git a/onionr/setup/defaultpluginsetup.py b/onionr/onionrsetup/defaultpluginsetup.py similarity index 100% rename from onionr/setup/defaultpluginsetup.py rename to onionr/onionrsetup/defaultpluginsetup.py diff --git a/onionr/setup/setupconfig.py b/onionr/onionrsetup/setupconfig.py similarity index 91% rename from onionr/setup/setupconfig.py rename to onionr/onionrsetup/setupconfig.py index bcd93df8..21c408b7 100755 --- a/onionr/setup/setupconfig.py +++ b/onionr/onionrsetup/setupconfig.py @@ -22,7 +22,7 @@ import config, logger, netcontroller from etc import onionrvalues from logger.settings import * -def setup_config(o_inst = None): +def setup_config(): config.reload() if not os.path.exists(config._configfile): @@ -42,14 +42,6 @@ def setup_config(o_inst = None): settings = settings | OUTPUT_TO_FILE set_settings(settings) - if not o_inst is None: - if str(config.get('general.dev_mode', True)).lower() == 'true': - o_inst._developmentMode = True - set_level(LEVEL_DEBUG) - else: - o_inst._developmentMode = False - set_level(LEVEL_INFO) - verbosity = str(config.get('log.verbosity', 'default')).lower().strip() if not verbosity in ['default', 'null', 'none', 'nil']: map = { diff --git a/onionr/onionrstorage/removeblock.py b/onionr/onionrstorage/removeblock.py index e5a439c6..46b6e45a 100644 --- a/onionr/onionrstorage/removeblock.py +++ b/onionr/onionrstorage/removeblock.py @@ -18,6 +18,6 @@ def remove_block(block): conn.commit() conn.close() dataSize = sys.getsizeof(onionrstorage.getData(block)) - storagecounter.StorageCounter().removeBytes(dataSize) + storagecounter.StorageCounter().remove_bytes(dataSize) else: raise onionrexceptions.InvalidHexHash \ No newline at end of file diff --git a/onionr/onionrstorage/setdata.py b/onionr/onionrstorage/setdata.py index d6cdc9e9..2ca93bb7 100644 --- a/onionr/onionrstorage/setdata.py +++ b/onionr/onionrstorage/setdata.py @@ -3,7 +3,7 @@ import onionrstorage, onionrexceptions, onionrcrypto as crypto import filepaths, storagecounter from coredb import dbfiles from onionrutils import blockmetadata, bytesconverter -def set_data(data): +def set_data(data)->str: ''' Set the data assciated with a hash ''' @@ -24,7 +24,7 @@ def set_data(data): try: onionrstorage.getData(dataHash) except onionrexceptions.NoDataAvailable: - if storage_counter.addBytes(dataSize) != False: + if storage_counter.add_bytes(dataSize) != False: onionrstorage.store(data, blockHash=dataHash) conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30) c = conn.cursor() diff --git a/onionr/storagecounter.py b/onionr/storagecounter.py index 4e77750a..42cf7c64 100755 --- a/onionr/storagecounter.py +++ b/onionr/storagecounter.py @@ -1,9 +1,9 @@ -''' +""" Onionr - Private P2P Communication Keeps track of how much disk space we're using -''' -''' +""" +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -16,52 +16,54 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' +""" import config, filepaths config.reload() class StorageCounter: def __init__(self): - self.dataFile = filepaths.usage_file + self.data_file = filepaths.usage_file return - def isFull(self): - retData = False - if config.get('allocations.disk', 2000000000) <= (self.getAmount() + 1000): - retData = True - return retData + def is_full(self)->bool: + """Returns if the allocated disk space is full (this is Onionr config, not true FS capacity)""" + ret_data = False + if config.get('allocations.disk', 2000000000) <= (self.get_amount() + 1000): + ret_data = True + return ret_data def _update(self, data): - with open(self.dataFile, 'w') as dataFile: - dataFile.write(str(data)) - def getAmount(self): - '''Return how much disk space we're using (according to record)''' - retData = 0 + with open(self.data_file, 'w') as data_file: + data_file.write(str(data)) + + def get_amount(self)->int: + """Return how much disk space we're using (according to record)""" + ret_data = 0 try: - with open(self.dataFile, 'r') as dataFile: - retData = int(dataFile.read()) + with open(self.data_file, 'r') as data_file: + ret_data = int(data_file.read()) except FileNotFoundError: pass except ValueError: pass # Possibly happens when the file is empty - return retData + return ret_data - def getPercent(self): - '''Return percent (decimal/float) of disk space we're using''' - amount = self.getAmount() + def get_percent(self)->int: + """Return percent (decimal/float) of disk space we're using""" + amount = self.get_amount() return round(amount / config.get('allocations.disk', 2000000000), 2) - def addBytes(self, amount): - '''Record that we are now using more disk space, unless doing so would exceed configured max''' - newAmount = amount + self.getAmount() - retData = newAmount - if newAmount > config.get('allocations.disk', 2000000000): - retData = False + def add_bytes(self, amount)->int: + """Record that we are now using more disk space, unless doing so would exceed configured max""" + new_amount = amount + self.get_amount() + ret_data = new_amount + if new_amount > config.get('allocations.disk', 2000000000): + ret_data = False else: - self._update(newAmount) - return retData + self._update(new_amount) + return ret_data - def removeBytes(self, amount): - '''Record that we are now using less disk space''' - newAmount = self.getAmount() - amount - self._update(newAmount) - return newAmount \ No newline at end of file + def remove_bytes(self, amount)->int: + """Record that we are now using less disk space""" + new_amount = self.get_amount() - amount + self._update(new_amount) + return new_amount diff --git a/onionr/tests/test_keymanager.py b/onionr/tests/test_keymanager.py index 4d477ba7..271e7231 100644 --- a/onionr/tests/test_keymanager.py +++ b/onionr/tests/test_keymanager.py @@ -8,7 +8,7 @@ print("Test directory:", TEST_DIR) os.environ["ONIONR_HOME"] = TEST_DIR from utils import createdirs from coredb import keydb -import setup, keymanager, filepaths +import onionrsetup as setup, keymanager, filepaths from onionrutils import stringvalidators createdirs.create_dirs() setup.setup_config() diff --git a/onionr/tests/test_networkmerger.py b/onionr/tests/test_networkmerger.py index 386a6a5b..79d01a36 100644 --- a/onionr/tests/test_networkmerger.py +++ b/onionr/tests/test_networkmerger.py @@ -8,7 +8,7 @@ print("Test directory:", TEST_DIR) os.environ["ONIONR_HOME"] = TEST_DIR from utils import networkmerger, createdirs from coredb import keydb -import setup +import onionrsetup as setup from utils import createdirs createdirs.create_dirs() setup.setup_config() diff --git a/onionr/tests/test_storagecounter.py b/onionr/tests/test_storagecounter.py new file mode 100644 index 00000000..ec3d9d96 --- /dev/null +++ b/onionr/tests/test_storagecounter.py @@ -0,0 +1,45 @@ +import sys, os +sys.path.append(".") +import unittest, uuid + +import logger +import config +from utils import createdirs +import onionrsetup as setup +from utils import createdirs +import onionrblocks +import filepaths +import onionrexceptions +import storagecounter +import onionrstorage + +def _test_setup(): + TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' + print("Test directory:", TEST_DIR) + os.environ["ONIONR_HOME"] = TEST_DIR + createdirs.create_dirs() + setup.setup_config() + config.reload() + +class TestStorageCounter(unittest.TestCase): + def test_basic_amount(self): + _test_setup() + self.assertIsNotNone(config.get('allocations.disk')) + self.assertGreater(config.get('allocations.disk'), 1000000) + + def test_insert_too_much(self): + _test_setup() + config.set('allocations.disk', 1000) + self.assertRaises(onionrexceptions.DiskAllocationReached, onionrblocks.insert, "test") + + def test_count(self): + _test_setup() + counter = storagecounter.StorageCounter() + start_value = counter.get_amount() + b_hash = onionrblocks.insert("test") + self.assertGreater(counter.get_amount(), start_value) + onionrstorage.removeblock.remove_block(b_hash) + self.assertEqual(counter.get_amount(), start_value) + + +unittest.main() \ No newline at end of file diff --git a/requirements.in b/requirements.in index 7ce9c985..7bbb9f7a 100644 --- a/requirements.in +++ b/requirements.in @@ -5,9 +5,9 @@ gevent==1.3.6 Flask==1.1.1 PySocks==1.6.8 stem==1.7.1 -deadsimplekv==0.1.1 +deadsimplekv==0.2.0 unpaddedbase32==0.1.0 streamedrequests==1.0.0 jinja2==2.10.1 toomanyobjs==1.1.0 -mnemonic==0.18 \ No newline at end of file +mnemonic==0.18 diff --git a/requirements.txt b/requirements.txt index 22b4e281..ca9ed0a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,8 +46,8 @@ click==7.0 \ --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \ --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \ # via flask -deadsimplekv==0.1.1 \ - --hash=sha256:4bf951e188c302006e37f95bde6117b1b938fb454153d583c6346090d9bead1a +deadsimplekv==0.2.0 \ + --hash=sha256:81405408a4d23cc94ac359f9570e0ff198b67e5a93e3ae32eca85e3b62252f38 flask==1.1.1 \ --hash=sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52 \ --hash=sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6