added storagecounter test and renamed setup

This commit is contained in:
Kevin Froman 2019-09-08 04:48:16 -05:00
parent 311dda91d6
commit 085b90f84c
20 changed files with 120 additions and 76 deletions

View File

@ -36,7 +36,7 @@ except ModuleNotFoundError:
# Onionr imports # Onionr imports
from etc import onionrvalues # For different Onionr related constants such as versions from etc import onionrvalues # For different Onionr related constants such as versions
import setup import onionrsetup as setup
# Ensure we have at least the minimum python version # Ensure we have at least the minimum python version
if sys.version_info[0] == 2 or sys.version_info[1] < onionrvalues.MIN_PY_VERSION: if sys.version_info[0] == 2 or sys.version_info[1] < onionrvalues.MIN_PY_VERSION:

View File

@ -46,7 +46,7 @@ def download_blocks_from_communicator(comm_inst):
if not shoulddownload.should_download(comm_inst, blockHash): if not shoulddownload.should_download(comm_inst, blockHash):
continue continue
if comm_inst.shutdown or not comm_inst.isOnline or storage_counter.isFull(): if comm_inst.shutdown or not comm_inst.isOnline or storage_counter.is_full():
# Exit loop if shutting down or offline, or disk allocation reached # Exit loop if shutting down or offline, or disk allocation reached
break break
# Do not download blocks being downloaded # Do not download blocks being downloaded

View File

@ -43,7 +43,7 @@ def clean_old_blocks(comm_inst):
__remove_from_upload(comm_inst, bHash) __remove_from_upload(comm_inst, bHash)
logger.info('Deleted block: %s' % (bHash,)) logger.info('Deleted block: %s' % (bHash,))
while comm_inst.storage_counter.isFull(): while comm_inst.storage_counter.is_full():
oldest = blockmetadb.get_block_list()[0] oldest = blockmetadb.get_block_list()[0]
blacklist.addToDB(oldest) blacklist.addToDB(oldest)
removeblock.remove_block(oldest) removeblock.remove_block(oldest)

View File

@ -40,7 +40,7 @@ def lookup_blocks_from_communicator(comm_inst):
if not comm_inst.isOnline: if not comm_inst.isOnline:
break break
# check if disk allocation is used # check if disk allocation is used
if comm_inst.storage_counter.isFull(): if comm_inst.storage_counter.is_full():
logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used') logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
break break
peer = onlinepeers.pick_online_peer(comm_inst) # select random online peer peer = onlinepeers.pick_online_peer(comm_inst) # select random online peer

View File

@ -28,33 +28,33 @@ raw = raw.raw
confirm = confirm.confirm confirm = confirm.confirm
# debug: when there is info that could be useful for debugging purposes only # debug: when there is info that could be useful for debugging purposes only
def debug(data, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_DEBUG): def debug(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_DEBUG):
if settings.get_level() <= level: if settings.get_level() <= level:
log('/', data, timestamp = timestamp, prompt = prompt, terminal = terminal) log('/', data, timestamp = timestamp, prompt = prompt, terminal = terminal)
if not error is None: if not error is None:
debug('Error: ' + str(error) + parse_error()) debug('Error: ' + str(error) + parse_error())
# info: when there is something to notify the user of, such as the success of a process # info: when there is something to notify the user of, such as the success of a process
def info(data, timestamp = False, prompt = True, terminal = False, level = settings.LEVEL_INFO): def info(data: str, timestamp = False, prompt = True, terminal = False, level = settings.LEVEL_INFO):
if settings.get_level() <= level: if settings.get_level() <= level:
log('+', data, colors.fg.green, timestamp = timestamp, prompt = prompt, terminal = terminal) log('+', data, colors.fg.green, timestamp = timestamp, prompt = prompt, terminal = terminal)
# warn: when there is a potential for something bad to happen # warn: when there is a potential for something bad to happen
def warn(data, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_WARN): def warn(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_WARN):
if not error is None: if not error is None:
debug('Error: ' + str(error) + parse_error()) debug('Error: ' + str(error) + parse_error())
if settings.get_level() <= level: if settings.get_level() <= level:
log('!', data, colors.fg.orange, timestamp = timestamp, prompt = prompt, terminal = terminal) log('!', data, colors.fg.orange, timestamp = timestamp, prompt = prompt, terminal = terminal)
# error: when only one function, module, or process of the program encountered a problem and must stop # error: when only one function, module, or process of the program encountered a problem and must stop
def error(data, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_ERROR): def error(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_ERROR):
if settings.get_level() <= level: if settings.get_level() <= level:
log('-', data, colors.fg.red, timestamp = timestamp, fd = sys.stderr, prompt = prompt, terminal = terminal) log('-', data, colors.fg.red, timestamp = timestamp, fd = sys.stderr, prompt = prompt, terminal = terminal)
if not error is None: if not error is None:
debug('Error: ' + str(error) + parse_error()) debug('Error: ' + str(error) + parse_error())
# fatal: when the something so bad has happened that the program must stop # fatal: when the something so bad has happened that the program must stop
def fatal(data, error = None, timestamp=True, prompt = True, terminal = False, level = settings.LEVEL_FATAL): def fatal(data: str, error = None, timestamp=True, prompt = True, terminal = False, level = settings.LEVEL_FATAL):
if not error is None: if not error is None:
debug('Error: ' + str(error) + parse_error(), terminal = terminal) debug('Error: ' + str(error) + parse_error(), terminal = terminal)
if settings.get_level() <= level: if settings.get_level() <= level:

View File

@ -64,7 +64,7 @@ def set_level(level):
global _level global _level
_level = level _level = level
def get_level(): def get_level()->int:
''' '''
Get the lowest log level currently being outputted Get the lowest log level currently being outputted
''' '''

View File

@ -1,3 +1,4 @@
from typing import Union
import json import json
from onionrutils import bytesconverter, epoch from onionrutils import bytesconverter, epoch
import storagecounter, filepaths, onionrstorage import storagecounter, filepaths, onionrstorage
@ -8,17 +9,21 @@ from onionrusers import onionrusers
from onionrutils import localcommand, blockmetadata, stringvalidators from onionrutils import localcommand, blockmetadata, stringvalidators
import coredb import coredb
import onionrproofs import onionrproofs
def insert_block(data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None, disableForward=False): import logger
''' def insert_block(data: Union[str, bytes], header: str ='txt',
sign: bool =False, encryptType:str ='', symKey:str ='',
asymPeer:str ='', meta:dict = {},
expire:Union[int, None] =None, disableForward:bool =False)->Union[str,bool]:
"""
Inserts a block into the network Inserts a block into the network
encryptType must be specified to encrypt a block encryptType must be specified to encrypt a block
''' """
use_subprocess = powchoice.use_subprocess(config) use_subprocess = powchoice.use_subprocess(config)
storage_counter = storagecounter.StorageCounter() storage_counter = storagecounter.StorageCounter()
allocationReachedMessage = 'Cannot insert block, disk allocation reached.' allocationReachedMessage = 'Cannot insert block, disk allocation reached.'
if storage_counter.isFull(): if storage_counter.is_full():
logger.error(allocationReachedMessage) logger.error(allocationReachedMessage)
return False raise onionrexceptions.DiskAllocationReached
retData = False retData = False
if type(data) is None: if type(data) is None:

View File

@ -1,9 +1,9 @@
''' """
Onionr - P2P Anonymous Storage Network Onionr - Private P2P Communication
This file contains exceptions for onionr This file contains exceptions for onionr
''' """
''' """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or the Free Software Foundation, either version 3 of the License, or
@ -16,7 +16,7 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' """
# general exceptions # general exceptions
class NotFound(Exception): class NotFound(Exception):
@ -67,11 +67,11 @@ class NoDataAvailable(Exception):
pass pass
class InvalidHexHash(Exception): class InvalidHexHash(Exception):
'''When a string is not a valid hex string of appropriate length for a hash value''' """When a string is not a valid hex string of appropriate length for a hash value"""
pass pass
class InvalidProof(Exception): class InvalidProof(Exception):
'''When a proof is invalid or inadequate''' """When a proof is invalid or inadequate"""
pass pass
# network level exceptions # network level exceptions

View File

@ -27,7 +27,7 @@ def getDifficultyModifier():
on a variety of factors, currently only disk use. on a variety of factors, currently only disk use.
''' '''
retData = 0 retData = 0
useFunc = storagecounter.StorageCounter().getPercent useFunc = storagecounter.StorageCounter().get_percent
percentUse = useFunc() percentUse = useFunc()

View File

@ -22,7 +22,7 @@ import config, logger, netcontroller
from etc import onionrvalues from etc import onionrvalues
from logger.settings import * from logger.settings import *
def setup_config(o_inst = None): def setup_config():
config.reload() config.reload()
if not os.path.exists(config._configfile): if not os.path.exists(config._configfile):
@ -42,14 +42,6 @@ def setup_config(o_inst = None):
settings = settings | OUTPUT_TO_FILE settings = settings | OUTPUT_TO_FILE
set_settings(settings) set_settings(settings)
if not o_inst is None:
if str(config.get('general.dev_mode', True)).lower() == 'true':
o_inst._developmentMode = True
set_level(LEVEL_DEBUG)
else:
o_inst._developmentMode = False
set_level(LEVEL_INFO)
verbosity = str(config.get('log.verbosity', 'default')).lower().strip() verbosity = str(config.get('log.verbosity', 'default')).lower().strip()
if not verbosity in ['default', 'null', 'none', 'nil']: if not verbosity in ['default', 'null', 'none', 'nil']:
map = { map = {

View File

@ -18,6 +18,6 @@ def remove_block(block):
conn.commit() conn.commit()
conn.close() conn.close()
dataSize = sys.getsizeof(onionrstorage.getData(block)) dataSize = sys.getsizeof(onionrstorage.getData(block))
storagecounter.StorageCounter().removeBytes(dataSize) storagecounter.StorageCounter().remove_bytes(dataSize)
else: else:
raise onionrexceptions.InvalidHexHash raise onionrexceptions.InvalidHexHash

View File

@ -3,7 +3,7 @@ import onionrstorage, onionrexceptions, onionrcrypto as crypto
import filepaths, storagecounter import filepaths, storagecounter
from coredb import dbfiles from coredb import dbfiles
from onionrutils import blockmetadata, bytesconverter from onionrutils import blockmetadata, bytesconverter
def set_data(data): def set_data(data)->str:
''' '''
Set the data assciated with a hash Set the data assciated with a hash
''' '''
@ -24,7 +24,7 @@ def set_data(data):
try: try:
onionrstorage.getData(dataHash) onionrstorage.getData(dataHash)
except onionrexceptions.NoDataAvailable: except onionrexceptions.NoDataAvailable:
if storage_counter.addBytes(dataSize) != False: if storage_counter.add_bytes(dataSize) != False:
onionrstorage.store(data, blockHash=dataHash) onionrstorage.store(data, blockHash=dataHash)
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30) conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
c = conn.cursor() c = conn.cursor()

View File

@ -1,9 +1,9 @@
''' """
Onionr - Private P2P Communication Onionr - Private P2P Communication
Keeps track of how much disk space we're using Keeps track of how much disk space we're using
''' """
''' """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or the Free Software Foundation, either version 3 of the License, or
@ -16,52 +16,54 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' """
import config, filepaths import config, filepaths
config.reload() config.reload()
class StorageCounter: class StorageCounter:
def __init__(self): def __init__(self):
self.dataFile = filepaths.usage_file self.data_file = filepaths.usage_file
return return
def isFull(self): def is_full(self)->bool:
retData = False """Returns if the allocated disk space is full (this is Onionr config, not true FS capacity)"""
if config.get('allocations.disk', 2000000000) <= (self.getAmount() + 1000): ret_data = False
retData = True if config.get('allocations.disk', 2000000000) <= (self.get_amount() + 1000):
return retData ret_data = True
return ret_data
def _update(self, data): def _update(self, data):
with open(self.dataFile, 'w') as dataFile: with open(self.data_file, 'w') as data_file:
dataFile.write(str(data)) data_file.write(str(data))
def getAmount(self):
'''Return how much disk space we're using (according to record)''' def get_amount(self)->int:
retData = 0 """Return how much disk space we're using (according to record)"""
ret_data = 0
try: try:
with open(self.dataFile, 'r') as dataFile: with open(self.data_file, 'r') as data_file:
retData = int(dataFile.read()) ret_data = int(data_file.read())
except FileNotFoundError: except FileNotFoundError:
pass pass
except ValueError: except ValueError:
pass # Possibly happens when the file is empty pass # Possibly happens when the file is empty
return retData return ret_data
def getPercent(self): def get_percent(self)->int:
'''Return percent (decimal/float) of disk space we're using''' """Return percent (decimal/float) of disk space we're using"""
amount = self.getAmount() amount = self.get_amount()
return round(amount / config.get('allocations.disk', 2000000000), 2) return round(amount / config.get('allocations.disk', 2000000000), 2)
def addBytes(self, amount): def add_bytes(self, amount)->int:
'''Record that we are now using more disk space, unless doing so would exceed configured max''' """Record that we are now using more disk space, unless doing so would exceed configured max"""
newAmount = amount + self.getAmount() new_amount = amount + self.get_amount()
retData = newAmount ret_data = new_amount
if newAmount > config.get('allocations.disk', 2000000000): if new_amount > config.get('allocations.disk', 2000000000):
retData = False ret_data = False
else: else:
self._update(newAmount) self._update(new_amount)
return retData return ret_data
def removeBytes(self, amount): def remove_bytes(self, amount)->int:
'''Record that we are now using less disk space''' """Record that we are now using less disk space"""
newAmount = self.getAmount() - amount new_amount = self.get_amount() - amount
self._update(newAmount) self._update(new_amount)
return newAmount return new_amount

View File

@ -8,7 +8,7 @@ print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR os.environ["ONIONR_HOME"] = TEST_DIR
from utils import createdirs from utils import createdirs
from coredb import keydb from coredb import keydb
import setup, keymanager, filepaths import onionrsetup as setup, keymanager, filepaths
from onionrutils import stringvalidators from onionrutils import stringvalidators
createdirs.create_dirs() createdirs.create_dirs()
setup.setup_config() setup.setup_config()

View File

@ -8,7 +8,7 @@ print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR os.environ["ONIONR_HOME"] = TEST_DIR
from utils import networkmerger, createdirs from utils import networkmerger, createdirs
from coredb import keydb from coredb import keydb
import setup import onionrsetup as setup
from utils import createdirs from utils import createdirs
createdirs.create_dirs() createdirs.create_dirs()
setup.setup_config() setup.setup_config()

View File

@ -0,0 +1,45 @@
import sys, os
sys.path.append(".")
import unittest, uuid
import logger
import config
from utils import createdirs
import onionrsetup as setup
from utils import createdirs
import onionrblocks
import filepaths
import onionrexceptions
import storagecounter
import onionrstorage
def _test_setup():
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
createdirs.create_dirs()
setup.setup_config()
config.reload()
class TestStorageCounter(unittest.TestCase):
def test_basic_amount(self):
_test_setup()
self.assertIsNotNone(config.get('allocations.disk'))
self.assertGreater(config.get('allocations.disk'), 1000000)
def test_insert_too_much(self):
_test_setup()
config.set('allocations.disk', 1000)
self.assertRaises(onionrexceptions.DiskAllocationReached, onionrblocks.insert, "test")
def test_count(self):
_test_setup()
counter = storagecounter.StorageCounter()
start_value = counter.get_amount()
b_hash = onionrblocks.insert("test")
self.assertGreater(counter.get_amount(), start_value)
onionrstorage.removeblock.remove_block(b_hash)
self.assertEqual(counter.get_amount(), start_value)
unittest.main()

View File

@ -5,7 +5,7 @@ gevent==1.3.6
Flask==1.1.1 Flask==1.1.1
PySocks==1.6.8 PySocks==1.6.8
stem==1.7.1 stem==1.7.1
deadsimplekv==0.1.1 deadsimplekv==0.2.0
unpaddedbase32==0.1.0 unpaddedbase32==0.1.0
streamedrequests==1.0.0 streamedrequests==1.0.0
jinja2==2.10.1 jinja2==2.10.1

View File

@ -46,8 +46,8 @@ click==7.0 \
--hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \ --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
--hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \ --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \
# via flask # via flask
deadsimplekv==0.1.1 \ deadsimplekv==0.2.0 \
--hash=sha256:4bf951e188c302006e37f95bde6117b1b938fb454153d583c6346090d9bead1a --hash=sha256:81405408a4d23cc94ac359f9570e0ff198b67e5a93e3ae32eca85e3b62252f38
flask==1.1.1 \ flask==1.1.1 \
--hash=sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52 \ --hash=sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52 \
--hash=sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6 --hash=sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6