added storagecounter test and renamed setup

This commit is contained in:
Kevin Froman 2019-09-08 04:48:16 -05:00
parent 311dda91d6
commit 085b90f84c
20 changed files with 120 additions and 76 deletions

View File

@ -36,7 +36,7 @@ except ModuleNotFoundError:
# Onionr imports
from etc import onionrvalues # For different Onionr related constants such as versions
import setup
import onionrsetup as setup
# Ensure we have at least the minimum python version
if sys.version_info[0] == 2 or sys.version_info[1] < onionrvalues.MIN_PY_VERSION:

View File

@ -46,7 +46,7 @@ def download_blocks_from_communicator(comm_inst):
if not shoulddownload.should_download(comm_inst, blockHash):
continue
if comm_inst.shutdown or not comm_inst.isOnline or storage_counter.isFull():
if comm_inst.shutdown or not comm_inst.isOnline or storage_counter.is_full():
# Exit loop if shutting down or offline, or disk allocation reached
break
# Do not download blocks being downloaded

View File

@ -43,7 +43,7 @@ def clean_old_blocks(comm_inst):
__remove_from_upload(comm_inst, bHash)
logger.info('Deleted block: %s' % (bHash,))
while comm_inst.storage_counter.isFull():
while comm_inst.storage_counter.is_full():
oldest = blockmetadb.get_block_list()[0]
blacklist.addToDB(oldest)
removeblock.remove_block(oldest)

View File

@ -40,7 +40,7 @@ def lookup_blocks_from_communicator(comm_inst):
if not comm_inst.isOnline:
break
# check if disk allocation is used
if comm_inst.storage_counter.isFull():
if comm_inst.storage_counter.is_full():
logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
break
peer = onlinepeers.pick_online_peer(comm_inst) # select random online peer

View File

@ -28,33 +28,33 @@ raw = raw.raw
confirm = confirm.confirm
# debug: when there is info that could be useful for debugging purposes only
def debug(data, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_DEBUG):
def debug(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_DEBUG):
if settings.get_level() <= level:
log('/', data, timestamp = timestamp, prompt = prompt, terminal = terminal)
if not error is None:
debug('Error: ' + str(error) + parse_error())
# info: when there is something to notify the user of, such as the success of a process
def info(data, timestamp = False, prompt = True, terminal = False, level = settings.LEVEL_INFO):
def info(data: str, timestamp = False, prompt = True, terminal = False, level = settings.LEVEL_INFO):
if settings.get_level() <= level:
log('+', data, colors.fg.green, timestamp = timestamp, prompt = prompt, terminal = terminal)
# warn: when there is a potential for something bad to happen
def warn(data, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_WARN):
def warn(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_WARN):
if not error is None:
debug('Error: ' + str(error) + parse_error())
if settings.get_level() <= level:
log('!', data, colors.fg.orange, timestamp = timestamp, prompt = prompt, terminal = terminal)
# error: when only one function, module, or process of the program encountered a problem and must stop
def error(data, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_ERROR):
def error(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_ERROR):
if settings.get_level() <= level:
log('-', data, colors.fg.red, timestamp = timestamp, fd = sys.stderr, prompt = prompt, terminal = terminal)
if not error is None:
debug('Error: ' + str(error) + parse_error())
# fatal: when the something so bad has happened that the program must stop
def fatal(data, error = None, timestamp=True, prompt = True, terminal = False, level = settings.LEVEL_FATAL):
def fatal(data: str, error = None, timestamp=True, prompt = True, terminal = False, level = settings.LEVEL_FATAL):
if not error is None:
debug('Error: ' + str(error) + parse_error(), terminal = terminal)
if settings.get_level() <= level:

View File

@ -64,7 +64,7 @@ def set_level(level):
global _level
_level = level
def get_level():
def get_level()->int:
'''
Get the lowest log level currently being outputted
'''

View File

@ -1,3 +1,4 @@
from typing import Union
import json
from onionrutils import bytesconverter, epoch
import storagecounter, filepaths, onionrstorage
@ -8,17 +9,21 @@ from onionrusers import onionrusers
from onionrutils import localcommand, blockmetadata, stringvalidators
import coredb
import onionrproofs
def insert_block(data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None, disableForward=False):
'''
import logger
def insert_block(data: Union[str, bytes], header: str ='txt',
sign: bool =False, encryptType:str ='', symKey:str ='',
asymPeer:str ='', meta:dict = {},
expire:Union[int, None] =None, disableForward:bool =False)->Union[str,bool]:
"""
Inserts a block into the network
encryptType must be specified to encrypt a block
'''
"""
use_subprocess = powchoice.use_subprocess(config)
storage_counter = storagecounter.StorageCounter()
allocationReachedMessage = 'Cannot insert block, disk allocation reached.'
if storage_counter.isFull():
if storage_counter.is_full():
logger.error(allocationReachedMessage)
return False
raise onionrexceptions.DiskAllocationReached
retData = False
if type(data) is None:

View File

@ -1,9 +1,9 @@
'''
Onionr - P2P Anonymous Storage Network
"""
Onionr - Private P2P Communication
This file contains exceptions for onionr
'''
'''
"""
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,7 +16,7 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
"""
# general exceptions
class NotFound(Exception):
@ -67,11 +67,11 @@ class NoDataAvailable(Exception):
pass
class InvalidHexHash(Exception):
'''When a string is not a valid hex string of appropriate length for a hash value'''
"""When a string is not a valid hex string of appropriate length for a hash value"""
pass
class InvalidProof(Exception):
'''When a proof is invalid or inadequate'''
"""When a proof is invalid or inadequate"""
pass
# network level exceptions

View File

@ -27,7 +27,7 @@ def getDifficultyModifier():
on a variety of factors, currently only disk use.
'''
retData = 0
useFunc = storagecounter.StorageCounter().getPercent
useFunc = storagecounter.StorageCounter().get_percent
percentUse = useFunc()

View File

@ -22,7 +22,7 @@ import config, logger, netcontroller
from etc import onionrvalues
from logger.settings import *
def setup_config(o_inst = None):
def setup_config():
config.reload()
if not os.path.exists(config._configfile):
@ -42,14 +42,6 @@ def setup_config(o_inst = None):
settings = settings | OUTPUT_TO_FILE
set_settings(settings)
if not o_inst is None:
if str(config.get('general.dev_mode', True)).lower() == 'true':
o_inst._developmentMode = True
set_level(LEVEL_DEBUG)
else:
o_inst._developmentMode = False
set_level(LEVEL_INFO)
verbosity = str(config.get('log.verbosity', 'default')).lower().strip()
if not verbosity in ['default', 'null', 'none', 'nil']:
map = {

View File

@ -18,6 +18,6 @@ def remove_block(block):
conn.commit()
conn.close()
dataSize = sys.getsizeof(onionrstorage.getData(block))
storagecounter.StorageCounter().removeBytes(dataSize)
storagecounter.StorageCounter().remove_bytes(dataSize)
else:
raise onionrexceptions.InvalidHexHash

View File

@ -3,7 +3,7 @@ import onionrstorage, onionrexceptions, onionrcrypto as crypto
import filepaths, storagecounter
from coredb import dbfiles
from onionrutils import blockmetadata, bytesconverter
def set_data(data):
def set_data(data)->str:
'''
Set the data assciated with a hash
'''
@ -24,7 +24,7 @@ def set_data(data):
try:
onionrstorage.getData(dataHash)
except onionrexceptions.NoDataAvailable:
if storage_counter.addBytes(dataSize) != False:
if storage_counter.add_bytes(dataSize) != False:
onionrstorage.store(data, blockHash=dataHash)
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
c = conn.cursor()

View File

@ -1,9 +1,9 @@
'''
"""
Onionr - Private P2P Communication
Keeps track of how much disk space we're using
'''
'''
"""
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,52 +16,54 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
"""
import config, filepaths
config.reload()
class StorageCounter:
def __init__(self):
self.dataFile = filepaths.usage_file
self.data_file = filepaths.usage_file
return
def isFull(self):
retData = False
if config.get('allocations.disk', 2000000000) <= (self.getAmount() + 1000):
retData = True
return retData
def is_full(self)->bool:
"""Returns if the allocated disk space is full (this is Onionr config, not true FS capacity)"""
ret_data = False
if config.get('allocations.disk', 2000000000) <= (self.get_amount() + 1000):
ret_data = True
return ret_data
def _update(self, data):
with open(self.dataFile, 'w') as dataFile:
dataFile.write(str(data))
def getAmount(self):
'''Return how much disk space we're using (according to record)'''
retData = 0
with open(self.data_file, 'w') as data_file:
data_file.write(str(data))
def get_amount(self)->int:
"""Return how much disk space we're using (according to record)"""
ret_data = 0
try:
with open(self.dataFile, 'r') as dataFile:
retData = int(dataFile.read())
with open(self.data_file, 'r') as data_file:
ret_data = int(data_file.read())
except FileNotFoundError:
pass
except ValueError:
pass # Possibly happens when the file is empty
return retData
return ret_data
def getPercent(self):
'''Return percent (decimal/float) of disk space we're using'''
amount = self.getAmount()
def get_percent(self)->int:
"""Return percent (decimal/float) of disk space we're using"""
amount = self.get_amount()
return round(amount / config.get('allocations.disk', 2000000000), 2)
def addBytes(self, amount):
'''Record that we are now using more disk space, unless doing so would exceed configured max'''
newAmount = amount + self.getAmount()
retData = newAmount
if newAmount > config.get('allocations.disk', 2000000000):
retData = False
def add_bytes(self, amount)->int:
"""Record that we are now using more disk space, unless doing so would exceed configured max"""
new_amount = amount + self.get_amount()
ret_data = new_amount
if new_amount > config.get('allocations.disk', 2000000000):
ret_data = False
else:
self._update(newAmount)
return retData
self._update(new_amount)
return ret_data
def removeBytes(self, amount):
'''Record that we are now using less disk space'''
newAmount = self.getAmount() - amount
self._update(newAmount)
return newAmount
def remove_bytes(self, amount)->int:
"""Record that we are now using less disk space"""
new_amount = self.get_amount() - amount
self._update(new_amount)
return new_amount

View File

@ -8,7 +8,7 @@ print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
from utils import createdirs
from coredb import keydb
import setup, keymanager, filepaths
import onionrsetup as setup, keymanager, filepaths
from onionrutils import stringvalidators
createdirs.create_dirs()
setup.setup_config()

View File

@ -8,7 +8,7 @@ print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
from utils import networkmerger, createdirs
from coredb import keydb
import setup
import onionrsetup as setup
from utils import createdirs
createdirs.create_dirs()
setup.setup_config()

View File

@ -0,0 +1,45 @@
import sys, os
sys.path.append(".")
import unittest, uuid
import logger
import config
from utils import createdirs
import onionrsetup as setup
from utils import createdirs
import onionrblocks
import filepaths
import onionrexceptions
import storagecounter
import onionrstorage
def _test_setup():
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
createdirs.create_dirs()
setup.setup_config()
config.reload()
class TestStorageCounter(unittest.TestCase):
def test_basic_amount(self):
_test_setup()
self.assertIsNotNone(config.get('allocations.disk'))
self.assertGreater(config.get('allocations.disk'), 1000000)
def test_insert_too_much(self):
_test_setup()
config.set('allocations.disk', 1000)
self.assertRaises(onionrexceptions.DiskAllocationReached, onionrblocks.insert, "test")
def test_count(self):
_test_setup()
counter = storagecounter.StorageCounter()
start_value = counter.get_amount()
b_hash = onionrblocks.insert("test")
self.assertGreater(counter.get_amount(), start_value)
onionrstorage.removeblock.remove_block(b_hash)
self.assertEqual(counter.get_amount(), start_value)
unittest.main()

View File

@ -5,7 +5,7 @@ gevent==1.3.6
Flask==1.1.1
PySocks==1.6.8
stem==1.7.1
deadsimplekv==0.1.1
deadsimplekv==0.2.0
unpaddedbase32==0.1.0
streamedrequests==1.0.0
jinja2==2.10.1

View File

@ -46,8 +46,8 @@ click==7.0 \
--hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
--hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \
# via flask
deadsimplekv==0.1.1 \
--hash=sha256:4bf951e188c302006e37f95bde6117b1b938fb454153d583c6346090d9bead1a
deadsimplekv==0.2.0 \
--hash=sha256:81405408a4d23cc94ac359f9570e0ff198b67e5a93e3ae32eca85e3b62252f38
flask==1.1.1 \
--hash=sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52 \
--hash=sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6