From 304df0fbb4e689fc5ed097b6376b270a3e0d1daa Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 29 Oct 2019 00:36:56 -0500 Subject: [PATCH] lock databases with timeout from onionrvalues --- onionr/coredb/blockmetadb/__init__.py | 9 ++++++--- onionr/coredb/blockmetadb/add.py | 3 ++- onionr/coredb/blockmetadb/expiredblocks.py | 4 +++- onionr/coredb/blockmetadb/updateblockinfo.py | 3 ++- onionr/coredb/daemonqueue/__init__.py | 7 ++++--- onionr/coredb/keydb/addkeys.py | 6 ++++-- onionr/coredb/keydb/listkeys.py | 5 +++-- onionr/coredb/keydb/removekeys.py | 5 +++-- onionr/coredb/keydb/transportinfo.py | 6 ++++-- onionr/coredb/keydb/userinfo.py | 6 ++++-- onionr/etc/onionrvalues.py | 2 ++ 11 files changed, 37 insertions(+), 19 deletions(-) diff --git a/onionr/coredb/blockmetadb/__init__.py b/onionr/coredb/blockmetadb/__init__.py index 2e40406a..0a2d3438 100644 --- a/onionr/coredb/blockmetadb/__init__.py +++ b/onionr/coredb/blockmetadb/__init__.py @@ -18,8 +18,11 @@ along with this program. If not, see . ''' import sqlite3 + +from etc import onionrvalues from . import expiredblocks, updateblockinfo, add from .. import dbfiles + update_block_info = updateblockinfo.update_block_info add_to_block_DB = add.add_to_block_DB def get_block_list(dateRec = None, unsaved = False): @@ -29,7 +32,7 @@ def get_block_list(dateRec = None, unsaved = False): if dateRec == None: dateRec = 0 - conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30) + conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;' @@ -46,7 +49,7 @@ def get_block_date(blockHash): Returns the date a block was received ''' - conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30) + conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() execute = 'SELECT dateReceived FROM hashes WHERE hash=?;' @@ -62,7 +65,7 @@ def get_blocks_by_type(blockType, orderDate=True): Returns a list of blocks by the type ''' - conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30) + conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() if orderDate: diff --git a/onionr/coredb/blockmetadb/add.py b/onionr/coredb/blockmetadb/add.py index 100f3992..e1c67f56 100644 --- a/onionr/coredb/blockmetadb/add.py +++ b/onionr/coredb/blockmetadb/add.py @@ -19,6 +19,7 @@ ''' import os, sqlite3, secrets from onionrutils import epoch, blockmetadata +from etc import onionrvalues from .. import dbfiles def add_to_block_DB(newHash, selfInsert=False, dataSaved=False): ''' @@ -29,7 +30,7 @@ def add_to_block_DB(newHash, selfInsert=False, dataSaved=False): if blockmetadata.has_block(newHash): return - conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30) + conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() currentTime = epoch.get_epoch() + secrets.randbelow(301) if selfInsert or dataSaved: diff --git a/onionr/coredb/blockmetadb/expiredblocks.py b/onionr/coredb/blockmetadb/expiredblocks.py index ac66be66..120e7566 100644 --- a/onionr/coredb/blockmetadb/expiredblocks.py +++ b/onionr/coredb/blockmetadb/expiredblocks.py @@ -20,9 +20,11 @@ import sqlite3 from onionrutils import epoch from .. import dbfiles +from etc import onionrvalues + def get_expired_blocks(): '''Returns a list of expired blocks''' - conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30) + conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() date = int(epoch.get_epoch()) diff --git a/onionr/coredb/blockmetadb/updateblockinfo.py b/onionr/coredb/blockmetadb/updateblockinfo.py index d79ebece..3fdc4033 100644 --- a/onionr/coredb/blockmetadb/updateblockinfo.py +++ b/onionr/coredb/blockmetadb/updateblockinfo.py @@ -19,6 +19,7 @@ ''' import sqlite3 from .. import dbfiles +from etc import onionrvalues def update_block_info(hash, key, data): ''' sets info associated with a block @@ -38,7 +39,7 @@ def update_block_info(hash, key, data): 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'): raise ValueError('Key must be in the allowed list') - conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30) + conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() args = (data, hash) # Unfortunately, not really possible diff --git a/onionr/coredb/daemonqueue/__init__.py b/onionr/coredb/daemonqueue/__init__.py index 67e05975..ee4d9943 100644 --- a/onionr/coredb/daemonqueue/__init__.py +++ b/onionr/coredb/daemonqueue/__init__.py @@ -23,6 +23,7 @@ from onionrplugins import onionrevents as events from onionrutils import localcommand, epoch from .. import dbfiles from onionrsetup import dbcreator +from etc import onionrvalues def daemon_queue()->str: ''' @@ -35,7 +36,7 @@ def daemon_queue()->str: if not os.path.exists(dbfiles.daemon_queue_db): dbcreator.createDaemonDB() else: - conn = sqlite3.connect(dbfiles.daemon_queue_db, timeout=30) + conn = sqlite3.connect(dbfiles.daemon_queue_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() try: for row in c.execute('SELECT command, data, date, min(ID), responseID FROM commands group by id'): @@ -59,7 +60,7 @@ def daemon_queue_add(command: str, data='', responseID: str =''): retData = True date = epoch.get_epoch() - conn = sqlite3.connect(dbfiles.daemon_queue_db, timeout=30) + conn = sqlite3.connect(dbfiles.daemon_queue_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() t = (command, data, date, responseID) try: @@ -83,7 +84,7 @@ def clear_daemon_queue(): ''' Clear the daemon queue (somewhat dangerous) ''' - conn = sqlite3.connect(dbfiles.daemon_queue_db, timeout=30) + conn = sqlite3.connect(dbfiles.daemon_queue_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() c.execute('DELETE FROM commands;') diff --git a/onionr/coredb/keydb/addkeys.py b/onionr/coredb/keydb/addkeys.py index 0d6a390e..ad3a34f5 100644 --- a/onionr/coredb/keydb/addkeys.py +++ b/onionr/coredb/keydb/addkeys.py @@ -24,6 +24,8 @@ from . import listkeys from utils import gettransports from .. import dbfiles import onionrcrypto +from etc import onionrvalues + def add_peer(peerID, name=''): ''' Adds a public key to the key database (misleading function name) @@ -37,7 +39,7 @@ def add_peer(peerID, name=''): #events.event('pubkey_add', data = {'key': peerID}, onionr = core_inst.onionrInst) - conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=30) + conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) hashID = "" c = conn.cursor() t = (peerID, name, 'unknown', hashID, 0) @@ -67,7 +69,7 @@ def add_address(address): if stringvalidators.validate_transport(address): if address in gettransports.get(): return False - conn = sqlite3.connect(dbfiles.address_info_db, timeout=30) + conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() # check if address is in database # this is safe to do because the address is validated above, but we strip some chars here too just in case diff --git a/onionr/coredb/keydb/listkeys.py b/onionr/coredb/keydb/listkeys.py index 65da2c5d..ff56b15e 100644 --- a/onionr/coredb/keydb/listkeys.py +++ b/onionr/coredb/keydb/listkeys.py @@ -20,6 +20,7 @@ import sqlite3 import logger from onionrutils import epoch +from etc import onionrvalues from .. import dbfiles from . import userinfo, transportinfo def list_peers(randomOrder=True, getPow=False, trust=0): @@ -29,7 +30,7 @@ def list_peers(randomOrder=True, getPow=False, trust=0): randomOrder determines if the list should be in a random order trust sets the minimum trust to list ''' - conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=30) + conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() payload = '' @@ -63,7 +64,7 @@ def list_adders(randomOrder=True, i2p=True, recent=0): ''' Return a list of transport addresses ''' - conn = sqlite3.connect(dbfiles.address_info_db, timeout=30) + conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() if randomOrder: addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();') diff --git a/onionr/coredb/keydb/removekeys.py b/onionr/coredb/keydb/removekeys.py index 7e8baaab..46d2d641 100644 --- a/onionr/coredb/keydb/removekeys.py +++ b/onionr/coredb/keydb/removekeys.py @@ -22,6 +22,7 @@ from onionrplugins import onionrevents as events from onionrutils import stringvalidators from onionrutils import mnemonickeys from .. import dbfiles +from etc import onionrvalues def remove_address(address): ''' @@ -29,7 +30,7 @@ def remove_address(address): ''' if stringvalidators.validate_transport(address): - conn = sqlite3.connect(dbfiles.address_info_db, timeout=30) + conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() t = (address,) c.execute('Delete from adders where address=?;', t) @@ -47,7 +48,7 @@ def remove_user(pubkey: str)->bool: ''' pubkey = mnemonickeys.get_base32(pubkey) if stringvalidators.validate_pub_key(pubkey): - conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=30) + conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() t = (pubkey,) c.execute('Delete from peers where id=?;', t) diff --git a/onionr/coredb/keydb/transportinfo.py b/onionr/coredb/keydb/transportinfo.py index 442d0c13..d9efd365 100644 --- a/onionr/coredb/keydb/transportinfo.py +++ b/onionr/coredb/keydb/transportinfo.py @@ -19,6 +19,8 @@ ''' import sqlite3 from .. import dbfiles +from etc import onionrvalues + def get_address_info(address, info): ''' Get info about an address from its database entry @@ -35,7 +37,7 @@ def get_address_info(address, info): introduced 9 ''' - conn = sqlite3.connect(dbfiles.address_info_db, timeout=30) + conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() command = (address,) @@ -60,7 +62,7 @@ def set_address_info(address, key, data): Update an address for a key ''' - conn = sqlite3.connect(dbfiles.address_info_db, timeout=30) + conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() command = (data, address) diff --git a/onionr/coredb/keydb/userinfo.py b/onionr/coredb/keydb/userinfo.py index c395f493..3f055b49 100644 --- a/onionr/coredb/keydb/userinfo.py +++ b/onionr/coredb/keydb/userinfo.py @@ -19,6 +19,8 @@ ''' import sqlite3 from .. import dbfiles +from etc import onionrvalues + def get_user_info(peer, info): ''' Get info about a peer from their database entry @@ -30,7 +32,7 @@ def get_user_info(peer, info): trust int 4 hashID text 5 ''' - conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=30) + conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() command = (peer,) @@ -56,7 +58,7 @@ def set_peer_info(peer, key, data): Update a peer for a key ''' - conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=30) + conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) c = conn.cursor() command = (data, peer) diff --git a/onionr/etc/onionrvalues.py b/onionr/etc/onionrvalues.py index a192fc71..bd410532 100755 --- a/onionr/etc/onionrvalues.py +++ b/onionr/etc/onionrvalues.py @@ -39,6 +39,8 @@ MAX_BLOCK_CLOCK_SKEW = 120 MAIN_PUBLIC_KEY_SIZE = 32 ORIG_RUN_DIR_ENV_VAR = 'ORIG_ONIONR_RUN_DIR' +DATABASE_LOCK_TIMEOUT = 60 + # Block creation anonymization requirements MIN_BLOCK_UPLOAD_PEER_PERCENT = 0.1