kev/Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

Added/corrected timeouts for sqlite3 database connections

Browse Source 
Bumped mail plugin patch version for sqlite3 timeout change
Code formatting corrections
This commit is contained in:
Kevin Froman 2020-08-16 19:52:50 -05:00
parent 2a7c933321
commit 0b2658374b
13 changed files with 158 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/communicatorutils/housekeeping.py
View File

@ -17,6 +17,7 @@ import onionrstorage
from onionrstorage import removeblock
from onionrblocks import onionrblacklist
from onionrblocks.storagecounter import StorageCounter
from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -71,7 +72,8 @@ def clean_old_blocks(comm_inst):
def clean_keys(comm_inst):
"""Delete expired forward secrecy keys"""
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
conn = sqlite3.connect(dbfiles.user_id_info_db,
timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
time = epoch.get_epoch()
deleteKeys = []

38
src/coredb/blockmetadb/__init__.py
View File

@ -1,7 +1,6 @@
"""
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
This module works with information relating to blocks stored on the node
Work with information relating to blocks stored on the node
"""
import sqlite3
@ -27,18 +26,18 @@ update_block_info = updateblockinfo.update_block_info
add_to_block_DB = add.add_to_block_DB
def get_block_list(dateRec = None, unsaved = False):
"""
Get list of our blocks
"""
if dateRec == None:
dateRec = 0
def get_block_list(date_rec=None, unsaved=False):
"""Get list of our blocks."""
if date_rec is None:
date_rec = 0
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
conn = sqlite3.connect(
dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
args = (dateRec,)
execute = 'SELECT hash FROM hashes WHERE dateReceived' + \
' >= ? ORDER BY dateReceived ASC;'
args = (date_rec,)
rows = list()
for row in c.execute(execute, args):
for i in row:
@ -48,11 +47,9 @@ def get_block_list(dateRec = None, unsaved = False):
def get_block_date(blockHash):
"""
Returns the date a block was received
"""
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
"""Return the date a block was received."""
conn = sqlite3.connect(
dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
@ -65,11 +62,10 @@ def get_block_date(blockHash):
def get_blocks_by_type(blockType, orderDate=True):
"""
Returns a list of blocks by the type
"""
"""Return a list of blocks by the type."""
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
conn = sqlite3.connect(
dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
if orderDate:

25
src/coredb/blockmetadb/expiredblocks.py
View File

@ -1,9 +1,12 @@
'''
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
Get a list of expired blocks still stored
'''
'''
Get a list of expired blocks still stored
"""
import sqlite3
from onionrutils import epoch
from .. import dbfiles
from etc import onionrvalues
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,15 +19,13 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3
from onionrutils import epoch
from .. import dbfiles
from etc import onionrvalues
"""
def get_expired_blocks():
'''Returns a list of expired blocks'''
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
"""Return a list of expired blocks"""
conn = sqlite3.connect(
dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
date = int(epoch.get_epoch())

57
src/coredb/keydb/transportinfo.py
View File

@ -1,9 +1,11 @@
'''
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
get or set transport address meta information
'''
'''
get or set transport address meta information
"""
import sqlite3
from .. import dbfiles
from etc import onionrvalues
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,28 +18,27 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3
from .. import dbfiles
from etc import onionrvalues
"""
def get_address_info(address, info):
'''
Get info about an address from its database entry
"""
Get info about an address from its database entry
address text, 0
type int, 1
knownPeer text, 2
speed int, 3
success int, 4
powValue 5
failure int 6
lastConnect 7
trust 8
introduced 9
'''
address text, 0
type int, 1
knownPeer text, 2
speed int, 3
success int, 4
powValue 5
failure int 6
lastConnect 7
trust 8
introduced 9
"""
conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
conn = sqlite3.connect(
dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
command = (address,)
@ -57,12 +58,12 @@ def get_address_info(address, info):
return retVal
def set_address_info(address, key, data):
'''
Update an address for a key
'''
conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
def set_address_info(address, key, data):
"""Update an address for a key."""
conn = sqlite3.connect(
dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
command = (data, address)

5
src/onionrblocks/onionrblacklist.py
View File

@ -9,6 +9,7 @@ from onionrplugins.onionrevents import event
import onionrcrypto
from onionrutils import epoch, bytesconverter
from coredb import dbfiles
from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -48,8 +49,8 @@ class OnionrBlackList:
return retData
def _dbExecute(self, toExec, params = ()):
conn = sqlite3.connect(self.blacklistDB)
def _dbExecute(self, toExec, params=()):
conn = sqlite3.connect(self.blacklistDB, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
retData = c.execute(toExec, params)
conn.commit()

64
src/onionrstorage/__init__.py
View File

@ -9,10 +9,11 @@ import os
from onionrutils import bytesconverter
from onionrutils import stringvalidators
from coredb import dbfiles
import filepaths
from filepaths import block_data_location
import onionrexceptions
from onionrcrypto import hashers
from . import setdata
from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -34,73 +35,80 @@ DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option
set_data = setdata.set_data
def _dbInsert(blockHash, data):
conn = sqlite3.connect(dbfiles.block_data_db, timeout=10)
def _dbInsert(block_hash, data):
conn = sqlite3.connect(dbfiles.block_data_db,
timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
data = (blockHash, data)
data = (block_hash, data)
c.execute('INSERT INTO blockData (hash, data) VALUES(?, ?);', data)
conn.commit()
conn.close()
def _dbFetch(blockHash):
conn = sqlite3.connect(dbfiles.block_data_db, timeout=10)
def _dbFetch(block_hash):
conn = sqlite3.connect(dbfiles.block_data_db,
timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
for i in c.execute('SELECT data from blockData where hash = ?', (blockHash,)):
for i in c.execute(
'SELECT data from blockData where hash = ?', (block_hash,)):
return i[0]
conn.commit()
conn.close()
return None
def deleteBlock(blockHash):
# You should call removeblock.remove_block if you automatically want to remove storage byte count
if os.path.exists('%s/%s.dat' % (filepaths.block_data_location, blockHash)):
os.remove('%s/%s.dat' % (filepaths.block_data_location, blockHash))
def deleteBlock(block_hash):
# Call removeblock.remove_block to automatically want to remove storage byte count
if os.path.exists(f'{block_data_location}/{block_hash}.dat'):
os.remove(f'{block_data_location}/{block_hash}.dat')
return True
conn = sqlite3.connect(dbfiles.block_data_db, timeout=10)
conn = sqlite3.connect(dbfiles.block_data_db,
timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
data = (blockHash,)
data = (block_hash,)
c.execute('DELETE FROM blockData where hash = ?', data)
conn.commit()
conn.close()
return True
def store(data, blockHash=''):
if not stringvalidators.validate_hash(blockHash): raise ValueError
def store(data, block_hash=''):
if not stringvalidators.validate_hash(block_hash):
raise ValueError
ourHash = hashers.sha3_hash(data)
if blockHash != '':
if not ourHash == blockHash:
if block_hash != '':
if not ourHash == block_hash:
raise ValueError('Hash specified does not meet internal hash check')
else:
blockHash = ourHash
block_hash = ourHash
if DB_ENTRY_SIZE_LIMIT >= sys.getsizeof(data):
_dbInsert(blockHash, data)
_dbInsert(block_hash, data)
else:
with open('%s/%s.dat' % (filepaths.block_data_location, blockHash), 'wb') as blockFile:
blockFile.write(data)
with open(
f'{block_data_location}/{block_hash}.dat', 'wb') as blck_file:
blck_file.write(data)
def getData(bHash):
if not stringvalidators.validate_hash(bHash): raise ValueError
if not stringvalidators.validate_hash(bHash):
raise ValueError
bHash = bytesconverter.bytes_to_str(bHash)
bHash = bHash.strip()
# First check DB for data entry by hash
# if no entry, check disk
# If no entry in either, raise an exception
retData = None
fileLocation = '%s/%s.dat' % (filepaths.block_data_location, bHash)
ret_data = None
fileLocation = '%s/%s.dat' % (block_data_location, bHash)
not_found_msg = "Block data not found for: "
if os.path.exists(fileLocation):
with open(fileLocation, 'rb') as block:
retData = block.read()
ret_data = block.read()
else:
retData = _dbFetch(bHash)
ret_data = _dbFetch(bHash)
if retData is None:
if ret_data is None:
raise onionrexceptions.NoDataAvailable(not_found_msg + str(bHash))
return retData
return ret_data

4
src/onionrstorage/removeblock.py
View File

@ -10,6 +10,7 @@ import onionrstorage
from onionrutils import stringvalidators
from coredb import dbfiles
from onionrblocks import storagecounter
from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -33,7 +34,8 @@ def remove_block(block):
**You may want blacklist.addToDB(blockHash)
"""
if stringvalidators.validate_hash(block):
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
conn = sqlite3.connect(
dbfiles.block_meta_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
t = (block,)
c.execute('Delete from hashes where hash=?;', t)

4
src/onionrstorage/setdata.py
View File

@ -12,6 +12,7 @@ import filepaths
from onionrblocks import storagecounter
from coredb import dbfiles
from onionrutils import blockmetadata, bytesconverter
from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -50,7 +51,8 @@ def set_data(data) -> str:
except onionrexceptions.NoDataAvailable:
if storage_counter.add_bytes(dataSize) is not False:
onionrstorage.store(data, blockHash=dataHash)
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
conn = sqlite3.connect(
dbfiles.block_meta_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
c.execute(
"UPDATE hashes SET dataSaved=1 WHERE hash = ?;",

71
src/onionrusers/onionrusers.py
View File

@ -14,6 +14,7 @@ import nacl.exceptions
from coredb import keydb, dbfiles
import onionrcrypto
from onionrcrypto import getourkeypair
from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -32,7 +33,8 @@ from onionrcrypto import getourkeypair
def deleteExpiredKeys():
# Fetch the keys we generated for the peer, that are still around
conn = sqlite3.connect(dbfiles.forward_keys_db, timeout=10)
conn = sqlite3.connect(
dbfiles.forward_keys_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
curTime = epoch.get_epoch()
@ -44,7 +46,8 @@ def deleteExpiredKeys():
def deleteTheirExpiredKeys(pubkey):
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
conn = sqlite3.connect(
dbfiles.user_id_info_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
# Prepare the insert
@ -63,12 +66,13 @@ class OnionrUser:
def __init__(self, publicKey, saveUser=False):
"""
OnionrUser is an abstraction for "users" of the network.
OnionrUser is an abstraction for "users" of the network.
Takes a base32 encoded ed25519 public key, and a bool saveUser
saveUser determines if we should add a user to our peer database or not.
Takes a base32 encoded ed25519 public key, and a bool saveUser
saveUser determines if we should add a user to our peer database or not.
"""
publicKey = unpaddedbase32.repad(bytesconverter.str_to_bytes(publicKey)).decode()
publicKey = unpaddedbase32.repad(
bytesconverter.str_to_bytes(publicKey)).decode()
self.trust = 0
self.publicKey = publicKey
@ -76,7 +80,7 @@ class OnionrUser:
if saveUser and not publicKey == getourkeypair.get_keypair():
try:
keydb.addkeys.add_peer(publicKey)
except (AssertionError, ValueError) as e:
except (AssertionError, ValueError) as _:
pass
self.trust = keydb.userinfo.get_user_info(self.publicKey, 'trust')
@ -102,11 +106,13 @@ class OnionrUser:
return retData
def encrypt(self, data):
encrypted = onionrcrypto.encryption.pub_key_encrypt(data, self.publicKey, encodedData=True)
encrypted = onionrcrypto.encryption.pub_key_encrypt(
data, self.publicKey, encodedData=True)
return encrypted
def decrypt(self, data):
decrypted = onionrcrypto.encryption.pub_key_decrypt(data, self.publicKey, encodedData=True)
decrypted = onionrcrypto.encryption.pub_key_decrypt(
data, self.publicKey, encodedData=True)
return decrypted
def forwardEncrypt(self, data):
@ -115,33 +121,39 @@ class OnionrUser:
retData = ''
forwardKey = self._getLatestForwardKey()
if stringvalidators.validate_pub_key(forwardKey[0]):
retData = onionrcrypto.encryption.pub_key_encrypt(data, forwardKey[0], encodedData=True)
retData = onionrcrypto.encryption.pub_key_encrypt(
data, forwardKey[0], encodedData=True)
else:
raise onionrexceptions.InvalidPubkey("No valid forward secrecy key available for this user")
#self.generateForwardKey()
raise onionrexceptions.InvalidPubkey(
"No valid forward secrecy key available for this user")
return (retData, forwardKey[0], forwardKey[1])
def forwardDecrypt(self, encrypted):
retData = ""
for key in self.getGeneratedForwardKeys(False):
try:
retData = onionrcrypto.encryption.pub_key_decrypt(encrypted, privkey=key[1], encodedData=True)
retData = onionrcrypto.encryption.pub_key_decrypt(
encrypted, privkey=key[1], encodedData=True)
except nacl.exceptions.CryptoError:
retData = False
else:
break
else:
raise onionrexceptions.DecryptionError("Could not decrypt forward secrecy content")
raise onionrexceptions.DecryptionError(
"Could not decrypt forward secrecy content")
return retData
def _getLatestForwardKey(self):
# Get the latest forward secrecy key for a peer
key = ""
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
conn = sqlite3.connect(
dbfiles.user_id_info_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
# TODO: account for keys created at the same time (same epoch)
for row in c.execute("SELECT forwardKey, max(EXPIRE) FROM forwardKeys WHERE peerKey = ? ORDER BY expire DESC", (self.publicKey,)):
for row in c.execute(
"SELECT forwardKey, max(EXPIRE) FROM forwardKeys WHERE peerKey = ? ORDER BY expire DESC", # noqa
(self.publicKey,)):
key = (row[0], row[1])
break
@ -151,11 +163,14 @@ class OnionrUser:
return key
def _getForwardKeys(self):
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
conn = sqlite3.connect(
dbfiles.user_id_info_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
keyList = []
for row in c.execute("SELECT forwardKey, date FROM forwardKeys WHERE peerKey = ? ORDER BY expire DESC", (self.publicKey,)):
for row in c.execute(
"SELECT forwardKey, date FROM forwardKeys WHERE peerKey = ? ORDER BY expire DESC", # noqa
(self.publicKey,)):
keyList.append((row[0], row[1]))
conn.commit()
@ -166,7 +181,8 @@ class OnionrUser:
def generateForwardKey(self, expire=DEFAULT_KEY_EXPIRE):
# Generate a forward secrecy key for the peer
conn = sqlite3.connect(dbfiles.forward_keys_db, timeout=10)
conn = sqlite3.connect(
dbfiles.forward_keys_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
# Prepare the insert
time = epoch.get_epoch()
@ -184,14 +200,16 @@ class OnionrUser:
def getGeneratedForwardKeys(self, genNew=True):
# Fetch the keys we generated for the peer, that are still around
conn = sqlite3.connect(dbfiles.forward_keys_db, timeout=10)
conn = sqlite3.connect(
dbfiles.forward_keys_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
pubkey = self.publicKey
pubkey = bytesconverter.bytes_to_str(pubkey)
command = (pubkey,)
keyList = [] # list of tuples containing pub, private for peer
keyList = [] # list of tuples containing pub, private for peer
for result in c.execute("SELECT * FROM myForwardKeys WHERE peer = ?", command):
for result in c.execute(
"SELECT * FROM myForwardKeys WHERE peer = ?", command):
keyList.append((result[1], result[2]))
if len(keyList) == 0:
@ -201,12 +219,14 @@ class OnionrUser:
return list(keyList)
def addForwardKey(self, newKey, expire=DEFAULT_KEY_EXPIRE):
newKey = bytesconverter.bytes_to_str(unpaddedbase32.repad(bytesconverter.str_to_bytes(newKey)))
newKey = bytesconverter.bytes_to_str(
unpaddedbase32.repad(bytesconverter.str_to_bytes(newKey)))
if not stringvalidators.validate_pub_key(newKey):
# Do not add if something went wrong with the key
raise onionrexceptions.InvalidPubkey(newKey)
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
conn = sqlite3.connect(
dbfiles.user_id_info_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
# Get the time we're inserting the key at
@ -218,7 +238,8 @@ class OnionrUser:
return False
if entry[1] == timeInsert:
timeInsert += 1
time.sleep(1) # Sleep if our time is the same in order to prevent duplicate time records
# Sleep if our time is the same to prevent dupe time records
time.sleep(1)
# Add a forward secrecy key for the peer
# Prepare the insert

3
src/onionrutils/blockmetadata/hasblock.py
View File

@ -27,7 +27,8 @@ def has_block(hash: str) -> bool:
'''
Check for new block in the block meta db
'''
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
conn = sqlite3.connect(dbfiles.block_meta_db,
timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
if not stringvalidators.validate_hash(hash):
raise onionrexceptions.InvalidHexHash("Invalid hash")

2
static-data/default-plugins/pms/info.json
View File

@ -1,5 +1,5 @@
{
"name" : "pms",
"version" : "0.1.0",
"version" : "0.1.1",
"author" : "onionr"
}

2
static-data/default-plugins/pms/main.py
View File

@ -31,7 +31,7 @@ import notifier
locale.setlocale(locale.LC_ALL, '')
plugin_name = 'pms'
PLUGIN_VERSION = '0.1.0'
PLUGIN_VERSION = '0.1.1'
sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
import sentboxdb, mailapi, loadinbox # import after path insert

2
static-data/default-plugins/pms/sentboxdb.py
View File

@ -31,7 +31,7 @@ class SentBox:
return
def connect(self):
self.conn = sqlite3.connect(self.dbLocation)
self.conn = sqlite3.connect(self.dbLocation, timeout=30)
self.cursor = self.conn.cursor()
def close(self):