kev/Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

Added/corrected timeouts for sqlite3 database connections

Browse Source 
Bumped mail plugin patch version for sqlite3 timeout change
Code formatting corrections
This commit is contained in:
Kevin Froman 2020-08-16 19:52:50 -05:00
parent 2a7c933321
commit 0b2658374b
13 changed files with 158 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/communicatorutils/housekeeping.py
View File

@ -17,6 +17,7 @@ import onionrstorage
from onionrstorage import removeblock
from onionrblocks import onionrblacklist
from onionrblocks.storagecounter import StorageCounter
from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -71,7 +72,8 @@ def clean_old_blocks(comm_inst):
def clean_keys(comm_inst):
"""Delete expired forward secrecy keys"""
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
conn = sqlite3.connect(dbfiles.user_id_info_db,
timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
time = epoch.get_epoch()
deleteKeys = []

38
src/coredb/blockmetadb/__init__.py
View File

@ -1,7 +1,6 @@
"""
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
This module works with information relating to blocks stored on the node
Work with information relating to blocks stored on the node
"""
import sqlite3
@ -27,18 +26,18 @@ update_block_info = updateblockinfo.update_block_info
add_to_block_DB = add.add_to_block_DB
def get_block_list(dateRec = None, unsaved = False):
"""
Get list of our blocks
"""
if dateRec == None:
dateRec = 0
def get_block_list(date_rec=None, unsaved=False):
"""Get list of our blocks."""
if date_rec is None:
date_rec = 0
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
conn = sqlite3.connect(
dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
args = (dateRec,)
execute = 'SELECT hash FROM hashes WHERE dateReceived' + \
' >= ? ORDER BY dateReceived ASC;'
args = (date_rec,)
rows = list()
for row in c.execute(execute, args):
for i in row:
@ -48,11 +47,9 @@ def get_block_list(dateRec = None, unsaved = False):
def get_block_date(blockHash):
"""
Returns the date a block was received
"""
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
"""Return the date a block was received."""
conn = sqlite3.connect(
dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
@ -65,11 +62,10 @@ def get_block_date(blockHash):
def get_blocks_by_type(blockType, orderDate=True):
"""
Returns a list of blocks by the type
"""
"""Return a list of blocks by the type."""
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
conn = sqlite3.connect(
dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
if orderDate:

23
src/coredb/blockmetadb/expiredblocks.py
View File

@ -1,9 +1,12 @@
'''
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
Get a list of expired blocks still stored
'''
'''
"""
import sqlite3
from onionrutils import epoch
from .. import dbfiles
from etc import onionrvalues
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,15 +19,13 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3
from onionrutils import epoch
from .. import dbfiles
from etc import onionrvalues
"""
def get_expired_blocks():
'''Returns a list of expired blocks'''
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
"""Return a list of expired blocks"""
conn = sqlite3.connect(
dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
date = int(epoch.get_epoch())

33
src/coredb/keydb/transportinfo.py
View File

@ -1,9 +1,11 @@
'''
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
get or set transport address meta information
'''
'''
"""
import sqlite3
from .. import dbfiles
from etc import onionrvalues
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,13 +18,11 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3
from .. import dbfiles
from etc import onionrvalues
"""
def get_address_info(address, info):
'''
"""
Get info about an address from its database entry
address text, 0
@ -35,9 +35,10 @@ def get_address_info(address, info):
lastConnect 7
trust 8
introduced 9
'''
"""
conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
conn = sqlite3.connect(
dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
command = (address,)
@ -57,12 +58,12 @@ def get_address_info(address, info):
return retVal
def set_address_info(address, key, data):
'''
Update an address for a key
'''
conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
def set_address_info(address, key, data):
"""Update an address for a key."""
conn = sqlite3.connect(
dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
command = (data, address)

3
src/onionrblocks/onionrblacklist.py
View File

@ -9,6 +9,7 @@ from onionrplugins.onionrevents import event
import onionrcrypto
from onionrutils import epoch, bytesconverter
from coredb import dbfiles
from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -49,7 +50,7 @@ class OnionrBlackList:
return retData
def _dbExecute(self, toExec, params=()):
conn = sqlite3.connect(self.blacklistDB)
conn = sqlite3.connect(self.blacklistDB, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
retData = c.execute(toExec, params)
conn.commit()

64
src/onionrstorage/__init__.py
View File

@ -9,10 +9,11 @@ import os
from onionrutils import bytesconverter
from onionrutils import stringvalidators
from coredb import dbfiles
import filepaths
from filepaths import block_data_location
import onionrexceptions
from onionrcrypto import hashers
from . import setdata
from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -34,73 +35,80 @@ DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option
set_data = setdata.set_data
def _dbInsert(blockHash, data):
conn = sqlite3.connect(dbfiles.block_data_db, timeout=10)
def _dbInsert(block_hash, data):
conn = sqlite3.connect(dbfiles.block_data_db,
timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
data = (blockHash, data)
data = (block_hash, data)
c.execute('INSERT INTO blockData (hash, data) VALUES(?, ?);', data)
conn.commit()
conn.close()
def _dbFetch(blockHash):
conn = sqlite3.connect(dbfiles.block_data_db, timeout=10)
def _dbFetch(block_hash):
conn = sqlite3.connect(dbfiles.block_data_db,
timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
for i in c.execute('SELECT data from blockData where hash = ?', (blockHash,)):
for i in c.execute(
'SELECT data from blockData where hash = ?', (block_hash,)):
return i[0]
conn.commit()
conn.close()
return None
def deleteBlock(blockHash):
# You should call removeblock.remove_block if you automatically want to remove storage byte count
if os.path.exists('%s/%s.dat' % (filepaths.block_data_location, blockHash)):
os.remove('%s/%s.dat' % (filepaths.block_data_location, blockHash))
def deleteBlock(block_hash):
# Call removeblock.remove_block to automatically want to remove storage byte count
if os.path.exists(f'{block_data_location}/{block_hash}.dat'):
os.remove(f'{block_data_location}/{block_hash}.dat')
return True
conn = sqlite3.connect(dbfiles.block_data_db, timeout=10)
conn = sqlite3.connect(dbfiles.block_data_db,
timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
data = (blockHash,)
data = (block_hash,)
c.execute('DELETE FROM blockData where hash = ?', data)
conn.commit()
conn.close()
return True
def store(data, blockHash=''):
if not stringvalidators.validate_hash(blockHash): raise ValueError
def store(data, block_hash=''):
if not stringvalidators.validate_hash(block_hash):
raise ValueError
ourHash = hashers.sha3_hash(data)
if blockHash != '':
if not ourHash == blockHash:
if block_hash != '':
if not ourHash == block_hash:
raise ValueError('Hash specified does not meet internal hash check')
else:
blockHash = ourHash
block_hash = ourHash
if DB_ENTRY_SIZE_LIMIT >= sys.getsizeof(data):
_dbInsert(blockHash, data)
_dbInsert(block_hash, data)
else:
with open('%s/%s.dat' % (filepaths.block_data_location, blockHash), 'wb') as blockFile:
blockFile.write(data)
with open(
f'{block_data_location}/{block_hash}.dat', 'wb') as blck_file:
blck_file.write(data)
def getData(bHash):
if not stringvalidators.validate_hash(bHash): raise ValueError
if not stringvalidators.validate_hash(bHash):
raise ValueError
bHash = bytesconverter.bytes_to_str(bHash)
bHash = bHash.strip()
# First check DB for data entry by hash
# if no entry, check disk
# If no entry in either, raise an exception
retData = None
fileLocation = '%s/%s.dat' % (filepaths.block_data_location, bHash)
ret_data = None
fileLocation = '%s/%s.dat' % (block_data_location, bHash)
not_found_msg = "Block data not found for: "
if os.path.exists(fileLocation):
with open(fileLocation, 'rb') as block:
retData = block.read()
ret_data = block.read()
else:
retData = _dbFetch(bHash)
ret_data = _dbFetch(bHash)
if retData is None:
if ret_data is None:
raise onionrexceptions.NoDataAvailable(not_found_msg + str(bHash))
return retData
return ret_data

4
src/onionrstorage/removeblock.py
View File

@ -10,6 +10,7 @@ import onionrstorage
from onionrutils import stringvalidators
from coredb import dbfiles
from onionrblocks import storagecounter
from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -33,7 +34,8 @@ def remove_block(block):
**You may want blacklist.addToDB(blockHash)
"""
if stringvalidators.validate_hash(block):
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
conn = sqlite3.connect(
dbfiles.block_meta_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
t = (block,)
c.execute('Delete from hashes where hash=?;', t)

4
src/onionrstorage/setdata.py
View File

@ -12,6 +12,7 @@ import filepaths
from onionrblocks import storagecounter
from coredb import dbfiles
from onionrutils import blockmetadata, bytesconverter
from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -50,7 +51,8 @@ def set_data(data) -> str:
except onionrexceptions.NoDataAvailable:
if storage_counter.add_bytes(dataSize) is not False:
onionrstorage.store(data, blockHash=dataHash)
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
conn = sqlite3.connect(
dbfiles.block_meta_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
c.execute(
"UPDATE hashes SET dataSaved=1 WHERE hash = ?;",

63
src/onionrusers/onionrusers.py
View File

@ -14,6 +14,7 @@ import nacl.exceptions
from coredb import keydb, dbfiles
import onionrcrypto
from onionrcrypto import getourkeypair
from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -32,7 +33,8 @@ from onionrcrypto import getourkeypair
def deleteExpiredKeys():
# Fetch the keys we generated for the peer, that are still around
conn = sqlite3.connect(dbfiles.forward_keys_db, timeout=10)
conn = sqlite3.connect(
dbfiles.forward_keys_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
curTime = epoch.get_epoch()
@ -44,7 +46,8 @@ def deleteExpiredKeys():
def deleteTheirExpiredKeys(pubkey):
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
conn = sqlite3.connect(
dbfiles.user_id_info_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
# Prepare the insert
@ -68,7 +71,8 @@ class OnionrUser:
Takes a base32 encoded ed25519 public key, and a bool saveUser
saveUser determines if we should add a user to our peer database or not.
"""
publicKey = unpaddedbase32.repad(bytesconverter.str_to_bytes(publicKey)).decode()
publicKey = unpaddedbase32.repad(
bytesconverter.str_to_bytes(publicKey)).decode()
self.trust = 0
self.publicKey = publicKey
@ -76,7 +80,7 @@ class OnionrUser:
if saveUser and not publicKey == getourkeypair.get_keypair():
try:
keydb.addkeys.add_peer(publicKey)
except (AssertionError, ValueError) as e:
except (AssertionError, ValueError) as _:
pass
self.trust = keydb.userinfo.get_user_info(self.publicKey, 'trust')
@ -102,11 +106,13 @@ class OnionrUser:
return retData
def encrypt(self, data):
encrypted = onionrcrypto.encryption.pub_key_encrypt(data, self.publicKey, encodedData=True)
encrypted = onionrcrypto.encryption.pub_key_encrypt(
data, self.publicKey, encodedData=True)
return encrypted
def decrypt(self, data):
decrypted = onionrcrypto.encryption.pub_key_decrypt(data, self.publicKey, encodedData=True)
decrypted = onionrcrypto.encryption.pub_key_decrypt(
data, self.publicKey, encodedData=True)
return decrypted
def forwardEncrypt(self, data):
@ -115,33 +121,39 @@ class OnionrUser:
retData = ''
forwardKey = self._getLatestForwardKey()
if stringvalidators.validate_pub_key(forwardKey[0]):
retData = onionrcrypto.encryption.pub_key_encrypt(data, forwardKey[0], encodedData=True)
retData = onionrcrypto.encryption.pub_key_encrypt(
data, forwardKey[0], encodedData=True)
else:
raise onionrexceptions.InvalidPubkey("No valid forward secrecy key available for this user")
#self.generateForwardKey()
raise onionrexceptions.InvalidPubkey(
"No valid forward secrecy key available for this user")
return (retData, forwardKey[0], forwardKey[1])
def forwardDecrypt(self, encrypted):
retData = ""
for key in self.getGeneratedForwardKeys(False):
try:
retData = onionrcrypto.encryption.pub_key_decrypt(encrypted, privkey=key[1], encodedData=True)
retData = onionrcrypto.encryption.pub_key_decrypt(
encrypted, privkey=key[1], encodedData=True)
except nacl.exceptions.CryptoError:
retData = False
else:
break
else:
raise onionrexceptions.DecryptionError("Could not decrypt forward secrecy content")
raise onionrexceptions.DecryptionError(
"Could not decrypt forward secrecy content")
return retData
def _getLatestForwardKey(self):
# Get the latest forward secrecy key for a peer
key = ""
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
conn = sqlite3.connect(
dbfiles.user_id_info_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
# TODO: account for keys created at the same time (same epoch)
for row in c.execute("SELECT forwardKey, max(EXPIRE) FROM forwardKeys WHERE peerKey = ? ORDER BY expire DESC", (self.publicKey,)):
for row in c.execute(
"SELECT forwardKey, max(EXPIRE) FROM forwardKeys WHERE peerKey = ? ORDER BY expire DESC", # noqa
(self.publicKey,)):
key = (row[0], row[1])
break
@ -151,11 +163,14 @@ class OnionrUser:
return key
def _getForwardKeys(self):
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
conn = sqlite3.connect(
dbfiles.user_id_info_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
keyList = []
for row in c.execute("SELECT forwardKey, date FROM forwardKeys WHERE peerKey = ? ORDER BY expire DESC", (self.publicKey,)):
for row in c.execute(
"SELECT forwardKey, date FROM forwardKeys WHERE peerKey = ? ORDER BY expire DESC", # noqa
(self.publicKey,)):
keyList.append((row[0], row[1]))
conn.commit()
@ -166,7 +181,8 @@ class OnionrUser:
def generateForwardKey(self, expire=DEFAULT_KEY_EXPIRE):
# Generate a forward secrecy key for the peer
conn = sqlite3.connect(dbfiles.forward_keys_db, timeout=10)
conn = sqlite3.connect(
dbfiles.forward_keys_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
# Prepare the insert
time = epoch.get_epoch()
@ -184,14 +200,16 @@ class OnionrUser:
def getGeneratedForwardKeys(self, genNew=True):
# Fetch the keys we generated for the peer, that are still around
conn = sqlite3.connect(dbfiles.forward_keys_db, timeout=10)
conn = sqlite3.connect(
dbfiles.forward_keys_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
pubkey = self.publicKey
pubkey = bytesconverter.bytes_to_str(pubkey)
command = (pubkey,)
keyList = [] # list of tuples containing pub, private for peer
for result in c.execute("SELECT * FROM myForwardKeys WHERE peer = ?", command):
for result in c.execute(
"SELECT * FROM myForwardKeys WHERE peer = ?", command):
keyList.append((result[1], result[2]))
if len(keyList) == 0:
@ -201,12 +219,14 @@ class OnionrUser:
return list(keyList)
def addForwardKey(self, newKey, expire=DEFAULT_KEY_EXPIRE):
newKey = bytesconverter.bytes_to_str(unpaddedbase32.repad(bytesconverter.str_to_bytes(newKey)))
newKey = bytesconverter.bytes_to_str(
unpaddedbase32.repad(bytesconverter.str_to_bytes(newKey)))
if not stringvalidators.validate_pub_key(newKey):
# Do not add if something went wrong with the key
raise onionrexceptions.InvalidPubkey(newKey)
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
conn = sqlite3.connect(
dbfiles.user_id_info_db, timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
# Get the time we're inserting the key at
@ -218,7 +238,8 @@ class OnionrUser:
return False
if entry[1] == timeInsert:
timeInsert += 1
time.sleep(1) # Sleep if our time is the same in order to prevent duplicate time records
# Sleep if our time is the same to prevent dupe time records
time.sleep(1)
# Add a forward secrecy key for the peer
# Prepare the insert

3
src/onionrutils/blockmetadata/hasblock.py
View File

@ -27,7 +27,8 @@ def has_block(hash: str) -> bool:
'''
Check for new block in the block meta db
'''
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
conn = sqlite3.connect(dbfiles.block_meta_db,
timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
if not stringvalidators.validate_hash(hash):
raise onionrexceptions.InvalidHexHash("Invalid hash")

2
static-data/default-plugins/pms/info.json
View File

@ -1,5 +1,5 @@
{
"name" : "pms",
"version" : "0.1.0",
"version" : "0.1.1",
"author" : "onionr"
}

2
static-data/default-plugins/pms/main.py
View File

@ -31,7 +31,7 @@ import notifier
locale.setlocale(locale.LC_ALL, '')
plugin_name = 'pms'
PLUGIN_VERSION = '0.1.0'
PLUGIN_VERSION = '0.1.1'
sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
import sentboxdb, mailapi, loadinbox # import after path insert

2
static-data/default-plugins/pms/sentboxdb.py
View File

@ -31,7 +31,7 @@ class SentBox:
return
def connect(self):
self.conn = sqlite3.connect(self.dbLocation)
self.conn = sqlite3.connect(self.dbLocation, timeout=30)
self.cursor = self.conn.cursor()
def close(self):