Added/corrected timeouts for sqlite3 database connections

Bumped mail plugin patch version for sqlite3 timeout change
Code formatting corrections

src/communicatorutils/housekeeping.py

@@ -17,6 +17,7 @@ import onionrstorage
 from onionrstorage import removeblock
 from onionrblocks import onionrblacklist
 from onionrblocks.storagecounter import StorageCounter
+from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -71,7 +72,8 @@ def clean_old_blocks(comm_inst):
 
 def clean_keys(comm_inst):
     """Delete expired forward secrecy keys"""
-    conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
+    conn = sqlite3.connect(dbfiles.user_id_info_db,
+                           timeout=DATABASE_LOCK_TIMEOUT)
     c = conn.cursor()
     time = epoch.get_epoch()
     deleteKeys = []

src/coredb/blockmetadb/__init__.py

@@ -1,7 +1,6 @@
-"""
+"""Onionr - Private P2P Communication.
-    Onionr - Private P2P Communication
 
-    This module works with information relating to blocks stored on the node
+Work with information relating to blocks stored on the node
 """
 import sqlite3
 
@@ -27,18 +26,18 @@ update_block_info = updateblockinfo.update_block_info
 add_to_block_DB = add.add_to_block_DB
 
 
-def get_block_list(dateRec = None, unsaved = False):
+def get_block_list(date_rec=None, unsaved=False):
-    """
+    """Get list of our blocks."""
-        Get list of our blocks
+    if date_rec is None:
-    """
+        date_rec = 0
-    if dateRec == None:
-        dateRec = 0
 
-    conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    conn = sqlite3.connect(
+        dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
     c = conn.cursor()
 
-    execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
+    execute = 'SELECT hash FROM hashes WHERE dateReceived' + \
-    args = (dateRec,)
+        ' >= ? ORDER BY dateReceived ASC;'
+    args = (date_rec,)
     rows = list()
     for row in c.execute(execute, args):
         for i in row:
@@ -48,11 +47,9 @@ def get_block_list(dateRec = None, unsaved = False):
 
 
 def get_block_date(blockHash):
-    """
+    """Return the date a block was received."""
-        Returns the date a block was received
+    conn = sqlite3.connect(
-    """
+        dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
-
-    conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
     c = conn.cursor()
 
     execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
@@ -65,11 +62,10 @@ def get_block_date(blockHash):
 
 
 def get_blocks_by_type(blockType, orderDate=True):
-    """
+    """Return a list of blocks by the type."""
-        Returns a list of blocks by the type
-    """
 
-    conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    conn = sqlite3.connect(
+        dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
     c = conn.cursor()
 
     if orderDate:

src/coredb/blockmetadb/expiredblocks.py

@@ -1,9 +1,12 @@
-'''
+"""Onionr - Private P2P Communication.
-    Onionr - Private P2P Communication
 
 Get a list of expired blocks still stored
-'''
+"""
-'''
+import sqlite3
+from onionrutils import epoch
+from .. import dbfiles
+from etc import onionrvalues
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,15 +19,13 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
-import sqlite3
+
-from onionrutils import epoch
-from .. import dbfiles
-from etc import onionrvalues
 
 def get_expired_blocks():
-    '''Returns a list of expired blocks'''
+    """Return a list of expired blocks"""
-    conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    conn = sqlite3.connect(
+        dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
     c = conn.cursor()
     date = int(epoch.get_epoch())
 

src/coredb/keydb/transportinfo.py

@@ -1,9 +1,11 @@
-'''
+"""Onionr - Private P2P Communication.
-    Onionr - Private P2P Communication
 
 get or set transport address meta information
-'''
+"""
-'''
+import sqlite3
+from .. import dbfiles
+from etc import onionrvalues
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,13 +18,11 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
-import sqlite3
+
-from .. import dbfiles
-from etc import onionrvalues
 
 def get_address_info(address, info):
-    '''
+    """
     Get info about an address from its database entry
 
     address text, 0
@@ -35,9 +35,10 @@ def get_address_info(address, info):
     lastConnect 7
     trust       8
     introduced  9
-    '''
+    """
 
-    conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    conn = sqlite3.connect(
+        dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
     c = conn.cursor()
 
     command = (address,)
@@ -57,12 +58,12 @@ def get_address_info(address, info):
 
     return retVal
 
-def set_address_info(address, key, data):
-    '''
-        Update an address for a key
-    '''
 
-    conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+def set_address_info(address, key, data):
+    """Update an address for a key."""
+
+    conn = sqlite3.connect(
+        dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
     c = conn.cursor()
 
     command = (data, address)

src/onionrblocks/onionrblacklist.py

@@ -9,6 +9,7 @@ from onionrplugins.onionrevents import event
 import onionrcrypto
 from onionrutils import epoch, bytesconverter
 from coredb import dbfiles
+from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -49,7 +50,7 @@ class OnionrBlackList:
         return retData
 
     def _dbExecute(self, toExec, params=()):
-        conn = sqlite3.connect(self.blacklistDB)
+        conn = sqlite3.connect(self.blacklistDB, timeout=DATABASE_LOCK_TIMEOUT)
         c = conn.cursor()
         retData = c.execute(toExec, params)
         conn.commit()

src/onionrstorage/__init__.py

@@ -9,10 +9,11 @@ import os
 from onionrutils import bytesconverter
 from onionrutils import stringvalidators
 from coredb import dbfiles
-import filepaths
+from filepaths import block_data_location
 import onionrexceptions
 from onionrcrypto import hashers
 from . import setdata
+from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -34,73 +35,80 @@ DB_ENTRY_SIZE_LIMIT = 10000  # Will be a config option
 set_data = setdata.set_data
 
 
-def _dbInsert(blockHash, data):
+def _dbInsert(block_hash, data):
-    conn = sqlite3.connect(dbfiles.block_data_db, timeout=10)
+    conn = sqlite3.connect(dbfiles.block_data_db,
+                           timeout=DATABASE_LOCK_TIMEOUT)
     c = conn.cursor()
-    data = (blockHash, data)
+    data = (block_hash, data)
     c.execute('INSERT INTO blockData (hash, data) VALUES(?, ?);', data)
     conn.commit()
     conn.close()
 
 
-def _dbFetch(blockHash):
+def _dbFetch(block_hash):
-    conn = sqlite3.connect(dbfiles.block_data_db, timeout=10)
+    conn = sqlite3.connect(dbfiles.block_data_db,
+                           timeout=DATABASE_LOCK_TIMEOUT)
     c = conn.cursor()
-    for i in c.execute('SELECT data from blockData where hash = ?', (blockHash,)):
+    for i in c.execute(
+            'SELECT data from blockData where hash = ?', (block_hash,)):
         return i[0]
     conn.commit()
     conn.close()
     return None
 
 
-def deleteBlock(blockHash):
+def deleteBlock(block_hash):
-    # You should call removeblock.remove_block if you automatically want to remove storage byte count
+    # Call removeblock.remove_block to automatically want to remove storage byte count
-    if os.path.exists('%s/%s.dat' % (filepaths.block_data_location, blockHash)):
+    if os.path.exists(f'{block_data_location}/{block_hash}.dat'):
-        os.remove('%s/%s.dat' % (filepaths.block_data_location, blockHash))
+        os.remove(f'{block_data_location}/{block_hash}.dat')
         return True
-    conn = sqlite3.connect(dbfiles.block_data_db, timeout=10)
+    conn = sqlite3.connect(dbfiles.block_data_db,
+                           timeout=DATABASE_LOCK_TIMEOUT)
     c = conn.cursor()
-    data = (blockHash,)
+    data = (block_hash,)
     c.execute('DELETE FROM blockData where hash = ?', data)
     conn.commit()
     conn.close()
     return True
 
 
-def store(data, blockHash=''):
+def store(data, block_hash=''):
-    if not stringvalidators.validate_hash(blockHash): raise ValueError
+    if not stringvalidators.validate_hash(block_hash):
+        raise ValueError
     ourHash = hashers.sha3_hash(data)
-    if blockHash != '':
+    if block_hash != '':
-        if not ourHash == blockHash:
+        if not ourHash == block_hash:
             raise ValueError('Hash specified does not meet internal hash check')
     else:
-        blockHash = ourHash
+        block_hash = ourHash
 
     if DB_ENTRY_SIZE_LIMIT >= sys.getsizeof(data):
-        _dbInsert(blockHash, data)
+        _dbInsert(block_hash, data)
     else:
-        with open('%s/%s.dat' % (filepaths.block_data_location, blockHash), 'wb') as blockFile:
+        with open(
-            blockFile.write(data)
+                f'{block_data_location}/{block_hash}.dat', 'wb') as blck_file:
+            blck_file.write(data)
 
 
 def getData(bHash):
 
-    if not stringvalidators.validate_hash(bHash): raise ValueError
+    if not stringvalidators.validate_hash(bHash):
+        raise ValueError
 
     bHash = bytesconverter.bytes_to_str(bHash)
     bHash = bHash.strip()
     # First check DB for data entry by hash
     # if no entry, check disk
     # If no entry in either, raise an exception
-    retData = None
+    ret_data = None
-    fileLocation = '%s/%s.dat' % (filepaths.block_data_location, bHash)
+    fileLocation = '%s/%s.dat' % (block_data_location, bHash)
     not_found_msg = "Block data not found for: "
     if os.path.exists(fileLocation):
         with open(fileLocation, 'rb') as block:
-            retData = block.read()
+            ret_data = block.read()
     else:
-        retData = _dbFetch(bHash)
+        ret_data = _dbFetch(bHash)
 
-        if retData is None:
+        if ret_data is None:
             raise onionrexceptions.NoDataAvailable(not_found_msg + str(bHash))
-    return retData
+    return ret_data

src/onionrstorage/removeblock.py

@@ -10,6 +10,7 @@ import onionrstorage
 from onionrutils import stringvalidators
 from coredb import dbfiles
 from onionrblocks import storagecounter
+from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -33,7 +34,8 @@ def remove_block(block):
     **You may want blacklist.addToDB(blockHash)
     """
     if stringvalidators.validate_hash(block):
-        conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
+        conn = sqlite3.connect(
+            dbfiles.block_meta_db, timeout=DATABASE_LOCK_TIMEOUT)
         c = conn.cursor()
         t = (block,)
         c.execute('Delete from hashes where hash=?;', t)

src/onionrstorage/setdata.py

@@ -12,6 +12,7 @@ import filepaths
 from onionrblocks import storagecounter
 from coredb import dbfiles
 from onionrutils import blockmetadata, bytesconverter
+from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -50,7 +51,8 @@ def set_data(data) -> str:
     except onionrexceptions.NoDataAvailable:
         if storage_counter.add_bytes(dataSize) is not False:
             onionrstorage.store(data, blockHash=dataHash)
-            conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
+            conn = sqlite3.connect(
+                dbfiles.block_meta_db, timeout=DATABASE_LOCK_TIMEOUT)
             c = conn.cursor()
             c.execute(
                 "UPDATE hashes SET dataSaved=1 WHERE hash = ?;",

src/onionrusers/onionrusers.py

@@ -14,6 +14,7 @@ import nacl.exceptions
 from coredb import keydb, dbfiles
 import onionrcrypto
 from onionrcrypto import getourkeypair
+from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -32,7 +33,8 @@ from onionrcrypto import getourkeypair
 
 def deleteExpiredKeys():
     # Fetch the keys we generated for the peer, that are still around
-    conn = sqlite3.connect(dbfiles.forward_keys_db, timeout=10)
+    conn = sqlite3.connect(
+        dbfiles.forward_keys_db, timeout=DATABASE_LOCK_TIMEOUT)
     c = conn.cursor()
 
     curTime = epoch.get_epoch()
@@ -44,7 +46,8 @@ def deleteExpiredKeys():
 
 
 def deleteTheirExpiredKeys(pubkey):
-    conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
+    conn = sqlite3.connect(
+        dbfiles.user_id_info_db, timeout=DATABASE_LOCK_TIMEOUT)
     c = conn.cursor()
 
     # Prepare the insert
@@ -68,7 +71,8 @@ class OnionrUser:
         Takes a base32 encoded ed25519 public key, and a bool saveUser
         saveUser determines if we should add a user to our peer database or not.
         """
-        publicKey = unpaddedbase32.repad(bytesconverter.str_to_bytes(publicKey)).decode()
+        publicKey = unpaddedbase32.repad(
+            bytesconverter.str_to_bytes(publicKey)).decode()
 
         self.trust = 0
         self.publicKey = publicKey
@@ -76,7 +80,7 @@ class OnionrUser:
         if saveUser and not publicKey == getourkeypair.get_keypair():
             try:
                 keydb.addkeys.add_peer(publicKey)
-            except (AssertionError, ValueError) as e:
+            except (AssertionError, ValueError) as _:
                 pass
 
         self.trust = keydb.userinfo.get_user_info(self.publicKey, 'trust')
@@ -102,11 +106,13 @@ class OnionrUser:
         return retData
 
     def encrypt(self, data):
-        encrypted = onionrcrypto.encryption.pub_key_encrypt(data, self.publicKey, encodedData=True)
+        encrypted = onionrcrypto.encryption.pub_key_encrypt(
+            data, self.publicKey, encodedData=True)
         return encrypted
 
     def decrypt(self, data):
-        decrypted = onionrcrypto.encryption.pub_key_decrypt(data, self.publicKey, encodedData=True)
+        decrypted = onionrcrypto.encryption.pub_key_decrypt(
+            data, self.publicKey, encodedData=True)
         return decrypted
 
     def forwardEncrypt(self, data):
@@ -115,33 +121,39 @@ class OnionrUser:
         retData = ''
         forwardKey = self._getLatestForwardKey()
         if stringvalidators.validate_pub_key(forwardKey[0]):
-            retData = onionrcrypto.encryption.pub_key_encrypt(data, forwardKey[0], encodedData=True)
+            retData = onionrcrypto.encryption.pub_key_encrypt(
+                data, forwardKey[0], encodedData=True)
         else:
-            raise onionrexceptions.InvalidPubkey("No valid forward secrecy key available for this user")
+            raise onionrexceptions.InvalidPubkey(
-        #self.generateForwardKey()
+                "No valid forward secrecy key available for this user")
         return (retData, forwardKey[0], forwardKey[1])
 
     def forwardDecrypt(self, encrypted):
         retData = ""
         for key in self.getGeneratedForwardKeys(False):
             try:
-                retData = onionrcrypto.encryption.pub_key_decrypt(encrypted, privkey=key[1], encodedData=True)
+                retData = onionrcrypto.encryption.pub_key_decrypt(
+                    encrypted, privkey=key[1], encodedData=True)
             except nacl.exceptions.CryptoError:
                 retData = False
             else:
                 break
         else:
-            raise onionrexceptions.DecryptionError("Could not decrypt forward secrecy content")
+            raise onionrexceptions.DecryptionError(
+                "Could not decrypt forward secrecy content")
         return retData
 
     def _getLatestForwardKey(self):
         # Get the latest forward secrecy key for a peer
         key = ""
-        conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
+        conn = sqlite3.connect(
+            dbfiles.user_id_info_db, timeout=DATABASE_LOCK_TIMEOUT)
         c = conn.cursor()
 
         # TODO: account for keys created at the same time (same epoch)
-        for row in c.execute("SELECT forwardKey, max(EXPIRE) FROM forwardKeys WHERE peerKey = ? ORDER BY expire DESC", (self.publicKey,)):
+        for row in c.execute(
+                "SELECT forwardKey, max(EXPIRE) FROM forwardKeys WHERE peerKey = ? ORDER BY expire DESC",  # noqa
+                (self.publicKey,)):
             key = (row[0], row[1])
             break
 
@@ -151,11 +163,14 @@ class OnionrUser:
         return key
 
     def _getForwardKeys(self):
-        conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
+        conn = sqlite3.connect(
+            dbfiles.user_id_info_db, timeout=DATABASE_LOCK_TIMEOUT)
         c = conn.cursor()
         keyList = []
 
-        for row in c.execute("SELECT forwardKey, date FROM forwardKeys WHERE peerKey = ? ORDER BY expire DESC", (self.publicKey,)):
+        for row in c.execute(
+                "SELECT forwardKey, date FROM forwardKeys WHERE peerKey = ? ORDER BY expire DESC",  # noqa
+                (self.publicKey,)):
             keyList.append((row[0], row[1]))
 
         conn.commit()
@@ -166,7 +181,8 @@ class OnionrUser:
     def generateForwardKey(self, expire=DEFAULT_KEY_EXPIRE):
 
         # Generate a forward secrecy key for the peer
-        conn = sqlite3.connect(dbfiles.forward_keys_db, timeout=10)
+        conn = sqlite3.connect(
+            dbfiles.forward_keys_db, timeout=DATABASE_LOCK_TIMEOUT)
         c = conn.cursor()
         # Prepare the insert
         time = epoch.get_epoch()
@@ -184,14 +200,16 @@ class OnionrUser:
 
     def getGeneratedForwardKeys(self, genNew=True):
         # Fetch the keys we generated for the peer, that are still around
-        conn = sqlite3.connect(dbfiles.forward_keys_db, timeout=10)
+        conn = sqlite3.connect(
+            dbfiles.forward_keys_db, timeout=DATABASE_LOCK_TIMEOUT)
         c = conn.cursor()
         pubkey = self.publicKey
         pubkey = bytesconverter.bytes_to_str(pubkey)
         command = (pubkey,)
         keyList = []  # list of tuples containing pub, private for peer
 
-        for result in c.execute("SELECT * FROM myForwardKeys WHERE peer = ?", command):
+        for result in c.execute(
+                "SELECT * FROM myForwardKeys WHERE peer = ?", command):
             keyList.append((result[1], result[2]))
 
         if len(keyList) == 0:
@@ -201,12 +219,14 @@ class OnionrUser:
         return list(keyList)
 
     def addForwardKey(self, newKey, expire=DEFAULT_KEY_EXPIRE):
-        newKey = bytesconverter.bytes_to_str(unpaddedbase32.repad(bytesconverter.str_to_bytes(newKey)))
+        newKey = bytesconverter.bytes_to_str(
+            unpaddedbase32.repad(bytesconverter.str_to_bytes(newKey)))
         if not stringvalidators.validate_pub_key(newKey):
             # Do not add if something went wrong with the key
             raise onionrexceptions.InvalidPubkey(newKey)
 
-        conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
+        conn = sqlite3.connect(
+            dbfiles.user_id_info_db, timeout=DATABASE_LOCK_TIMEOUT)
         c = conn.cursor()
 
         # Get the time we're inserting the key at
@@ -218,7 +238,8 @@ class OnionrUser:
                 return False
             if entry[1] == timeInsert:
                 timeInsert += 1
-                time.sleep(1) # Sleep if our time is the same in order to prevent duplicate time records
+                # Sleep if our time is the same to prevent dupe time records
+                time.sleep(1)
 
         # Add a forward secrecy key for the peer
         # Prepare the insert

src/onionrutils/blockmetadata/hasblock.py

@@ -27,7 +27,8 @@ def has_block(hash: str) -> bool:
     '''
         Check for new block in the block meta db
     '''
-    conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    conn = sqlite3.connect(dbfiles.block_meta_db,
+        timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
     c = conn.cursor()
     if not stringvalidators.validate_hash(hash):
         raise onionrexceptions.InvalidHexHash("Invalid hash")

static-data/default-plugins/pms/info.json

@@ -1,5 +1,5 @@
 {
     "name" : "pms",
-    "version" : "0.1.0",
+    "version" : "0.1.1",
     "author" : "onionr"
 }

static-data/default-plugins/pms/main.py

@@ -31,7 +31,7 @@ import notifier
 locale.setlocale(locale.LC_ALL, '')
 
 plugin_name = 'pms'
-PLUGIN_VERSION = '0.1.0'
+PLUGIN_VERSION = '0.1.1'
 
 sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
 import sentboxdb, mailapi, loadinbox # import after path insert

static-data/default-plugins/pms/sentboxdb.py

@@ -31,7 +31,7 @@ class SentBox:
         return
 
     def connect(self):
-        self.conn = sqlite3.connect(self.dbLocation)
+        self.conn = sqlite3.connect(self.dbLocation, timeout=30)
         self.cursor = self.conn.cursor()
 
     def close(self):