From bf8a9c4f27aca9c4c98162222cc81b3eec656e23 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Wed, 17 Jul 2019 11:58:40 -0500 Subject: [PATCH] refactoring core databases to not use core anymore --- onionr/blockimporter.py | 3 +- .../downloadblocks/__init__.py | 2 +- onionr/communicatorutils/housekeeping.py | 2 +- onionr/core.py | 33 ++----------------- onionr/coredb/blockmetadb/add.py | 13 ++++---- onionr/coredb/blockmetadb/expiredblocks.py | 5 +-- onionr/coredb/blockmetadb/updateblockinfo.py | 20 +++++++++-- onionr/onionrutils/blockmetadata.py | 11 ++++--- onionr/onionrutils/importnewblocks.py | 2 +- 9 files changed, 39 insertions(+), 52 deletions(-) diff --git a/onionr/blockimporter.py b/onionr/blockimporter.py index 067ec045..ffd9ba63 100755 --- a/onionr/blockimporter.py +++ b/onionr/blockimporter.py @@ -19,6 +19,7 @@ ''' import core, onionrexceptions, logger from onionrutils import validatemetadata, blockmetadata +from coredb import blockmetadb def importBlockFromData(content, coreInst): retData = False @@ -45,7 +46,7 @@ def importBlockFromData(content, coreInst): except onionrexceptions.DiskAllocationReached: pass else: - coreInst.addToBlockDB(blockHash, dataSaved=True) + blockmetadb.add_to_block_DB(blockHash, dataSaved=True) blockmetadata.process_block_metadata(coreInst, blockHash) # caches block metadata values to block database retData = True return retData \ No newline at end of file diff --git a/onionr/communicatorutils/downloadblocks/__init__.py b/onionr/communicatorutils/downloadblocks/__init__.py index 46732728..4ecb28a6 100755 --- a/onionr/communicatorutils/downloadblocks/__init__.py +++ b/onionr/communicatorutils/downloadblocks/__init__.py @@ -82,7 +82,7 @@ def download_blocks_from_communicator(comm_inst): logger.error('Reached disk allocation allowance, cannot save block %s.' % (blockHash,)) removeFromQueue = False else: - comm_inst._core.addToBlockDB(blockHash, dataSaved=True) + blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db blockmetadata.process_block_metadata(comm_inst._core, blockHash) # caches block metadata values to block database else: logger.warn('POW failed for block %s.' % (blockHash,)) diff --git a/onionr/communicatorutils/housekeeping.py b/onionr/communicatorutils/housekeeping.py index 9c041f9a..37772230 100755 --- a/onionr/communicatorutils/housekeeping.py +++ b/onionr/communicatorutils/housekeeping.py @@ -26,7 +26,7 @@ def clean_old_blocks(comm_inst): '''Delete old blocks if our disk allocation is full/near full, and also expired blocks''' # Delete expired blocks - for bHash in comm_inst._core.getExpiredBlocks(): + for bHash in blockmetadb.get_expired_blocks(): comm_inst._core._blacklist.addToDB(bHash) comm_inst._core.removeBlock(bHash) logger.info('Deleted block: %s' % (bHash,)) diff --git a/onionr/core.py b/onionr/core.py index b48505b2..1f83b8f4 100755 --- a/onionr/core.py +++ b/onionr/core.py @@ -164,14 +164,6 @@ class Core: ''' self.dbCreate.createBlockDB() - def addToBlockDB(self, newHash, selfInsert=False, dataSaved=False): - ''' - Add a hash value to the block db - - Should be in hex format! - ''' - coredb.blockmetadb.add.add_to_block_DB(self, newHash, selfInsert, dataSaved) - def setData(self, data): ''' Set the data assciated with a hash @@ -267,27 +259,6 @@ class Core: ''' return coredb.keydb.transportinfo.set_address_info(self, address, key, data) - def getExpiredBlocks(self): - '''Returns a list of expired blocks''' - return coredb.blockmetadb.expiredblocks.get_expired_blocks(self) - - def updateBlockInfo(self, hash, key, data): - ''' - sets info associated with a block - - hash - the hash of a block - dateReceived - the date the block was recieved, not necessarily when it was created - decrypted - if we can successfully decrypt the block (does not describe its current state) - dataType - data type of the block - dataFound - if the data has been found for the block - dataSaved - if the data has been saved for the block - sig - optional signature by the author (not optional if author is specified) - author - multi-round partial sha3-256 hash of authors public key - dateClaimed - timestamp claimed inside the block, only as trustworthy as the block author is - expire - expire date for a block - ''' - return coredb.blockmetadb.updateblockinfo.update_block_info(self, hash, key, data) - def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None, disableForward=False): ''' Inserts a block into the network @@ -422,8 +393,8 @@ class Core: self.daemonQueueAdd('uploadBlock', retData) else: pass - self.addToBlockDB(retData, selfInsert=True, dataSaved=True) - blockmetadata.process_block_metadata(self, retData) + coredb.blockmetadb.add_to_block_DB(retData, selfInsert=True, dataSaved=True) + coredb.blockmetadata.process_block_metadata(self, retData) if retData != False: if plaintextPeer == onionrvalues.DENIABLE_PEER_ADDRESS: diff --git a/onionr/coredb/blockmetadb/add.py b/onionr/coredb/blockmetadb/add.py index cfd8305c..100f3992 100644 --- a/onionr/coredb/blockmetadb/add.py +++ b/onionr/coredb/blockmetadb/add.py @@ -17,22 +17,21 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import os, sqlite3 +import os, sqlite3, secrets from onionrutils import epoch, blockmetadata -def add_to_block_DB(core_inst, newHash, selfInsert=False, dataSaved=False): +from .. import dbfiles +def add_to_block_DB(newHash, selfInsert=False, dataSaved=False): ''' Add a hash value to the block db Should be in hex format! ''' - if not os.path.exists(core_inst.blockDB): - raise Exception('Block db does not exist') - if blockmetadata.has_block(core_inst, newHash): + if blockmetadata.has_block(newHash): return - conn = sqlite3.connect(core_inst.blockDB, timeout=30) + conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30) c = conn.cursor() - currentTime = epoch.get_epoch() + core_inst._crypto.secrets.randbelow(301) + currentTime = epoch.get_epoch() + secrets.randbelow(301) if selfInsert or dataSaved: selfInsert = 1 else: diff --git a/onionr/coredb/blockmetadb/expiredblocks.py b/onionr/coredb/blockmetadb/expiredblocks.py index 9e01d3ac..859abc9d 100644 --- a/onionr/coredb/blockmetadb/expiredblocks.py +++ b/onionr/coredb/blockmetadb/expiredblocks.py @@ -19,9 +19,10 @@ ''' import sqlite3 from onionrutils import epoch -def get_expired_blocks(core_inst): +from .. import dbfiles +def get_expired_blocks(): '''Returns a list of expired blocks''' - conn = sqlite3.connect(core_inst.blockDB, timeout=30) + conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30) c = conn.cursor() date = int(epoch.get_epoch()) diff --git a/onionr/coredb/blockmetadb/updateblockinfo.py b/onionr/coredb/blockmetadb/updateblockinfo.py index bbb5d969..e32c37c8 100644 --- a/onionr/coredb/blockmetadb/updateblockinfo.py +++ b/onionr/coredb/blockmetadb/updateblockinfo.py @@ -17,13 +17,27 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' - import sqlite3 -def update_block_info(core_inst, hash, key, data): +from .. import dbfiles +def update_block_info(hash, key, data): + ''' + sets info associated with a block + + hash - the hash of a block + dateReceived - the date the block was recieved, not necessarily when it was created + decrypted - if we can successfully decrypt the block (does not describe its current state) + dataType - data type of the block + dataFound - if the data has been found for the block + dataSaved - if the data has been saved for the block + sig - optional signature by the author (not optional if author is specified) + author - multi-round partial sha3-256 hash of authors public key + dateClaimed - timestamp claimed inside the block, only as trustworthy as the block author is + expire - expire date for a block + ''' if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'): return False - conn = sqlite3.connect(core_inst.blockDB, timeout=30) + conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30) c = conn.cursor() args = (data, hash) c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args) diff --git a/onionr/onionrutils/blockmetadata.py b/onionr/onionrutils/blockmetadata.py index 7b789edf..97323d4d 100644 --- a/onionr/onionrutils/blockmetadata.py +++ b/onionr/onionrutils/blockmetadata.py @@ -23,6 +23,7 @@ from onionrusers import onionrusers from etc import onionrvalues import onionrblockapi from . import epoch, stringvalidators, bytesconverter +from coredb import dbfiles, blockmetadb def get_block_metadata_from_data(blockData): ''' accepts block contents as string, returns a tuple of @@ -70,7 +71,7 @@ def process_block_metadata(core_inst, blockHash): try: if len(blockType) <= 10: - core_inst.updateBlockInfo(blockHash, 'dataType', blockType) + blockmetadb.update_block_info(blockHash, 'dataType', blockType) except TypeError: logger.warn("Missing block information") pass @@ -81,18 +82,18 @@ def process_block_metadata(core_inst, blockHash): except (AssertionError, ValueError, TypeError) as e: expireTime = onionrvalues.OnionrValues().default_expire + curTime finally: - core_inst.updateBlockInfo(blockHash, 'expire', expireTime) + blockmetadb.update_block_info(blockHash, 'expire', expireTime) if not blockType is None: - core_inst.updateBlockInfo(blockHash, 'dataType', blockType) + blockmetadb.update_block_info(blockHash, 'dataType', blockType) onionrevents.event('processblocks', data = {'block': myBlock, 'type': blockType, 'signer': signer, 'validSig': valid}, onionr = core_inst.onionrInst) else: pass -def has_block(core_inst, hash): +def has_block(hash): ''' Check for new block in the list ''' - conn = sqlite3.connect(core_inst.blockDB) + conn = sqlite3.connect(dbfiles.block_meta_db) c = conn.cursor() if not stringvalidators.validate_hash(hash): raise Exception("Invalid hash") diff --git a/onionr/onionrutils/importnewblocks.py b/onionr/onionrutils/importnewblocks.py index b467f5fb..5420d0a6 100644 --- a/onionr/onionrutils/importnewblocks.py +++ b/onionr/onionrutils/importnewblocks.py @@ -40,7 +40,7 @@ def import_new_blocks(core_inst=None, scanDir=''): with open(block, 'rb') as newBlock: block = block.replace(scanDir, '').replace('.dat', '') if core_inst._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''): - core_inst.addToBlockDB(block.replace('.dat', ''), dataSaved=True) + blockmetadb.add_to_block_DB(block.replace('.dat', ''), dataSaved=True) logger.info('Imported block %s.' % block, terminal=True) blockmetadata.process_block_metadata(core_inst, block) else: