diff --git a/onionr/blockimporter.py b/onionr/blockimporter.py index f5698d00..1b37c3cd 100644 --- a/onionr/blockimporter.py +++ b/onionr/blockimporter.py @@ -36,7 +36,7 @@ def importBlockFromData(content, coreInst): metas = coreInst._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata metadata = metas[0] - if coreInst._utils.validateMetadata(metadata): # check if metadata is valid + if coreInst._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid if coreInst._crypto.verifyPow(content): # check if POW is enough/correct logger.info('Block passed proof, saving.') blockHash = coreInst.setData(content) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 53edda76..019dedb8 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -206,7 +206,7 @@ class OnionrCommunicatorDaemon: metas = self._core._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata metadata = metas[0] #meta = metas[1] - if self._core._utils.validateMetadata(metadata): # check if metadata is valid + if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce if self._core._crypto.verifyPow(content): # check if POW is enough/correct logger.info('Block passed proof, saving.') self._core.setData(content) diff --git a/onionr/core.py b/onionr/core.py index 8f517e82..f3834ed3 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -22,7 +22,7 @@ from onionrblockapi import Block import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues import onionrblacklist - +import dbcreator if sys.version_info < (3, 6): try: import sha3 @@ -46,6 +46,8 @@ class Core: self.bootstrapList = [] self.requirements = onionrvalues.OnionrValues() self.torPort = torPort + self.dataNonceFile = 'data/block-nonces.dat' + self.dbCreate = dbcreator.DBCreator(self) self.usageFile = 'data/disk-usage.txt' @@ -188,89 +190,20 @@ class Core: def createAddressDB(self): ''' Generate the address database - - types: - 1: I2P b32 address - 2: Tor v2 (like facebookcorewwwi.onion) - 3: Tor v3 ''' - conn = sqlite3.connect(self.addressDB) - c = conn.cursor() - c.execute('''CREATE TABLE adders( - address text, - type int, - knownPeer text, - speed int, - success int, - DBHash text, - powValue text, - failure int, - lastConnect int, - lastConnectAttempt int, - trust int - ); - ''') - conn.commit() - conn.close() + self.dbCreate.createAddressDB() def createPeerDB(self): ''' Generate the peer sqlite3 database and populate it with the peers table. ''' - # generate the peer database - conn = sqlite3.connect(self.peerDB) - c = conn.cursor() - c.execute('''CREATE TABLE peers( - ID text not null, - name text, - adders text, - blockDBHash text, - forwardKey text, - dateSeen not null, - bytesStored int, - trust int, - pubkeyExchanged int, - hashID text, - pow text not null); - ''') - conn.commit() - conn.close() - return + self.dbCreate.createPeerDB() def createBlockDB(self): ''' Create a database for blocks - - hash - the hash of a block - dateReceived - the date the block was recieved, not necessarily when it was created - decrypted - if we can successfully decrypt the block (does not describe its current state) - dataType - data type of the block - dataFound - if the data has been found for the block - dataSaved - if the data has been saved for the block - sig - optional signature by the author (not optional if author is specified) - author - multi-round partial sha3-256 hash of authors public key - dateClaimed - timestamp claimed inside the block, only as trustworthy as the block author is ''' - if os.path.exists(self.blockDB): - raise Exception("Block database already exists") - conn = sqlite3.connect(self.blockDB) - c = conn.cursor() - c.execute('''CREATE TABLE hashes( - hash text not null, - dateReceived int, - decrypted int, - dataType text, - dataFound int, - dataSaved int, - sig text, - author text, - dateClaimed int - ); - ''') - conn.commit() - conn.close() - - return + self.dbCreate.createBlockDB() def addToBlockDB(self, newHash, selfInsert=False, dataSaved=False): ''' @@ -702,6 +635,7 @@ class Core: signature = '' signer = '' metadata = {} + # metadata is full block metadata, meta is internal, user specified metadata # only use header if not set in provided meta if not header is None: @@ -749,6 +683,12 @@ class Core: metadata['sig'] = signature metadata['signer'] = signer metadata['time'] = str(self._utils.getEpoch()) + + nonce = self._utils.bytesToStr(self._crypto.sha3Hash(data)) + + # TODO check in advance + with open(self.dataNonceFile, 'a') as nonceFile: + nonceFile.write(nonce + '\n') # send block data (and metadata) to POW module to get tokenized block data proof = onionrproofs.POW(metadata, data) diff --git a/onionr/onionrexceptions.py b/onionr/onionrexceptions.py index 7b0de6c7..b26a97d7 100644 --- a/onionr/onionrexceptions.py +++ b/onionr/onionrexceptions.py @@ -41,6 +41,9 @@ class InvalidMetadata(Exception): class BlacklistedBlock(Exception): pass +class DataExists(Exception): + pass + class InvalidHexHash(Exception): '''When a string is not a valid hex string of appropriate length for a hash value''' pass diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 1f2111a6..331c8e8b 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -334,7 +334,7 @@ class OnionrUtils: return retVal - def validateMetadata(self, metadata): + def validateMetadata(self, metadata, blockData): '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string''' # TODO, make this check sane sizes retData = False @@ -364,7 +364,24 @@ class OnionrUtils: break else: # if metadata loop gets no errors, it does not break, therefore metadata is valid - retData = True + # make sure we do not have another block with the same data content (prevent data duplication and replay attacks) + try: + with open(self._core.dataNonceFile, 'r') as nonceFile: + nonce = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(blockData)) + if nonce in nonceFile.read(): + retData = False # we've seen that nonce before, so we can't pass metadata + raise onionrexceptions.DataExists + except FileNotFoundError: + retData = True + except onionrexceptions.DataExists: + # do not set retData to True, because nonce has been seen before + pass + else: + retData = True + if retData: + # Executes if data not seen + with open(self._core.dataNonceFile, 'a') as nonceFile: + nonceFile.write(nonce + '\n') else: logger.warn('In call to utils.validateMetadata, metadata must be JSON string or a dictionary object')