@ -1,5 +1,5 @@
'''
Onionr - P2P Microblogging Platform & Social n etwork
Onionr - P2P Anonymous Storage N etwork
Core Onionr library , useful for external programs . Handles peer & data processing
'''
@ -17,11 +17,11 @@
You should have received a copy of the GNU General Public License
along with this program . If not , see < https : / / www . gnu . org / licenses / > .
'''
import sqlite3 , os , sys , time , math , base64 , tarfile , getpass , simplecrypt , hashlib , nacl , logger , json , netcontroller , math , config
import sqlite3 , os , sys , time , math , base64 , tarfile , nacl , logger , json , netcontroller , math , config
from onionrblockapi import Block
import onionrutils , onionrcrypto , onionrproofs , onionrevents as events , onionrexceptions , onionrvalues
import onionrblacklist
import onionrblacklist , onionrchat , onionrusers
import dbcreator
if sys . version_info < ( 3 , 6 ) :
try :
@ -35,34 +35,52 @@ class Core:
'''
Initialize Core Onionr library
'''
try :
self . dataDir = os . environ [ ' ONIONR_HOME ' ]
if not self . dataDir . endswith ( ' / ' ) :
self . dataDir + = ' / '
except KeyError :
self . dataDir = ' data/ '
try :
self . queueDB = ' data/queue.db '
self . peerDB = ' data/peers.db '
self . blockDB = ' data/blocks.db '
self . blockDataLocation = ' data/blocks/ '
self . addressDB = ' data/address.db '
self . queueDB = self . dataDir + ' queue.db '
self . peerDB = self . dataDir + ' peers.db '
self . blockDB = self . dataDir + ' blocks.db '
self . blockDataLocation = self . dataDir + ' blocks/ '
self . addressDB = self . dataDir + ' address.db '
self . hsAddress = ' '
self . bootstrapFileLocation = ' static-data/bootstrap-nodes.txt '
self . bootstrapList = [ ]
self . requirements = onionrvalues . OnionrValues ( )
self . torPort = torPort
self . dataNonceFile = ' data/ block-nonces.dat'
self . dataNonceFile = self . dataDir + ' block-nonces.dat '
self . dbCreate = dbcreator . DBCreator ( self )
self . forwardKeysFile = self . dataDir + ' forward-keys.db '
# Socket data, defined here because of multithreading constraints with gevent
self . killSockets = False
self . startSocket = { }
self . socketServerConnData = { }
self . socketReasons = { }
self . socketServerResponseData = { }
self . usageFile = ' data/disk-usage.txt '
self . usageFile = self . dataDir + ' disk-usage.txt '
self . config = config
self . maxBlockSize = 10000000 # max block size in bytes
if not os . path . exists ( ' data/ ' ) :
os . mkdir ( ' data/ ' )
if not os . path . exists ( ' data/ blocks/' ) :
os . mkdir ( ' data/ blocks/' )
if not os . path . exists ( self . dataDir ) :
os . mkdir ( self . dataDir )
if not os . path . exists ( self . dataDir + ' blocks/ ' ) :
os . mkdir ( self . dataDir + ' blocks/ ' )
if not os . path . exists ( self . blockDB ) :
self . createBlockDB ( )
if not os . path . exists ( self . forwardKeysFile ) :
self . dbCreate . createForwardKeyDB ( )
if os . path . exists ( ' data/hs/hostname ' ) :
with open ( ' data /hs/hostname' , ' r ' ) as hs :
if os . path . exists ( self . dataDir + ' /hs/hostname ' ) :
with open ( self . dataDir + ' /hs/hostname ' , ' r ' ) as hs :
self . hsAddress = hs . read ( ) . strip ( )
# Load bootstrap address list
@ -87,8 +105,8 @@ class Core:
def refreshFirstStartVars ( self ) :
''' Hack to refresh some vars which may not be set on first start '''
if os . path . exists ( ' data /hs/hostname' ) :
with open ( ' data /hs/hostname' , ' r ' ) as hs :
if os . path . exists ( self . dataDir + ' /hs/hostname ' ) :
with open ( self . dataDir + ' /hs/hostname ' , ' r ' ) as hs :
self . hsAddress = hs . read ( ) . strip ( )
def addPeer ( self , peerID , powID , name = ' ' ) :
@ -102,10 +120,12 @@ class Core:
logger . warn ( " POW token for pubkey base64 representation exceeded 120 bytes, is " + str ( sys . getsizeof ( powID ) ) )
return False
conn = sqlite3 . connect ( self . peerDB )
events . event ( ' pubkey_add ' , data = { ' key ' : peerID } , onionr = None )
conn = sqlite3 . connect ( self . peerDB , timeout = 10 )
hashID = self . _crypto . pubKeyHashID ( peerID )
c = conn . cursor ( )
t = ( peerID , name , ' unknown ' , hashID , powID )
t = ( peerID , name , ' unknown ' , hashID , powID , 0 )
for i in c . execute ( " SELECT * FROM PEERS where id = ' " + peerID + " ' ; " ) :
try :
@ -116,7 +136,7 @@ class Core:
pass
except IndexError :
pass
c . execute ( ' INSERT INTO peers (id, name, dateSeen, pow, hashID) VALUES(?, ?, ?, ?, ?); ' , t )
c . execute ( ' INSERT INTO peers (id, name, dateSeen, pow, hashID, trust ) VALUES(?, ?, ?, ?, ?, ?); ' , t )
conn . commit ( )
conn . close ( )
@ -126,11 +146,11 @@ class Core:
'''
Add an address to the address database ( only tor currently )
'''
if address == config . get ( ' i2p.ownAddr ' , None ) :
if address == config . get ( ' i2p.ownAddr ' , None ) or address == self . hsAddress :
return False
if self . _utils . validateID ( address ) :
conn = sqlite3 . connect ( self . addressDB )
conn = sqlite3 . connect ( self . addressDB , timeout = 10 )
c = conn . cursor ( )
# check if address is in database
# this is safe to do because the address is validated above, but we strip some chars here too just in case
@ -162,7 +182,7 @@ class Core:
Remove an address from the address database
'''
if self . _utils . validateID ( address ) :
conn = sqlite3 . connect ( self . addressDB )
conn = sqlite3 . connect ( self . addressDB , timeout = 10 )
c = conn . cursor ( )
t = ( address , )
c . execute ( ' Delete from adders where address=?; ' , t )
@ -181,13 +201,13 @@ class Core:
* * You may want blacklist . addToDB ( blockHash )
'''
if self . _utils . validateHash ( block ) :
conn = sqlite3 . connect ( self . blockDB )
conn = sqlite3 . connect ( self . blockDB , timeout = 10 )
c = conn . cursor ( )
t = ( block , )
c . execute ( ' Delete from hashes where hash=?; ' , t )
conn . commit ( )
conn . close ( )
blockFile = ' data /blocks/' + block + ' .dat '
blockFile = self . dataDir + ' /blocks/ ' + block + ' .dat '
dataSize = 0
try :
''' Get size of data when loaded as an object/var, rather than on disk,
@ -228,7 +248,7 @@ class Core:
raise Exception ( ' Block db does not exist ' )
if self . _utils . hasBlock ( newHash ) :
return
conn = sqlite3 . connect ( self . blockDB )
conn = sqlite3 . connect ( self . blockDB , timeout = 10 )
c = conn . cursor ( )
currentTime = self . _utils . getEpoch ( )
if selfInsert or dataSaved :
@ -256,14 +276,6 @@ class Core:
return data
def _getSha3Hash ( self , data ) :
hasher = hashlib . sha3_256 ( )
if not type ( data ) is bytes :
data = data . encode ( )
hasher . update ( data )
dataHash = hasher . hexdigest ( )
return dataHash
def setData ( self , data ) :
'''
Set the data assciated with a hash
@ -274,7 +286,7 @@ class Core:
if not type ( data ) is bytes :
data = data . encode ( )
dataHash = self . _getS ha3Hash ( data )
dataHash = self . _crypto . s ha3Hash ( data )
if type ( dataHash ) is bytes :
dataHash = dataHash . decode ( )
@ -287,7 +299,7 @@ class Core:
blockFile = open ( blockFileName , ' wb ' )
blockFile . write ( data )
blockFile . close ( )
conn = sqlite3 . connect ( self . blockDB )
conn = sqlite3 . connect ( self . blockDB , timeout = 10 )
c = conn . cursor ( )
c . execute ( " UPDATE hashes SET dataSaved=1 WHERE hash = ' " + dataHash + " ' ; " )
conn . commit ( )
@ -299,42 +311,6 @@ class Core:
return dataHash
def dataDirEncrypt ( self , password ) :
'''
Encrypt the data directory on Onionr shutdown
'''
if os . path . exists ( ' data.tar ' ) :
os . remove ( ' data.tar ' )
tar = tarfile . open ( " data.tar " , " w " )
for name in [ ' data ' ] :
tar . add ( name )
tar . close ( )
tarData = open ( ' data.tar ' , ' r ' , encoding = " ISO-8859-1 " ) . read ( )
encrypted = simplecrypt . encrypt ( password , tarData )
open ( ' data-encrypted.dat ' , ' wb ' ) . write ( encrypted )
os . remove ( ' data.tar ' )
return
def dataDirDecrypt ( self , password ) :
'''
Decrypt the data directory on startup
'''
if not os . path . exists ( ' data-encrypted.dat ' ) :
return ( False , ' encrypted archive does not exist ' )
data = open ( ' data-encrypted.dat ' , ' rb ' ) . read ( )
try :
decrypted = simplecrypt . decrypt ( password , data )
except simplecrypt . DecryptionException :
return ( False , ' wrong password (or corrupted archive) ' )
else :
open ( ' data.tar ' , ' wb ' ) . write ( decrypted )
tar = tarfile . open ( ' data.tar ' )
tar . extractall ( )
tar . close ( )
return ( True , ' ' )
def daemonQueue ( self ) :
'''
Gives commands to the communication proccess / daemon by reading an sqlite3 database
@ -343,16 +319,16 @@ class Core:
'''
retData = False
if not os . path . exists ( self . queueDB ) :
self . mak eDaemonDB( )
self . dbCreate . creat eDaemonDB( )
else :
conn = sqlite3 . connect ( self . queueDB )
conn = sqlite3 . connect ( self . queueDB , timeout = 10 )
c = conn . cursor ( )
try :
for row in c . execute ( ' SELECT command, data, date, min(ID) FROM commands group by id ' ) :
retData = row
break
except sqlite3 . OperationalError :
self . mak eDaemonDB( )
self . dbCreate . creat eDaemonDB( )
else :
if retData != False :
c . execute ( ' DELETE FROM commands WHERE id=?; ' , ( retData [ 3 ] , ) )
@ -363,38 +339,32 @@ class Core:
return retData
def makeDaemonDB ( self ) :
''' generate the daemon queue db '''
conn = sqlite3 . connect ( self . queueDB )
c = conn . cursor ( )
# Create table
c . execute ( ''' CREATE TABLE commands
( id integer primary key autoincrement , command text , data text , date text ) ''' )
conn . commit ( )
conn . close ( )
def daemonQueueAdd ( self , command , data = ' ' ) :
'''
Add a command to the daemon queue , used by the communication daemon ( communicator . py )
'''
retData = True
# Intended to be used by the web server
date = self . _utils . getEpoch ( )
conn = sqlite3 . connect ( self . queueDB )
conn = sqlite3 . connect ( self . queueDB , timeout = 10 )
c = conn . cursor ( )
t = ( command , data , date )
c . execute ( ' INSERT INTO commands (command, data, date) VALUES(?, ?, ?) ' , t )
conn . commit ( )
conn . close ( )
try :
c . execute ( ' INSERT INTO commands (command, data, date) VALUES(?, ?, ?) ' , t )
conn . commit ( )
conn . close ( )
except sqlite3 . OperationalError :
retData = False
self . daemonQueue ( )
events . event ( ' queue_push ' , data = { ' command ' : command , ' data ' : data } , onionr = None )
return
return retData
def clearDaemonQueue ( self ) :
'''
Clear the daemon queue ( somewhat dangerous )
'''
conn = sqlite3 . connect ( self . queueDB )
conn = sqlite3 . connect ( self . queueDB , timeout = 10 )
c = conn . cursor ( )
try :
c . execute ( ' DELETE FROM commands; ' )
@ -410,7 +380,7 @@ class Core:
'''
Return a list of addresses
'''
conn = sqlite3 . connect ( self . addressDB )
conn = sqlite3 . connect ( self . addressDB , timeout = 10 )
c = conn . cursor ( )
if randomOrder :
addresses = c . execute ( ' SELECT * FROM adders ORDER BY RANDOM(); ' )
@ -422,19 +392,23 @@ class Core:
conn . close ( )
return addressList
def listPeers ( self , randomOrder = True , getPow = False ) :
def listPeers ( self , randomOrder = True , getPow = False , trust = 0 ) :
'''
Return a list of public keys ( misleading function name )
randomOrder determines if the list should be in a random order
trust sets the minimum trust to list
'''
conn = sqlite3 . connect ( self . peerDB )
conn = sqlite3 . connect ( self . peerDB , timeout = 10 )
c = conn . cursor ( )
payload = " "
if trust not in ( 0 , 1 , 2 ) :
logger . error ( ' Tried to select invalid trust. ' )
return
if randomOrder :
payload = ' SELECT * FROM peers ORDER BY RANDOM(); '
payload = ' SELECT * FROM peers where trust >= %s ORDER BY RANDOM(); ' % ( trust , )
else :
payload = ' SELECT * FROM peers; '
payload = ' SELECT * FROM peers where trust >= %s ' % ( trust , )
peerList = [ ]
for i in c . execute ( payload ) :
try :
@ -462,18 +436,17 @@ class Core:
id text 0
name text , 1
adders text , 2
forwardKey text , 3
dateSeen not null , 4
bytesStored int , 5
trust int 6
pubkeyExchanged int 7
hashID text 8
pow text 9
'''
conn = sqlite3 . connect ( self . peerDB )
dateSeen not null , 3
bytesStored int , 4
trust int 5
pubkeyExchanged int 6
hashID text 7
pow text 8
'''
conn = sqlite3 . connect ( self . peerDB , timeout = 10 )
c = conn . cursor ( )
command = ( peer , )
infoNumbers = { ' id ' : 0 , ' name ' : 1 , ' adders ' : 2 , ' forwardKey ' : 3 , ' ' : 4 , ' bytesStored ' : 5 , ' trust ' : 6 , ' pubkeyExchanged ' : 7 , ' hashID ' : 8 }
infoNumbers = { ' id ' : 0 , ' name ' : 1 , ' adders ' : 2 , ' dateSeen ' : 3 , ' bytesStored ' : 4 , ' trust ' : 5 , ' pubkeyExchanged ' : 6 , ' hashID ' : 7 }
info = infoNumbers [ info ]
iterCount = 0
retVal = ' '
@ -492,7 +465,7 @@ class Core:
'''
Update a peer for a key
'''
conn = sqlite3 . connect ( self . peerDB )
conn = sqlite3 . connect ( self . peerDB , timeout = 10 )
c = conn . cursor ( )
command = ( data , peer )
# TODO: validate key on whitelist
@ -516,7 +489,7 @@ class Core:
failure int 6
lastConnect 7
'''
conn = sqlite3 . connect ( self . addressDB )
conn = sqlite3 . connect ( self . addressDB , timeout = 10 )
c = conn . cursor ( )
command = ( address , )
infoNumbers = { ' address ' : 0 , ' type ' : 1 , ' knownPeer ' : 2 , ' speed ' : 3 , ' success ' : 4 , ' DBHash ' : 5 , ' failure ' : 6 , ' lastConnect ' : 7 }
@ -537,7 +510,7 @@ class Core:
'''
Update an address for a key
'''
conn = sqlite3 . connect ( self . addressDB )
conn = sqlite3 . connect ( self . addressDB , timeout = 10 )
c = conn . cursor ( )
command = ( data , address )
# TODO: validate key on whitelist
@ -553,7 +526,7 @@ class Core:
'''
Get list of our blocks
'''
conn = sqlite3 . connect ( self . blockDB )
conn = sqlite3 . connect ( self . blockDB , timeout = 10 )
c = conn . cursor ( )
if unsaved :
execute = ' SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM(); '
@ -570,7 +543,7 @@ class Core:
'''
Returns the date a block was received
'''
conn = sqlite3 . connect ( self . blockDB )
conn = sqlite3 . connect ( self . blockDB , timeout = 10 )
c = conn . cursor ( )
execute = ' SELECT dateReceived FROM hashes WHERE hash=?; '
args = ( blockHash , )
@ -584,7 +557,7 @@ class Core:
'''
Returns a list of blocks by the type
'''
conn = sqlite3 . connect ( self . blockDB )
conn = sqlite3 . connect ( self . blockDB , timeout = 10 )
c = conn . cursor ( )
if orderDate :
execute = ' SELECT hash FROM hashes WHERE dataType=? ORDER BY dateReceived; '
@ -595,15 +568,28 @@ class Core:
for row in c . execute ( execute , args ) :
for i in row :
rows . append ( i )
return rows
def getExpiredBlocks ( self ) :
''' Returns a list of expired blocks '''
conn = sqlite3 . connect ( self . blockDB , timeout = 10 )
c = conn . cursor ( )
date = int ( self . _utils . getEpoch ( ) )
execute = ' SELECT hash FROM hashes WHERE expire <= %s ORDER BY dateReceived; ' % ( date , )
rows = list ( )
for row in c . execute ( execute ) :
for i in row :
rows . append ( i )
return rows
def setBlockType ( self , hash , blockType ) :
'''
Sets the type of block
'''
conn = sqlite3 . connect ( self . blockDB )
conn = sqlite3 . connect ( self . blockDB , timeout = 10 )
c = conn . cursor ( )
c . execute ( " UPDATE hashes SET dataType= ' " + blockType + " ' WHERE hash = ' " + hash + " ' ; " )
conn . commit ( )
@ -623,12 +609,13 @@ class Core:
sig - optional signature by the author ( not optional if author is specified )
author - multi - round partial sha3 - 256 hash of authors public key
dateClaimed - timestamp claimed inside the block , only as trustworthy as the block author is
expire - expire date for a block
'''
if key not in ( ' dateReceived ' , ' decrypted ' , ' dataType ' , ' dataFound ' , ' dataSaved ' , ' sig ' , ' author ' , ' dateClaimed ' ) :
if key not in ( ' dateReceived ' , ' decrypted ' , ' dataType ' , ' dataFound ' , ' dataSaved ' , ' sig ' , ' author ' , ' dateClaimed ' , ' expire ' ) :
return False
conn = sqlite3 . connect ( self . blockDB )
conn = sqlite3 . connect ( self . blockDB , timeout = 10 )
c = conn . cursor ( )
args = ( data , hash )
c . execute ( " UPDATE hashes SET " + key + " = ? where hash = ?; " , args )
@ -636,7 +623,7 @@ class Core:
conn . close ( )
return True
def insertBlock ( self , data , header = ' txt ' , sign = False , encryptType = ' ' , symKey = ' ' , asymPeer = ' ' , meta = None ) :
def insertBlock ( self , data , header = ' txt ' , sign = False , encryptType = ' ' , symKey = ' ' , asymPeer = ' ' , meta = None , expire = None ) :
'''
Inserts a block into the network
encryptType must be specified to encrypt a block
@ -673,8 +660,6 @@ class Core:
meta [ ' type ' ] = header
meta [ ' type ' ] = str ( meta [ ' type ' ] )
jsonMeta = json . dumps ( meta )
if encryptType in ( ' asym ' , ' sym ' , ' ' ) :
metadata [ ' encryptType ' ] = encryptType
else :
@ -684,7 +669,20 @@ class Core:
data = data . encode ( )
except AttributeError :
pass
# sign before encrypt, as unauthenticated crypto should not be a problem here
if encryptType == ' asym ' :
try :
forwardEncrypted = onionrusers . OnionrUser ( self , asymPeer ) . forwardEncrypt ( data )
data = forwardEncrypted [ 0 ]
meta [ ' forwardEnc ' ] = True
except onionrexceptions . InvalidPubkey :
onionrusers . OnionrUser ( self , asymPeer ) . generateForwardKey ( )
else :
logger . info ( forwardEncrypted )
onionrusers . OnionrUser ( self , asymPeer ) . generateForwardKey ( )
fsKey = onionrusers . OnionrUser ( self , asymPeer ) . getGeneratedForwardKeys ( ) [ 0 ]
meta [ ' newFSKey ' ] = fsKey [ 0 ]
jsonMeta = json . dumps ( meta )
if sign :
signature = self . _crypto . edSign ( jsonMeta . encode ( ) + data , key = self . _crypto . privKey , encodeResult = True )
signer = self . _crypto . pubKey
@ -692,8 +690,11 @@ class Core:
if len ( jsonMeta ) > 1000 :
raise onionrexceptions . InvalidMetadata ( ' meta in json encoded form must not exceed 1000 bytes ' )
user = onionrusers . OnionrUser ( self , symKey )
# encrypt block metadata/sig/content
if encryptType == ' sym ' :
if len ( symKey ) < self . requirements . passwordLength :
raise onionrexceptions . SecurityError ( ' Weak encryption key ' )
jsonMeta = self . _crypto . symmetricEncrypt ( jsonMeta , key = symKey , returnEncoded = True ) . decode ( )
@ -702,6 +703,8 @@ class Core:
signer = self . _crypto . symmetricEncrypt ( signer , key = symKey , returnEncoded = True ) . decode ( )
elif encryptType == ' asym ' :
if self . _utils . validatePubKey ( asymPeer ) :
# Encrypt block data with forward secrecy key first, but not meta
jsonMeta = json . dumps ( meta )
jsonMeta = self . _crypto . pubKeyEncrypt ( jsonMeta , asymPeer , encodedData = True , anonymous = True ) . decode ( )
data = self . _crypto . pubKeyEncrypt ( data , asymPeer , encodedData = True , anonymous = True ) . decode ( )
signature = self . _crypto . pubKeyEncrypt ( signature , asymPeer , encodedData = True , anonymous = True ) . decode ( )
@ -714,6 +717,11 @@ class Core:
metadata [ ' sig ' ] = signature
metadata [ ' signer ' ] = signer
metadata [ ' time ' ] = str ( self . _utils . getEpoch ( ) )
# ensure expire is integer and of sane length
if type ( expire ) is not type ( None ) :
assert len ( str ( int ( expire ) ) ) < 14
metadata [ ' expire ' ] = expire
# send block data (and metadata) to POW module to get tokenized block data
proof = onionrproofs . POW ( metadata , data )
@ -721,7 +729,8 @@ class Core:
if payload != False :
retData = self . setData ( payload )
self . addToBlockDB ( retData , selfInsert = True , dataSaved = True )
self . setBlockType ( retData , meta [ ' type ' ] )
#self.setBlockType(retData, meta['type'])
self . _utils . processBlockMetadata ( retData )
self . daemonQueueAdd ( ' uploadBlock ' , retData )
if retData != False :
Kevin
2 years ago