Onionr/onionr/core.py

462 lines
19 KiB
Python
Raw Normal View History

'''
Onionr - Private P2P Communication
2018-02-01 22:45:15 +00:00
Core Onionr library, useful for external programs. Handles peer & data processing
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
2019-06-26 04:48:24 +00:00
import os, sys, json
import logger, netcontroller, config
2018-05-19 22:11:51 +00:00
from onionrblockapi import Block
import coredb
2019-03-02 06:22:59 +00:00
import deadsimplekv as simplekv
2019-06-26 04:48:24 +00:00
import onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions
import onionrblacklist
from onionrusers import onionrusers
2019-06-22 22:54:41 +00:00
from onionrstorage import removeblock, setdata
2019-03-12 18:23:46 +00:00
import dbcreator, onionrstorage, serializeddata, subprocesspow
from etc import onionrvalues, powchoice
from onionrutils import localcommand, stringvalidators, bytesconverter, epoch
from onionrutils import blockmetadata
import storagecounter
class Core:
def __init__(self, torPort=0):
'''
Initialize Core Onionr library
'''
2019-03-29 03:33:14 +00:00
# set data dir
self.dataDir = os.environ.get('ONIONR_HOME', os.environ.get('DATA_DIR', 'data/'))
if not self.dataDir.endswith('/'):
self.dataDir += '/'
try:
self.usageFile = self.dataDir + 'disk-usage.txt'
self.config = config
self.maxBlockSize = 10000000 # max block size in bytes
self.onionrInst = None
self.queueDB = self.dataDir + 'queue.db'
self.peerDB = self.dataDir + 'peers.db'
self.blockDB = self.dataDir + 'blocks.db'
self.blockDataLocation = self.dataDir + 'blocks/'
self.blockDataDB = self.blockDataLocation + 'block-data.db'
self.publicApiHostFile = self.dataDir + 'public-host.txt'
self.privateApiHostFile = self.dataDir + 'private-host.txt'
self.addressDB = self.dataDir + 'address.db'
self.hsAddress = ''
self.i2pAddress = config.get('i2p.own_addr', None)
self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
self.bootstrapList = []
self.requirements = onionrvalues.OnionrValues()
self.torPort = torPort
self.dataNonceFile = self.dataDir + 'block-nonces.dat'
self.dbCreate = dbcreator.DBCreator(self)
self.forwardKeysFile = self.dataDir + 'forward-keys.db'
self.keyStore = simplekv.DeadSimpleKV(self.dataDir + 'cachedstorage.dat', refresh_seconds=5)
self.storage_counter = storagecounter.StorageCounter(self)
# Socket data, defined here because of multithreading constraints with gevent
self.killSockets = False
self.startSocket = {}
self.socketServerConnData = {}
self.socketReasons = {}
self.socketServerResponseData = {}
if not os.path.exists(self.dataDir):
os.mkdir(self.dataDir)
if not os.path.exists(self.dataDir + 'blocks/'):
os.mkdir(self.dataDir + 'blocks/')
if not os.path.exists(self.blockDB):
self.createBlockDB()
if not os.path.exists(self.forwardKeysFile):
self.dbCreate.createForwardKeyDB()
if not os.path.exists(self.peerDB):
self.createPeerDB()
if not os.path.exists(self.addressDB):
self.createAddressDB()
if os.path.exists(self.dataDir + '/hs/hostname'):
with open(self.dataDir + '/hs/hostname', 'r') as hs:
self.hsAddress = hs.read().strip()
# Load bootstrap address list
if os.path.exists(self.bootstrapFileLocation):
with open(self.bootstrapFileLocation, 'r') as bootstrap:
bootstrap = bootstrap.read()
for i in bootstrap.split('\n'):
self.bootstrapList.append(i)
else:
logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation)
self.use_subprocess = powchoice.use_subprocess(self)
# Initialize the crypto object
self._crypto = onionrcrypto.OnionrCrypto(self)
self._blacklist = onionrblacklist.OnionrBlackList(self)
self.serializer = serializeddata.SerializedData(self)
except Exception as error:
logger.error('Failed to initialize core Onionr library.', error=error, terminal=True)
logger.fatal('Cannot recover from error.', terminal=True)
sys.exit(1)
2018-01-06 08:51:26 +00:00
return
2018-01-10 03:50:38 +00:00
def refreshFirstStartVars(self):
'''
Hack to refresh some vars which may not be set on first start
'''
if os.path.exists(self.dataDir + '/hs/hostname'):
with open(self.dataDir + '/hs/hostname', 'r') as hs:
self.hsAddress = hs.read().strip()
2018-12-09 17:29:39 +00:00
def addPeer(self, peerID, name=''):
'''
2018-03-16 15:35:37 +00:00
Adds a public key to the key database (misleading function name)
'''
return coredb.keydb.addkeys.add_peer(self, peerID, name)
2018-02-27 21:23:49 +00:00
def addAddress(self, address):
2018-04-21 03:10:50 +00:00
'''
Add an address to the address database (only tor currently)
'''
return coredb.keydb.addkeys.add_address(self, address)
2018-02-28 00:00:37 +00:00
2018-02-27 21:23:49 +00:00
def removeAddress(self, address):
2018-04-21 03:10:50 +00:00
'''
Remove an address from the address database
'''
return coredb.keydb.removekeys.remove_address(self, address)
2018-05-02 06:01:20 +00:00
def removeBlock(self, block):
'''
remove a block from this node (does not automatically blacklist)
**You may want blacklist.addToDB(blockHash)
'''
2019-06-22 22:54:41 +00:00
removeblock.remove_block(self, block)
2018-02-27 21:23:49 +00:00
2018-02-21 09:32:31 +00:00
def createAddressDB(self):
'''
Generate the address database
'''
self.dbCreate.createAddressDB()
2018-01-10 03:50:38 +00:00
def createPeerDB(self):
'''
Generate the peer sqlite3 database and populate it with the peers table.
'''
self.dbCreate.createPeerDB()
def createBlockDB(self):
'''
Create a database for blocks
'''
self.dbCreate.createBlockDB()
2018-05-10 07:42:24 +00:00
def addToBlockDB(self, newHash, selfInsert=False, dataSaved=False):
'''
Add a hash value to the block db
Should be in hex format!
'''
2019-06-22 23:01:55 +00:00
coredb.blockmetadb.add.add_to_block_DB(self, newHash, selfInsert, dataSaved)
def setData(self, data):
'''
Set the data assciated with a hash
'''
2019-06-22 22:54:41 +00:00
return onionrstorage.setdata.set_data(self, data)
def getData(self, hash):
'''
Simply return the data associated to a hash
'''
return onionrstorage.getData(self, hash)
def daemonQueue(self):
'''
Gives commands to the communication proccess/daemon by reading an sqlite3 database
This function intended to be used by the client. Queue to exchange data between "client" and server.
'''
return coredb.daemonqueue.daemon_queue(self)
2018-07-04 19:07:17 +00:00
def daemonQueueAdd(self, command, data='', responseID=''):
'''
Add a command to the daemon queue, used by the communication daemon (communicator.py)
'''
return coredb.daemonqueue.daemon_queue_add(self, command, data, responseID)
2019-01-18 01:14:26 +00:00
def daemonQueueGetResponse(self, responseID=''):
'''
Get a response sent by communicator to the API, by requesting to the API
'''
return coredb.daemonqueue.daemon_queue_get_response(self, responseID)
2018-01-27 01:16:15 +00:00
def clearDaemonQueue(self):
'''
Clear the daemon queue (somewhat dangerous)
'''
return coredb.daemonqueue.clear_daemon_queue(self)
2018-04-19 02:16:10 +00:00
2019-01-11 22:59:21 +00:00
def listAdders(self, randomOrder=True, i2p=True, recent=0):
'''
Return a list of addresses
'''
return coredb.keydb.listkeys.list_adders(self, randomOrder, i2p, recent)
def listPeers(self, randomOrder=True, getPow=False, trust=0):
'''
2018-03-16 15:35:37 +00:00
Return a list of public keys (misleading function name)
2018-01-28 01:53:24 +00:00
randomOrder determines if the list should be in a random order
trust sets the minimum trust to list
2018-01-26 06:28:11 +00:00
'''
return coredb.keydb.listkeys.list_peers(self, randomOrder, getPow, trust)
2018-01-26 06:28:11 +00:00
def getPeerInfo(self, peer, info):
'''
Get info about a peer from their database entry
2018-01-26 06:28:11 +00:00
id text 0
name text, 1
2018-04-04 01:54:49 +00:00
adders text, 2
2018-09-11 19:45:06 +00:00
dateSeen not null, 3
2018-12-09 17:29:39 +00:00
trust int 4
hashID text 5
2018-01-26 06:28:11 +00:00
'''
return coredb.keydb.userinfo.get_user_info(self, peer, info)
2018-01-28 01:53:24 +00:00
def setPeerInfo(self, peer, key, data):
'''
Update a peer for a key
'''
return coredb.keydb.userinfo.set_peer_info(self, peer, key, data)
2018-01-26 06:28:11 +00:00
2018-02-28 00:00:37 +00:00
def getAddressInfo(self, address, info):
'''
Get info about an address from its database entry
address text, 0
type int, 1
knownPeer text, 2
speed int, 3
success int, 4
2019-02-12 05:30:56 +00:00
powValue 5
failure int 6
lastConnect 7
trust 8
introduced 9
2018-02-28 00:00:37 +00:00
'''
return coredb.keydb.transportinfo.get_address_info(self, address, info)
2018-02-28 00:00:37 +00:00
def setAddressInfo(self, address, key, data):
'''
Update an address for a key
'''
return coredb.keydb.transportinfo.set_address_info(self, address, key, data)
def getBlockList(self, dateRec = None, unsaved = False):
'''
Get list of our blocks
'''
return coredb.blockmetadb.get_block_list(self, dateRec, unsaved)
2018-05-16 01:47:58 +00:00
def getBlockDate(self, blockHash):
'''
Returns the date a block was received
'''
return coredb.blockmetadb.get_block_date(self, blockHash)
2018-05-16 01:47:58 +00:00
def getBlocksByType(self, blockType, orderDate=True):
'''
Returns a list of blocks by the type
'''
return coredb.blockmetadb.get_blocks_by_type(self, blockType, orderDate)
2018-09-30 04:42:31 +00:00
def getExpiredBlocks(self):
'''Returns a list of expired blocks'''
return coredb.blockmetadb.expiredblocks.get_expired_blocks(self)
2018-04-26 07:40:39 +00:00
def updateBlockInfo(self, hash, key, data):
'''
sets info associated with a block
2018-08-04 02:52:45 +00:00
hash - the hash of a block
dateReceived - the date the block was recieved, not necessarily when it was created
decrypted - if we can successfully decrypt the block (does not describe its current state)
dataType - data type of the block
dataFound - if the data has been found for the block
dataSaved - if the data has been saved for the block
sig - optional signature by the author (not optional if author is specified)
author - multi-round partial sha3-256 hash of authors public key
dateClaimed - timestamp claimed inside the block, only as trustworthy as the block author is
2018-09-30 04:42:31 +00:00
expire - expire date for a block
2018-04-26 07:40:39 +00:00
'''
return coredb.blockmetadb.updateblockinfo.update_block_info(self, hash, key, data)
def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None, disableForward=False):
'''
Inserts a block into the network
encryptType must be specified to encrypt a block
'''
2018-12-24 06:12:46 +00:00
allocationReachedMessage = 'Cannot insert block, disk allocation reached.'
if self.storage_counter.isFull():
2018-12-24 06:12:46 +00:00
logger.error(allocationReachedMessage)
return False
retData = False
2019-02-10 18:43:45 +00:00
if type(data) is None:
raise ValueError('Data cannot be none')
createTime = epoch.get_epoch()
dataNonce = bytesconverter.bytes_to_str(self._crypto.sha3Hash(data))
try:
with open(self.dataNonceFile, 'r') as nonces:
if dataNonce in nonces:
return retData
except FileNotFoundError:
pass
# record nonce
with open(self.dataNonceFile, 'a') as nonceFile:
nonceFile.write(dataNonce + '\n')
2018-07-04 19:07:17 +00:00
if type(data) is bytes:
data = data.decode()
data = str(data)
2019-02-10 18:43:45 +00:00
plaintext = data
plaintextMeta = {}
plaintextPeer = asymPeer
retData = ''
signature = ''
signer = ''
metadata = {}
# metadata is full block metadata, meta is internal, user specified metadata
# only use header if not set in provided meta
2018-12-09 17:29:39 +00:00
meta['type'] = str(header)
if encryptType in ('asym', 'sym', ''):
metadata['encryptType'] = encryptType
else:
raise onionrexceptions.InvalidMetadata('encryptType must be asym or sym, or blank')
try:
data = data.encode()
except AttributeError:
pass
2018-11-09 19:07:26 +00:00
if encryptType == 'asym':
meta['rply'] = createTime # Duplicate the time in encrypted messages to prevent replays
if not disableForward and sign and asymPeer != self._crypto.pubKey:
try:
forwardEncrypted = onionrusers.OnionrUser(self, asymPeer).forwardEncrypt(data)
data = forwardEncrypted[0]
meta['forwardEnc'] = True
expire = forwardEncrypted[2] # Expire time of key. no sense keeping block after that
except onionrexceptions.InvalidPubkey:
pass
#onionrusers.OnionrUser(self, asymPeer).generateForwardKey()
fsKey = onionrusers.OnionrUser(self, asymPeer).generateForwardKey()
#fsKey = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys().reverse()
meta['newFSKey'] = fsKey
2018-10-08 05:11:46 +00:00
jsonMeta = json.dumps(meta)
plaintextMeta = jsonMeta
if sign:
signature = self._crypto.edSign(jsonMeta.encode() + data, key=self._crypto.privKey, encodeResult=True)
2018-07-17 07:18:17 +00:00
signer = self._crypto.pubKey
if len(jsonMeta) > 1000:
raise onionrexceptions.InvalidMetadata('meta in json encoded form must not exceed 1000 bytes')
2018-07-04 19:07:17 +00:00
2018-10-06 18:06:46 +00:00
user = onionrusers.OnionrUser(self, symKey)
2018-06-20 20:56:28 +00:00
# encrypt block metadata/sig/content
if encryptType == 'sym':
2018-10-06 18:06:46 +00:00
if len(symKey) < self.requirements.passwordLength:
raise onionrexceptions.SecurityError('Weak encryption key')
jsonMeta = self._crypto.symmetricEncrypt(jsonMeta, key=symKey, returnEncoded=True).decode()
data = self._crypto.symmetricEncrypt(data, key=symKey, returnEncoded=True).decode()
signature = self._crypto.symmetricEncrypt(signature, key=symKey, returnEncoded=True).decode()
signer = self._crypto.symmetricEncrypt(signer, key=symKey, returnEncoded=True).decode()
elif encryptType == 'asym':
if stringvalidators.validate_pub_key(asymPeer):
2018-10-07 20:39:22 +00:00
# Encrypt block data with forward secrecy key first, but not meta
jsonMeta = json.dumps(meta)
jsonMeta = self._crypto.pubKeyEncrypt(jsonMeta, asymPeer, encodedData=True).decode()
data = self._crypto.pubKeyEncrypt(data, asymPeer, encodedData=True).decode()
signature = self._crypto.pubKeyEncrypt(signature, asymPeer, encodedData=True).decode()
signer = self._crypto.pubKeyEncrypt(signer, asymPeer, encodedData=True).decode()
try:
onionrusers.OnionrUser(self, asymPeer, saveUser=True)
except ValueError:
# if peer is already known
pass
else:
raise onionrexceptions.InvalidPubkey(asymPeer + ' is not a valid base32 encoded ed25519 key')
2018-08-04 02:52:45 +00:00
2018-06-20 20:56:28 +00:00
# compile metadata
metadata['meta'] = jsonMeta
metadata['sig'] = signature
metadata['signer'] = signer
metadata['time'] = createTime
2018-09-24 23:48:00 +00:00
2018-09-30 16:53:39 +00:00
# ensure expire is integer and of sane length
if type(expire) is not type(None):
assert len(str(int(expire))) < 14
metadata['expire'] = expire
2018-07-08 07:51:23 +00:00
# send block data (and metadata) to POW module to get tokenized block data
if self.use_subprocess:
payload = subprocesspow.SubprocessPOW(data, metadata, self).start()
else:
payload = onionrproofs.POW(metadata, data).waitForResult()
if payload != False:
2018-12-24 06:12:46 +00:00
try:
retData = self.setData(payload)
except onionrexceptions.DiskAllocationReached:
logger.error(allocationReachedMessage)
retData = False
else:
# Tell the api server through localCommand to wait for the daemon to upload this block to make statistical analysis more difficult
if localcommand.local_command(self, '/ping', maxWait=10) == 'pong!':
2019-06-26 04:48:24 +00:00
if self.config.get('general.security_level', 1) == 0:
localcommand.local_command(self, '/waitforshare/' + retData, post=True, maxWait=5)
2019-03-29 17:37:51 +00:00
self.daemonQueueAdd('uploadBlock', retData)
else:
pass
2018-12-24 06:12:46 +00:00
self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
blockmetadata.process_block_metadata(self, retData)
2018-07-11 07:35:22 +00:00
if retData != False:
if plaintextPeer == onionrvalues.DENIABLE_PEER_ADDRESS:
events.event('insertdeniable', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': bytesconverter.bytes_to_str(asymPeer)}, onionr = self.onionrInst, threaded = True)
else:
events.event('insertblock', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': bytesconverter.bytes_to_str(asymPeer)}, onionr = self.onionrInst, threaded = True)
2018-04-19 01:17:47 +00:00
return retData
2018-04-19 02:16:10 +00:00
2018-04-19 01:17:47 +00:00
def introduceNode(self):
'''
Introduces our node into the network by telling X many nodes our HS address
'''
if localcommand.local_command(self, '/ping', maxWait=10) == 'pong!':
self.daemonQueueAdd('announceNode')
2019-06-22 22:54:41 +00:00
logger.info('Introduction command will be processed.', terminal=True)
else:
2019-06-22 22:54:41 +00:00
logger.warn('No running node detected. Cannot introduce.', terminal=True)