kev
/
Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

Merge branch 'config-all-the-things' into 'master' 

Configuration for more things

See merge request beardog/Onionr!8
This commit is contained in:
Kevin 2018-08-25 14:57:12 +00:00
parent 1db4d48fb5 7a7b627259
commit f15d00f61f
10 changed files with 204 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
onionr/blockimporter.py
View File

@ -39,8 +39,12 @@ def importBlockFromData(content, coreInst):
if coreInst._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid
if coreInst._crypto.verifyPow(content): # check if POW is enough/correct
logger.info('Block passed proof, saving.')
blockHash = coreInst.setData(content)
coreInst.addToBlockDB(blockHash, dataSaved=True)
coreInst._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
retData = True
try:
blockHash = coreInst.setData(content)
except onionrexceptions.DiskAllocationReached:
pass
else:
coreInst.addToBlockDB(blockHash, dataSaved=True)
coreInst._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
retData = True
return retData

60
onionr/communicator2.py
View File

@ -27,6 +27,8 @@ from defusedxml import minidom
class OnionrCommunicatorDaemon:
def __init__(self, debug, developmentMode):
self.isOnline = True # Assume we're connected to the internet
# list of timer instances
self.timers = []
@ -78,21 +80,19 @@ class OnionrCommunicatorDaemon:
if debug or developmentMode:
OnionrCommunicatorTimers(self, self.heartbeat, 10)
# Print nice header thing :)
if config.get('general.display_header', True) and not self.shutdown:
self.header()
# Set timers, function reference, seconds
# requiresPeer True means the timer function won't fire if we have no connected peers
# TODO: make some of these timer counts configurable
OnionrCommunicatorTimers(self, self.daemonCommands, 5)
OnionrCommunicatorTimers(self, self.detectAPICrash, 5)
peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60)
OnionrCommunicatorTimers(self, self.lookupBlocks, 7, requiresPeer=True, maxThreads=1)
OnionrCommunicatorTimers(self, self.getBlocks, 10, requiresPeer=True)
OnionrCommunicatorTimers(self, self.lookupBlocks, self._core.config.get('timers.lookupBlocks'), requiresPeer=True, maxThreads=1)
OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.getBlocks'), requiresPeer=True)
OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58)
OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 65)
OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True)
OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True)
netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600)
announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 305, requiresPeer=True, maxThreads=1)
cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requiresPeer=True)
@ -114,14 +114,14 @@ class OnionrCommunicatorDaemon:
pass
logger.info('Goodbye.')
self._core._utils.localCommand('shutdown')
self._core._utils.localCommand('shutdown') # shutdown the api
time.sleep(0.5)
def lookupKeys(self):
'''Lookup new keys'''
logger.debug('Looking up new keys...')
tryAmount = 1
for i in range(tryAmount):
for i in range(tryAmount): # amount of times to ask peers for new keys
# Download new key list from random online peers
peer = self.pickOnlinePeer()
newKeys = self.peerAction(peer, action='kex')
@ -148,6 +148,12 @@ class OnionrCommunicatorDaemon:
existingBlocks = self._core.getBlockList()
triedPeers = [] # list of peers we've tried this time around
for i in range(tryAmount):
# check if disk allocation is used
if not self.isOnline:
break
if self._core._utils.storageCounter.isFull():
logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
break
peer = self.pickOnlinePeer() # select random online peer
# if we've already tried all the online peers this time around, stop
if peer in triedPeers:
@ -162,7 +168,7 @@ class OnionrCommunicatorDaemon:
if newDBHash != self._core.getAddressInfo(peer, 'DBHash'):
self._core.setAddressInfo(peer, 'DBHash', newDBHash)
try:
newBlocks = self.peerAction(peer, 'getBlockHashes')
newBlocks = self.peerAction(peer, 'getBlockHashes') # get list of new block hashes
except Exception as error:
logger.warn("could not get new blocks with " + peer, error=error)
newBlocks = False
@ -174,15 +180,18 @@ class OnionrCommunicatorDaemon:
if not i in existingBlocks:
# if block does not exist on disk and is not already in block queue
if i not in self.blockQueue and not self._core._blacklist.inBlacklist(i):
self.blockQueue.append(i)
self.blockQueue.append(i) # add blocks to download queue
self.decrementThreadCount('lookupBlocks')
return
def getBlocks(self):
'''download new blocks in queue'''
for blockHash in self.blockQueue:
if self.shutdown:
removeFromQueue = True
if self.shutdown or not self.isOnline:
# Exit loop if shutting down or offline
break
# Do not download blocks being downloaded or that are already saved (edge cases)
if blockHash in self.currentDownloading:
logger.debug('ALREADY DOWNLOADING ' + blockHash)
continue
@ -190,7 +199,11 @@ class OnionrCommunicatorDaemon:
logger.debug('%s is already saved' % (blockHash,))
self.blockQueue.remove(blockHash)
continue
self.currentDownloading.append(blockHash)
if self._core._blacklist.inBlacklist(blockHash):
continue
if self._core._utils.storageCounter.isFull():
break
self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
logger.info("Attempting to download %s..." % blockHash)
peerUsed = self.pickOnlinePeer()
content = self.peerAction(peerUsed, 'getData', data=blockHash) # block content from random peer (includes metadata)
@ -212,10 +225,15 @@ class OnionrCommunicatorDaemon:
#meta = metas[1]
if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce
if self._core._crypto.verifyPow(content): # check if POW is enough/correct
logger.info('Block passed proof, saving.')
self._core.setData(content)
self._core.addToBlockDB(blockHash, dataSaved=True)
self._core._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
logger.info('Block passed proof, attempting save.')
try:
self._core.setData(content)
except onionrexceptions.DiskAllocationReached:
logger.error("Reached disk allocation allowance, cannot save this block.")
removeFromQueue = False
else:
self._core.addToBlockDB(blockHash, dataSaved=True)
self._core._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
else:
logger.warn('POW failed for block ' + blockHash)
else:
@ -234,7 +252,8 @@ class OnionrCommunicatorDaemon:
# Punish peer for sharing invalid block (not always malicious, but is bad regardless)
onionrpeers.PeerProfiles(peerUsed, self._core).addScore(-50)
logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash)
self.blockQueue.remove(blockHash) # remove from block queue both if success or false
if removeFromQueue:
self.blockQueue.remove(blockHash) # remove from block queue both if success or false
self.currentDownloading.remove(blockHash)
self.decrementThreadCount('getBlocks')
return
@ -476,13 +495,6 @@ class OnionrCommunicatorDaemon:
self.shutdown = True
self.decrementThreadCount('detectAPICrash')
def header(self, message = logger.colors.fg.pink + logger.colors.bold + 'Onionr' + logger.colors.reset + logger.colors.fg.pink + ' has started.'):
if os.path.exists('static-data/header.txt'):
with open('static-data/header.txt', 'rb') as file:
# only to stdout, not file or log or anything
sys.stderr.write(file.read().decode().replace('P', logger.colors.fg.pink).replace('W', logger.colors.reset + logger.colors.bold).replace('G', logger.colors.fg.green).replace('\n', logger.colors.reset + '\n').replace('B', logger.colors.bold).replace('V', onionr.ONIONR_VERSION))
logger.info(logger.colors.fg.lightgreen + '-> ' + str(message) + logger.colors.reset + logger.colors.fg.lightgreen + ' <-\n')
class OnionrCommunicatorTimers:
def __init__(self, daemonInstance, timerFunction, frequency, makeThread=True, threadAmount=1, maxThreads=5, requiresPeer=False):
self.timerFunction = timerFunction

41
onionr/core.py
View File

@ -50,6 +50,9 @@ class Core:
self.dbCreate = dbcreator.DBCreator(self)
self.usageFile = 'data/disk-usage.txt'
self.config = config
self.maxBlockSize = 10000000 # max block size in bytes
if not os.path.exists('data/'):
os.mkdir('data/')
@ -174,6 +177,8 @@ class Core:
def removeBlock(self, block):
'''
remove a block from this node (does not automatically blacklist)
**You may want blacklist.addToDB(blockHash)
'''
if self._utils.validateHash(block):
conn = sqlite3.connect(self.blockDB)
@ -182,8 +187,16 @@ class Core:
c.execute('Delete from hashes where hash=?;', t)
conn.commit()
conn.close()
blockFile = 'data/blocks/' + block + '.dat'
dataSize = 0
try:
os.remove('data/blocks/' + block + '.dat')
''' Get size of data when loaded as an object/var, rather than on disk,
to avoid conflict with getsizeof when saving blocks
'''
with open(blockFile, 'r') as data:
dataSize = sys.getsizeof(data.read())
self._utils.storageCounter.removeBytes(dataSize)
os.remove(blockFile)
except FileNotFoundError:
pass
@ -256,6 +269,8 @@ class Core:
Set the data assciated with a hash
'''
data = data
dataSize = sys.getsizeof(data)
if not type(data) is bytes:
data = data.encode()
@ -268,15 +283,19 @@ class Core:
pass # TODO: properly check if block is already saved elsewhere
#raise Exception("Data is already set for " + dataHash)
else:
blockFile = open(blockFileName, 'wb')
blockFile.write(data)
blockFile.close()
conn = sqlite3.connect(self.blockDB)
c = conn.cursor()
c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = '" + dataHash + "';")
conn.commit()
conn.close()
if self._utils.storageCounter.addBytes(dataSize) != False:
blockFile = open(blockFileName, 'wb')
blockFile.write(data)
blockFile.close()
conn = sqlite3.connect(self.blockDB)
c = conn.cursor()
c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = '" + dataHash + "';")
conn.commit()
conn.close()
with open(self.dataNonceFile, 'a') as nonceFile:
nonceFile.write(dataHash + '\n')
else:
raise onionrexceptions.DiskAllocationReached
return dataHash
@ -539,7 +558,7 @@ class Core:
if unsaved:
execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
else:
execute = 'SELECT hash FROM hashes ORDER BY dateReceived DESC;'
execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;'
rows = list()
for row in c.execute(execute):
for i in row:

10
onionr/onionr.py
View File

@ -588,6 +588,9 @@ class Onionr:
time.sleep(1)
#TODO make runable on windows
subprocess.Popen([communicatorDaemon, "run", str(net.socksPort)])
# Print nice header thing :)
if config.get('general.display_header', True):
self.header()
logger.debug('Started communicator')
events.event('daemon_start', onionr = self)
try:
@ -759,5 +762,12 @@ class Onionr:
print('Opening %s ...' % url)
webbrowser.open(url, new = 1, autoraise = True)
def header(self, message = logger.colors.fg.pink + logger.colors.bold + 'Onionr' + logger.colors.reset + logger.colors.fg.pink + ' has started.'):
if os.path.exists('static-data/header.txt'):
with open('static-data/header.txt', 'rb') as file:
# only to stdout, not file or log or anything
sys.stderr.write(file.read().decode().replace('P', logger.colors.fg.pink).replace('W', logger.colors.reset + logger.colors.bold).replace('G', logger.colors.fg.green).replace('\n', logger.colors.reset + '\n').replace('B', logger.colors.bold).replace('V', ONIONR_VERSION))
logger.info(logger.colors.fg.lightgreen + '-> ' + str(message) + logger.colors.reset + logger.colors.fg.lightgreen + ' <-\n')
if __name__ == "__main__":
Onionr()

19
onionr/onionrdaemontools.py
View File

@ -53,4 +53,21 @@ class DaemonTools:
if self.daemon._core._utils.doPostRequest(url, data) == 'Success':
retData = True
self.daemon.decrementThreadCount('announceNode')
return retData
return retData
def netCheck(self):
'''Check if we are connected to the internet or not when we can't connect to any peers'''
if len(self.daemon.onlinePeers) != 0:
if not self.daemon._core._utils.checkNetwork(torPort=self.daemon.proxyPort):
logger.warn('Network check failed, are you connected to the internet?')
self.daemon.isOnline = False
self.daemon.decrementThreadCount('netCheck')
def cleanOldBlocks(self):
'''Delete old blocks if our disk allocation is full/near full'''
while self.daemon._core._utils.storageCounter.isFull():
oldest = self.daemon._core.getBlockList()[0]
self.daemon._core._blacklist.addToDB(oldest)
self.daemon._core.removeBlock(oldest)
logger.info('Deleted block: %s' % (oldest,))
self.daemon.decrementThreadCount('cleanOldBlocks')

5
onionr/onionrexceptions.py
View File

@ -58,3 +58,8 @@ class MissingPort(Exception):
class InvalidAddress(Exception):
pass
# file exceptions
class DiskAllocationReached(Exception):
pass

6
onionr/onionrpeers.py
View File

@ -90,7 +90,11 @@ def peerCleanup(coreInst):
if PeerProfiles(address, coreInst).score < minScore:
coreInst.removeAddress(address)
try:
coreInst._blacklist.addToDB(address, dataType=1, expire=300)
if (coreInst._utils.getEpoch() - coreInst.getPeerInfo(address, 4)) >= 600:
expireTime = 600
else:
expireTime = 86400
coreInst._blacklist.addToDB(address, dataType=1, expire=expireTime)
except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue
pass
logger.warn('Removed address ' + address + '.')

34
onionr/onionrutils.py
View File

@ -23,7 +23,7 @@ import nacl.signing, nacl.encoding
from onionrblockapi import Block
import onionrexceptions
from defusedxml import minidom
import pgpwords
import pgpwords, storagecounter
if sys.version_info < (3, 6):
try:
import sha3
@ -40,9 +40,9 @@ class OnionrUtils:
self._core = coreInstance
self.timingToken = ''
self.avoidDupe = [] # list used to prevent duplicate requests per peer for certain actions
self.peerProcessing = {} # dict of current peer actions: peer, actionList
self.storageCounter = storagecounter.StorageCounter(self._core)
config.reload()
return
@ -131,8 +131,12 @@ class OnionrUtils:
if not config.get('tor.v3onions') and len(adder) == 62:
continue
if self._core.addAddress(adder):
logger.info('Added %s to db.' % adder, timestamp = True)
retVal = True
# Check if we have the maxmium amount of allowed stored peers
if config.get('peers.maxStoredPeers') > len(self._core.listAdders()):
logger.info('Added %s to db.' % adder, timestamp = True)
retVal = True
else:
logger.warn('Reached the maximum amount of peers in the net database as allowed by your config.')
else:
pass
#logger.debug('%s is either our address or already in our DB' % adder)
@ -380,10 +384,6 @@ class OnionrUtils:
pass
else:
retData = True
if retData:
# Executes if data not seen
with open(self._core.dataNonceFile, 'a') as nonceFile:
nonceFile.write(nonce + '\n')
else:
logger.warn('In call to utils.validateMetadata, metadata must be JSON string or a dictionary object')
@ -630,6 +630,22 @@ class OnionrUtils:
except AttributeError:
pass
return data
def checkNetwork(self, torPort=0):
'''Check if we are connected to the internet (through Tor)'''
retData = False
connectURLs = []
try:
with open('static-data/connect-check.txt', 'r') as connectTest:
connectURLs = connectTest.read().split(',')
for url in connectURLs:
if self.doGetRequest(url, port=torPort) != False:
retData = True
break
except FileNotFoundError:
pass
return retData
def size(path='.'):
'''
@ -655,4 +671,4 @@ def humanSize(num, suffix='B'):
if abs(num) < 1024.0:
return "%.1f %s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f %s%s" % (num, 'Yi', suffix)
return "%.1f %s%s" % (num, 'Yi', suffix)

8
onionr/static-data/default_config.json
View File

@ -51,14 +51,18 @@
},
"allocations":{
"disk": 9000000000,
"disk": 10000000000,
"netTotal": 1000000000,
"blockCache": 5000000,
"blockCacheTotal": 50000000
},
"peers":{
"minimumScore": -100,
"maxStoredPeers": 500,
"maxStoredPeers": 5000,
"maxConnect": 5
},
"timers":{
"lookupBlocks": 25,
"getBlocks": 30
}
}

61
onionr/storagecounter.py Normal file
View File

@ -0,0 +1,61 @@
'''
Onionr - P2P Microblogging Platform & Social network.
Keeps track of how much disk space we're using
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import config
class StorageCounter:
def __init__(self, coreInst):
self._core = coreInst
self.dataFile = self._core.usageFile
return
def isFull(self):
retData = False
if self._core.config.get('allocations.disk') <= (self.getAmount() + 1000):
retData = True
return retData
def _update(self, data):
with open(self.dataFile, 'w') as dataFile:
dataFile.write(str(data))
def getAmount(self):
'''Return how much disk space we're using (according to record)'''
retData = 0
try:
with open(self.dataFile, 'r') as dataFile:
retData = int(dataFile.read())
except FileNotFoundError:
pass
return retData
def addBytes(self, amount):
'''Record that we are now using more disk space, unless doing so would exceed configured max'''
newAmount = amount + self.getAmount()
retData = newAmount
if newAmount > self._core.config.get('allocations.disk'):
retData = False
else:
self._update(newAmount)
return retData
def removeBytes(self, amount):
'''Record that we are now using less disk space'''
newAmount = self.getAmount() - amount
self._update(newAmount)
return newAmount