added dynamic proof of work

This commit is contained in:
Kevin Froman 2018-12-24 00:12:46 -06:00
parent 8c79cd9583
commit b45bb94375
No known key found for this signature in database
GPG Key ID: 0D414D0FE405B63B
9 changed files with 104 additions and 29 deletions

View File

@ -102,7 +102,7 @@ class OnionrCommunicatorDaemon:
OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True) OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True)
OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1) OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1)
OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1) OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1)
OnionrCommunicatorTimers(self, self.detectAPICrash, 5, maxThreads=1) OnionrCommunicatorTimers(self, self.detectAPICrash, 30, maxThreads=1)
deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1) deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1)
netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600)

View File

@ -680,7 +680,10 @@ class Core:
Inserts a block into the network Inserts a block into the network
encryptType must be specified to encrypt a block encryptType must be specified to encrypt a block
''' '''
allocationReachedMessage = 'Cannot insert block, disk allocation reached.'
if self._utils.storageCounter.isFull():
logger.error(allocationReachedMessage)
return False
retData = False retData = False
# check nonce # check nonce
dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data)) dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data))
@ -774,7 +777,12 @@ class Core:
proof = onionrproofs.POW(metadata, data) proof = onionrproofs.POW(metadata, data)
payload = proof.waitForResult() payload = proof.waitForResult()
if payload != False: if payload != False:
try:
retData = self.setData(payload) retData = self.setData(payload)
except onionrexceptions.DiskAllocationReached:
logger.error(allocationReachedMessage)
retData = False
else:
# Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult # Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult
self._utils.localCommand('waitforshare/' + retData) self._utils.localCommand('waitforshare/' + retData)
self.addToBlockDB(retData, selfInsert=True, dataSaved=True) self.addToBlockDB(retData, selfInsert=True, dataSaved=True)

View File

@ -719,7 +719,6 @@ class Onionr:
''' '''
Starts the Onionr communication daemon Starts the Onionr communication daemon
''' '''
communicatorDaemon = './communicator2.py' communicatorDaemon = './communicator2.py'
# remove runcheck if it exists # remove runcheck if it exists

View File

@ -245,7 +245,7 @@ class Block:
blockFile.write(self.getRaw().encode()) blockFile.write(self.getRaw().encode())
else: else:
self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire()) self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire())
if self.hash != False:
self.update() self.update()
return self.getHash() return self.getHash()

View File

@ -269,7 +269,8 @@ class OnionrCrypto:
except AttributeError: except AttributeError:
pass pass
difficulty = math.floor(dataLen / 1000000) difficulty = onionrproofs.getDifficultyForNewBlock(blockContent, ourBlock=False)
if difficulty < int(config.get('general.minimum_block_pow')): if difficulty < int(config.get('general.minimum_block_pow')):
difficulty = int(config.get('general.minimum_block_pow')) difficulty = int(config.get('general.minimum_block_pow'))
mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode() mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()

View File

@ -19,7 +19,55 @@
''' '''
import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys, base64, json import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys, base64, json
import core, config import core, onionrutils, config
import onionrblockapi
def getDifficultyModifier(coreOrUtilsInst=None):
'''Accepts a core or utils instance returns
the difficulty modifier for block storage based
on a variety of factors, currently only disk use.
'''
classInst = coreOrUtilsInst
retData = 0
if isinstance(classInst, core.Core):
useFunc = classInst._utils.storageCounter.getPercent
elif isinstance(classInst, onionrutils.OnionrUtils):
useFunc = classInst.storageCounter.getPercent
else:
useFunc = core.Core()._utils.storageCounter.getPercent
percentUse = useFunc()
if percentUse >= 0.50:
retData += 1
elif percentUse >= 0.75:
retData += 2
elif percentUse >= 0.95:
retData += 3
return retData
def getDifficultyForNewBlock(data, ourBlock=True):
'''
Get difficulty for block. Accepts size in integer, Block instance, or str/bytes full block contents
'''
retData = 0
dataSize = 0
if isinstance(data, onionrblockapi.Block):
dataSize = len(data.getRaw().encode('utf-8'))
elif isinstance(data, str):
dataSize = len(data.encode('utf-8'))
elif isinstance(data, int):
dataSize = data
else:
raise ValueError('not Block, str, or int')
if ourBlock:
minDifficulty = config.get('general.minimum_send_pow')
else:
minDifficulty = config.get('general.minimum_block_pow')
retData = max(minDifficulty, math.floor(dataSize / 1000000)) + getDifficultyModifier()
return retData
def getHashDifficulty(h): def getHashDifficulty(h):
''' '''
@ -55,6 +103,7 @@ class DataPOW:
self.difficulty = 0 self.difficulty = 0
self.data = data self.data = data
self.threadCount = threadCount self.threadCount = threadCount
self.rounds = 0
config.reload() config.reload()
if forceDifficulty == 0: if forceDifficulty == 0:
@ -96,6 +145,7 @@ class DataPOW:
while self.hashing: while self.hashing:
rand = nacl.utils.random() rand = nacl.utils.random()
token = nacl.hash.blake2b(rand + self.data).decode() token = nacl.hash.blake2b(rand + self.data).decode()
self.rounds += 1
#print(token) #print(token)
if self.puzzle == token[0:self.difficulty]: if self.puzzle == token[0:self.difficulty]:
self.hashing = False self.hashing = False
@ -106,6 +156,7 @@ class DataPOW:
endTime = math.floor(time.time()) endTime = math.floor(time.time())
if self.reporting: if self.reporting:
logger.debug('Found token after %s seconds: %s' % (endTime - startTime, token), timestamp=True) logger.debug('Found token after %s seconds: %s' % (endTime - startTime, token), timestamp=True)
logger.debug('Round count: %s' % (self.rounds,))
self.result = (token, rand) self.result = (token, rand)
def shutdown(self): def shutdown(self):
@ -146,17 +197,27 @@ class DataPOW:
return result return result
class POW: class POW:
def __init__(self, metadata, data, threadCount = 5): def __init__(self, metadata, data, threadCount = 5, forceDifficulty=0, coreInst=None):
self.foundHash = False self.foundHash = False
self.difficulty = 0 self.difficulty = 0
self.data = data self.data = data
self.metadata = metadata self.metadata = metadata
self.threadCount = threadCount self.threadCount = threadCount
try:
assert isinstance(coreInst, core.Core)
except AssertionError:
myCore = core.Core()
else:
myCore = coreInst
dataLen = len(data) + len(json.dumps(metadata)) dataLen = len(data) + len(json.dumps(metadata))
self.difficulty = math.floor(dataLen / 1000000)
if self.difficulty <= 2: if forceDifficulty > 0:
self.difficulty = int(config.get('general.minimum_block_pow')) self.difficulty = forceDifficulty
else:
# Calculate difficulty. Dumb for now, may use good algorithm in the future.
self.difficulty = getDifficultyForNewBlock(dataLen)
try: try:
self.data = self.data.encode() self.data = self.data.encode()
@ -168,7 +229,6 @@ class POW:
self.mainHash = '0' * 64 self.mainHash = '0' * 64
self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))] self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))]
myCore = core.Core()
for i in range(max(1, threadCount)): for i in range(max(1, threadCount)):
t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,myCore)) t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,myCore))
t.start() t.start()

View File

@ -155,18 +155,21 @@ class OnionrUtils:
''' '''
Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers. Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers.
''' '''
config.reload() config.reload()
self.getTimeBypassToken() self.getTimeBypassToken()
# TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless. # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
hostname = '' hostname = ''
maxWait = 5
waited = 0
while hostname == '': while hostname == '':
try: try:
with open(self._core.privateApiHostFile, 'r') as host: with open(self._core.privateApiHostFile, 'r') as host:
hostname = host.read() hostname = host.read()
except FileNotFoundError: except FileNotFoundError:
print('wat')
time.sleep(1) time.sleep(1)
waited += 1
if waited == maxWait:
return False
if data != '': if data != '':
data = '&data=' + urllib.parse.quote_plus(data) data = '&data=' + urllib.parse.quote_plus(data)
payload = 'http://%s:%s/%s%s' % (hostname, config.get('client.client.port'), command, data) payload = 'http://%s:%s/%s%s' % (hostname, config.get('client.client.port'), command, data)

View File

@ -1,14 +1,13 @@
{ {
"general" : { "general" : {
"dev_mode" : true, "dev_mode" : true,
"display_header" : true, "display_header" : false,
"minimum_block_pow": 5, "minimum_block_pow": 4,
"minimum_send_pow": 5, "minimum_send_pow": 4,
"socket_servers": false, "socket_servers": false,
"security_level": 0, "security_level": 0,
"max_block_age": 2678400, "max_block_age": 2678400,
"public_key": "", "public_key": ""
"use_new_api_server": false
}, },
"www" : { "www" : {
@ -70,7 +69,7 @@
}, },
"allocations" : { "allocations" : {
"disk" : 10000000000, "disk" : 2000,
"net_total" : 1000000000, "net_total" : 1000000000,
"blockCache" : 5000000, "blockCache" : 5000000,
"blockCacheTotal" : 50000000 "blockCacheTotal" : 50000000

View File

@ -44,6 +44,11 @@ class StorageCounter:
pass pass
return retData return retData
def getPercent(self):
'''Return percent (decimal/float) of disk space we're using'''
amount = self.getAmount()
return round(amount / self._core.config.get('allocations.disk'), 2)
def addBytes(self, amount): def addBytes(self, amount):
'''Record that we are now using more disk space, unless doing so would exceed configured max''' '''Record that we are now using more disk space, unless doing so would exceed configured max'''
newAmount = amount + self.getAmount() newAmount = amount + self.getAmount()