start using very simple DHT

This commit is contained in:
Kevin Froman 2019-01-22 11:40:27 -06:00
parent aef6d5d8e6
commit ee9023b150
2 changed files with 38 additions and 10 deletions

View File

@ -65,7 +65,7 @@ class OnionrCommunicatorDaemon:
self.shutdown = False self.shutdown = False
# list of new blocks to download, added to when new block lists are fetched from peers # list of new blocks to download, added to when new block lists are fetched from peers
self.blockQueue = [] self.blockQueue = {}
# list of blocks currently downloading, avoid s # list of blocks currently downloading, avoid s
self.currentDownloading = [] self.currentDownloading = []
@ -216,16 +216,24 @@ class OnionrCommunicatorDaemon:
# if newline seperated string is valid hash # if newline seperated string is valid hash
if not i in existingBlocks: if not i in existingBlocks:
# if block does not exist on disk and is not already in block queue # if block does not exist on disk and is not already in block queue
if i not in self.blockQueue and not self._core._blacklist.inBlacklist(i): if i not in self.blockQueue:
if onionrproofs.hashMeetsDifficulty(i): if onionrproofs.hashMeetsDifficulty(i) and not self._core._blacklist.inBlacklist(i):
if len(self.blockQueue) <= 1000000: if len(self.blockQueue) <= 1000000:
self.blockQueue.append(i) # add blocks to download queue self.blockQueue[i] = [peer] # add blocks to download queue
else:
if peer not in self.blockQueue[i]:
self.blockQueue[i].append(peer)
self.decrementThreadCount('lookupBlocks') self.decrementThreadCount('lookupBlocks')
return return
def getBlocks(self): def getBlocks(self):
'''download new blocks in queue''' '''download new blocks in queue'''
for blockHash in self.blockQueue: for blockHash in list(self.blockQueue):
triedQueuePeers = [] # List of peers we've tried for a block
try:
blockPeers = list(self.blockQueue[blockHash])
except KeyError:
blockPeers = []
removeFromQueue = True removeFromQueue = True
if self.shutdown or not self.isOnline: if self.shutdown or not self.isOnline:
# Exit loop if shutting down or offline # Exit loop if shutting down or offline
@ -236,14 +244,19 @@ class OnionrCommunicatorDaemon:
continue continue
if blockHash in self._core.getBlockList(): if blockHash in self._core.getBlockList():
logger.debug('Block %s is already saved.' % (blockHash,)) logger.debug('Block %s is already saved.' % (blockHash,))
self.blockQueue.remove(blockHash) del self.blockQueue[blockHash]
continue continue
if self._core._blacklist.inBlacklist(blockHash): if self._core._blacklist.inBlacklist(blockHash):
continue continue
if self._core._utils.storageCounter.isFull(): if self._core._utils.storageCounter.isFull():
break break
self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
if len(blockPeers) == 0:
peerUsed = self.pickOnlinePeer() peerUsed = self.pickOnlinePeer()
else:
blockPeers = self._core._crypto.randomShuffle(blockPeers)
peerUsed = blockPeers.pop(0)
if not self.shutdown and peerUsed.strip() != '': if not self.shutdown and peerUsed.strip() != '':
logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed)) logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed))
content = self.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata) content = self.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata)
@ -300,8 +313,8 @@ class OnionrCommunicatorDaemon:
removeFromQueue = False # Don't remove from queue if 404 removeFromQueue = False # Don't remove from queue if 404
if removeFromQueue: if removeFromQueue:
try: try:
self.blockQueue.remove(blockHash) # remove from block queue both if success or false del self.blockQueue[blockHash] # remove from block queue both if success or false
except ValueError: except KeyError:
pass pass
self.currentDownloading.remove(blockHash) self.currentDownloading.remove(blockHash)
self.decrementThreadCount('getBlocks') self.decrementThreadCount('getBlocks')

View File

@ -281,5 +281,20 @@ class OnionrCrypto:
return retData return retData
def safeCompare(self, one, two): @staticmethod
def safeCompare(one, two):
return hmac.compare_digest(one, two) return hmac.compare_digest(one, two)
@staticmethod
def randomShuffle(theList):
myList = list(theList)
shuffledList = []
myListLength = len(myList) + 1
while myListLength > 0:
removed = secrets.randbelow(myListLength)
try:
shuffledList.append(myList.pop(removed))
except IndexError:
pass
myListLength = len(myList)
return shuffledList