From ee9023b15005d1e3421d795255bcaaa6cc69a12e Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 22 Jan 2019 11:40:27 -0600 Subject: [PATCH] start using very simple DHT --- onionr/communicator2.py | 31 ++++++++++++++++++++++--------- onionr/onionrcrypto.py | 17 ++++++++++++++++- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 632142d4..bda9e29a 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -65,7 +65,7 @@ class OnionrCommunicatorDaemon: self.shutdown = False # list of new blocks to download, added to when new block lists are fetched from peers - self.blockQueue = [] + self.blockQueue = {} # list of blocks currently downloading, avoid s self.currentDownloading = [] @@ -216,16 +216,24 @@ class OnionrCommunicatorDaemon: # if newline seperated string is valid hash if not i in existingBlocks: # if block does not exist on disk and is not already in block queue - if i not in self.blockQueue and not self._core._blacklist.inBlacklist(i): - if onionrproofs.hashMeetsDifficulty(i): + if i not in self.blockQueue: + if onionrproofs.hashMeetsDifficulty(i) and not self._core._blacklist.inBlacklist(i): if len(self.blockQueue) <= 1000000: - self.blockQueue.append(i) # add blocks to download queue + self.blockQueue[i] = [peer] # add blocks to download queue + else: + if peer not in self.blockQueue[i]: + self.blockQueue[i].append(peer) self.decrementThreadCount('lookupBlocks') return def getBlocks(self): '''download new blocks in queue''' - for blockHash in self.blockQueue: + for blockHash in list(self.blockQueue): + triedQueuePeers = [] # List of peers we've tried for a block + try: + blockPeers = list(self.blockQueue[blockHash]) + except KeyError: + blockPeers = [] removeFromQueue = True if self.shutdown or not self.isOnline: # Exit loop if shutting down or offline @@ -236,14 +244,19 @@ class OnionrCommunicatorDaemon: continue if blockHash in self._core.getBlockList(): logger.debug('Block %s is already saved.' % (blockHash,)) - self.blockQueue.remove(blockHash) + del self.blockQueue[blockHash] continue if self._core._blacklist.inBlacklist(blockHash): continue if self._core._utils.storageCounter.isFull(): break self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block - peerUsed = self.pickOnlinePeer() + if len(blockPeers) == 0: + peerUsed = self.pickOnlinePeer() + else: + blockPeers = self._core._crypto.randomShuffle(blockPeers) + peerUsed = blockPeers.pop(0) + if not self.shutdown and peerUsed.strip() != '': logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed)) content = self.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata) @@ -300,8 +313,8 @@ class OnionrCommunicatorDaemon: removeFromQueue = False # Don't remove from queue if 404 if removeFromQueue: try: - self.blockQueue.remove(blockHash) # remove from block queue both if success or false - except ValueError: + del self.blockQueue[blockHash] # remove from block queue both if success or false + except KeyError: pass self.currentDownloading.remove(blockHash) self.decrementThreadCount('getBlocks') diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index 8c15f144..074c8aa1 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -281,5 +281,20 @@ class OnionrCrypto: return retData - def safeCompare(self, one, two): + @staticmethod + def safeCompare(one, two): return hmac.compare_digest(one, two) + + @staticmethod + def randomShuffle(theList): + myList = list(theList) + shuffledList = [] + myListLength = len(myList) + 1 + while myListLength > 0: + removed = secrets.randbelow(myListLength) + try: + shuffledList.append(myList.pop(removed)) + except IndexError: + pass + myListLength = len(myList) + return shuffledList \ No newline at end of file