From 6e55636e781b289ff8dd59798218c7a223daee41 Mon Sep 17 00:00:00 2001 From: Arinerron Date: Mon, 24 Sep 2018 16:48:00 -0700 Subject: [PATCH] test --- onionr/api.py | 12 +++-- onionr/communicator2.py | 18 ++++---- onionr/core.py | 8 ++-- onionr/netcontroller.py | 2 +- onionr/onionrblockapi.py | 2 +- onionr/onionrdaemontools.py | 8 ++-- onionr/onionrpeers.py | 8 ++-- onionr/onionrutils.py | 12 ++--- onionr/static-data/default_config.json | 61 ++++++++++++++------------ 9 files changed, 72 insertions(+), 59 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index 761bb0a0..ec540814 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -150,14 +150,17 @@ class API: self.validateHost('private') + if config.get('www.public.guess_mime', True): + self.mimeType = API.guessMime(path) + endTime = math.floor(time.time()) elapsed = endTime - startTime if not hmac.compare_digest(timingToken, self.timeBypassToken): - if elapsed < self._privateDelayTime: + if (elapsed < self._privateDelayTime) and config.get('www.private.timing_protection', True): time.sleep(self._privateDelayTime - elapsed) - return send_from_directory('static-data/www/private/', path) + return send_from_directory(config.get('www.private.path', 'static-data/www/private/'), path) @app.route('/www/public/') def www_public(path): @@ -166,7 +169,10 @@ class API: self.validateHost('public') - return send_from_directory('static-data/www/public/', path) + if config.get('www.public.guess_mime', True): + self.mimeType = API.guessMime(path) + + return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path) @app.route('/ui/') def ui_private(path): diff --git a/onionr/communicator2.py b/onionr/communicator2.py index f203dd70..4a80e6f7 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -87,8 +87,8 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.daemonCommands, 5) OnionrCommunicatorTimers(self, self.detectAPICrash, 5) peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60, maxThreads=1) - OnionrCommunicatorTimers(self, self.lookupBlocks, self._core.config.get('timers.lookupBlocks'), requiresPeer=True, maxThreads=1) - OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.getBlocks'), requiresPeer=True) + OnionrCommunicatorTimers(self, self.lookupBlocks, self._core.config.get('timers.lookup_blocks'), requiresPeer=True, maxThreads=1) + OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.get_blocks'), requiresPeer=True) OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58) OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 65) OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True) @@ -252,7 +252,7 @@ class OnionrCommunicatorDaemon: except AttributeError: pass # Punish peer for sharing invalid block (not always malicious, but is bad regardless) - onionrpeers.PeerProfiles(peerUsed, self._core).addScore(-50) + onionrpeers.PeerProfiles(peerUsed, self._core).addScore(-50) logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash) if removeFromQueue: self.blockQueue.remove(blockHash) # remove from block queue both if success or false @@ -298,7 +298,7 @@ class OnionrCommunicatorDaemon: '''Manages the self.onlinePeers attribute list, connects to more peers if we have none connected''' logger.info('Refreshing peer pool.') - maxPeers = int(config.get('peers.maxConnect')) + maxPeers = int(config.get('peers.max_connect', 10)) needed = maxPeers - len(self.onlinePeers) for i in range(needed): @@ -331,7 +331,7 @@ class OnionrCommunicatorDaemon: raise onionrexceptions.InvalidAddress('Will not attempt connection test to invalid address') else: peerList = self._core.listAdders() - + peerList = onionrpeers.getScoreSortedPeerList(self._core) if len(peerList) == 0 or useBootstrap: @@ -339,7 +339,7 @@ class OnionrCommunicatorDaemon: self.addBootstrapListToPeerList(peerList) for address in peerList: - if not config.get('tor.v3onions') and len(address) == 62: + if not config.get('tor.v3_onions') and len(address) == 62: continue if len(address) == 0 or address in tried or address in self.onlinePeers or address in self.cooldownPeer: continue @@ -352,7 +352,7 @@ class OnionrCommunicatorDaemon: self.onlinePeers.append(address) self.connectTimes[address] = self._core._utils.getEpoch() retData = address - + # add peer to profile list if they're not in it for profile in self.peerProfiles: if profile.address == address: @@ -416,7 +416,7 @@ class OnionrCommunicatorDaemon: self._core.setAddressInfo(peer, 'lastConnect', self._core._utils.getEpoch()) self.getPeerProfileInstance(peer).addScore(1) return retData - + def getPeerProfileInstance(self, peer): '''Gets a peer profile instance from the list of profiles, by address name''' for i in self.peerProfiles: @@ -543,7 +543,7 @@ class OnionrCommunicatorTimers: if self.makeThread: for i in range(self.threadAmount): if self.daemonInstance.threadCounts[self.timerFunction.__name__] >= self.maxThreads: - logger.warn(self.timerFunction.__name__ + ' has too many current threads to start anymore.') + logger.warn('%s is currently using the maximum number of threads, not starting another.' % self.timerFunction.__name__) else: self.daemonInstance.threadCounts[self.timerFunction.__name__] += 1 newThread = threading.Thread(target=self.timerFunction) diff --git a/onionr/core.py b/onionr/core.py index ab8b640b..bff27e45 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -126,7 +126,7 @@ class Core: ''' Add an address to the address database (only tor currently) ''' - if address == config.get('i2p.ownAddr', None): + if address == config.get('i2p.own_addr', None): return False if self._utils.validateID(address): @@ -190,7 +190,7 @@ class Core: blockFile = 'data/blocks/' + block + '.dat' dataSize = 0 try: - ''' Get size of data when loaded as an object/var, rather than on disk, + ''' Get size of data when loaded as an object/var, rather than on disk, to avoid conflict with getsizeof when saving blocks ''' with open(blockFile, 'r') as data: @@ -273,7 +273,7 @@ class Core: if not type(data) is bytes: data = data.encode() - + dataHash = self._getSha3Hash(data) if type(dataHash) is bytes: @@ -722,7 +722,7 @@ class Core: metadata['sig'] = signature metadata['signer'] = signer metadata['time'] = str(self._utils.getEpoch()) - + # send block data (and metadata) to POW module to get tokenized block data proof = onionrproofs.POW(metadata, data) payload = proof.waitForResult() diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py index 3749ce7a..75034f27 100644 --- a/onionr/netcontroller.py +++ b/onionr/netcontroller.py @@ -49,7 +49,7 @@ class NetController: Generate a torrc file for our tor instance ''' hsVer = '# v2 onions' - if config.get('tor.v3onions'): + if config.get('tor.v3_onions'): hsVer = 'HiddenServiceVersion 3' logger.info('Using v3 onions :)') if os.path.exists(self.torConfigLocation): diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index 97b34730..825e61e5 100644 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -762,7 +762,7 @@ class Block: return False # dump old cached blocks if the size exeeds the maximum - if sys.getsizeof(Block.blockCacheOrder) >= config.get('allocations.blockCacheTotal', 50000000): # 50MB default cache size + if sys.getsizeof(Block.blockCacheOrder) >= config.get('allocations.block_cache_total', 50000000): # 50MB default cache size del Block.blockCache[blockCacheOrder.pop(0)] # cache block content diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index bbd7af64..f2c53fa7 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -63,14 +63,14 @@ class DaemonTools: logger.warn('Network check failed, are you connected to the internet?') self.daemon.isOnline = False self.daemon.decrementThreadCount('netCheck') - + def cleanOldBlocks(self): '''Delete old blocks if our disk allocation is full/near full''' while self.daemon._core._utils.storageCounter.isFull(): oldest = self.daemon._core.getBlockList()[0] self.daemon._core._blacklist.addToDB(oldest) self.daemon._core.removeBlock(oldest) - logger.info('Deleted block: %s' % (oldest,)) + logger.info('Deleted block: %s' % (oldest,)) self.daemon.decrementThreadCount('cleanOldBlocks') def cooldownPeer(self): @@ -88,7 +88,7 @@ class DaemonTools: del self.daemon.cooldownPeer[peer] # Cool down a peer, if we have max connections alive for long enough - if onlinePeerAmount >= self.daemon._core.config.get('peers.maxConnect'): + if onlinePeerAmount >= self.daemon._core.config.get('peers.max_connect', 10): finding = True while finding: try: @@ -102,4 +102,4 @@ class DaemonTools: else: self.daemon.removeOnlinePeer(toCool) self.daemon.cooldownPeer[toCool] = self.daemon._core._utils.getEpoch() - self.daemon.decrementThreadCount('cooldownPeer') \ No newline at end of file + self.daemon.decrementThreadCount('cooldownPeer') diff --git a/onionr/onionrpeers.py b/onionr/onionrpeers.py index 322db9ba..62716eee 100644 --- a/onionr/onionrpeers.py +++ b/onionr/onionrpeers.py @@ -44,7 +44,7 @@ class PeerProfiles: except (TypeError, ValueError) as e: self.success = 0 self.score = self.success - + def saveScore(self): '''Save the node's score to the database''' self.coreInst.setAddressInfo(self.address, 'success', self.score) @@ -79,8 +79,8 @@ def peerCleanup(coreInst): logger.info('Cleaning peers...') config.reload() - minScore = int(config.get('peers.minimumScore')) - maxPeers = int(config.get('peers.maxStoredPeers')) + minScore = int(config.get('peers.minimum_score', -100)) + maxPeers = int(config.get('peers.max_stored', 5000)) adders = getScoreSortedPeerList(coreInst) adders.reverse() @@ -102,4 +102,4 @@ def peerCleanup(coreInst): logger.warn('Removed address ' + address + '.') # Unban probably not malicious peers TODO improve - coreInst._blacklist.deleteExpired(dataType=1) \ No newline at end of file + coreInst._blacklist.deleteExpired(dataType=1) diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index a383684c..8f0ebcca 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -131,11 +131,11 @@ class OnionrUtils: for adder in newAdderList.split(','): adder = adder.strip() if not adder in self._core.listAdders(randomOrder = False) and adder != self.getMyAddress() and not self._core._blacklist.inBlacklist(adder): - if not config.get('tor.v3onions') and len(adder) == 62: + if not config.get('tor.v3_onions') and len(adder) == 62: continue if self._core.addAddress(adder): # Check if we have the maxmium amount of allowed stored peers - if config.get('peers.maxStoredPeers') > len(self._core.listAdders()): + if config.get('peers.max_stored') > len(self._core.listAdders()): logger.info('Added %s to db.' % adder, timestamp = True) retVal = True else: @@ -635,7 +635,7 @@ class OnionrUtils: else: self.powSalt = retData return retData - + def strToBytes(self, data): try: data = data.encode() @@ -648,7 +648,7 @@ class OnionrUtils: except AttributeError: pass return data - + def checkNetwork(self, torPort=0): '''Check if we are connected to the internet (through Tor)''' retData = False @@ -656,7 +656,7 @@ class OnionrUtils: try: with open('static-data/connect-check.txt', 'r') as connectTest: connectURLs = connectTest.read().split(',') - + for url in connectURLs: if self.doGetRequest(url, port=torPort) != False: retData = True @@ -689,4 +689,4 @@ def humanSize(num, suffix='B'): if abs(num) < 1024.0: return "%.1f %s%s" % (num, unit, suffix) num /= 1024.0 - return "%.1f %s%s" % (num, 'Yi', suffix) \ No newline at end of file + return "%.1f %s%s" % (num, 'Yi', suffix) diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 5ebcfa4a..a902469e 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -1,6 +1,6 @@ { "general" : { - "dev_mode": true, + "dev_mode" : true, "display_header" : true, "direct_connect" : { @@ -11,11 +11,16 @@ "www" : { "public" : { - "run" : true + "run" : true, + "path" : "static-data/www/public/", + "guess_mime" : true }, "private" : { - "run" : true + "run" : true, + "path" : "static-data/www/private/", + "guess_mime" : true, + "timing_protection" : true }, "ui" : { @@ -28,41 +33,43 @@ }, - "log": { - "file": { - "output": true, - "path": "data/output.log" + "log" : { + "file" : { + "output" : true, + "path" : "data/output.log" }, - "console": { - "output": true, - "color": true + "console" : { + "output" : true, + "color" : true } }, "tor" : { - "v3onions": false + "v3onions" : false }, - "i2p":{ - "host": false, - "connect": true, - "ownAddr": "" + "i2p" : { + "host" : false, + "connect" : true, + "own_addr" : "" }, - "allocations":{ - "disk": 10000000000, - "netTotal": 1000000000, - "blockCache": 5000000, - "blockCacheTotal": 50000000 + "allocations" : { + "disk" : 10000000000, + "net_total" : 1000000000, + "blockCache" : 5000000, + "blockCacheTotal" : 50000000 }, - "peers":{ - "minimumScore": -100, - "maxStoredPeers": 5000, - "maxConnect": 10 + + "peers" : { + "minimum_score" : -100, + "max_stored_peers" : 5000, + "max_connect" : 10 }, - "timers":{ - "lookupBlocks": 25, - "getBlocks": 30 + + "timers" : { + "lookup_blocks" : 25, + "get_blocks" : 30 } }