From 4b8fe7eeb3ba341ea9b6a164e62a70737c9146c0 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 16 Sep 2019 02:50:13 -0500 Subject: [PATCH] bumped min python version to 3.7, added upload sessions to use in preventing infinite upload fails --- onionr/communicator/__init__.py | 9 ++-- onionr/communicator/uploadqueue/__init__.py | 18 +++---- .../__init__.py} | 5 +- .../communicatorutils/uploadblocks/session.py | 51 +++++++++++++++++++ .../uploadblocks/sessionmanager.py | 0 onionr/etc/onionrvalues.py | 2 +- tests/test_peerprofiles.py | 2 +- tests/test_upload_session.py | 36 +++++++++++++ 8 files changed, 105 insertions(+), 18 deletions(-) rename onionr/communicatorutils/{uploadblocks.py => uploadblocks/__init__.py} (97%) create mode 100644 onionr/communicatorutils/uploadblocks/session.py create mode 100644 onionr/communicatorutils/uploadblocks/sessionmanager.py create mode 100644 tests/test_upload_session.py diff --git a/onionr/communicator/__init__.py b/onionr/communicator/__init__.py index 0a853f3a..ff4d1853 100755 --- a/onionr/communicator/__init__.py +++ b/onionr/communicator/__init__.py @@ -24,7 +24,8 @@ import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as p from . import onlinepeers, uploadqueue from communicatorutils import servicecreator, onionrcommunicatortimers from communicatorutils import downloadblocks, lookupblocks, lookupadders -from communicatorutils import servicecreator, connectnewpeers, uploadblocks +from communicatorutils import servicecreator, connectnewpeers +from communicatorutils import uploadblocks from communicatorutils import daemonqueuehandler, announcenode, deniableinserts from communicatorutils import cooldownpeer, housekeeping, netcheck from onionrutils import localcommand, epoch @@ -50,6 +51,7 @@ class OnionrCommunicatorDaemon: # initialize core with Tor socks port being 3rd argument self.proxyPort = shared_state.get(NetController).socksPort + # Upload information, list of blocks to upload self.blocksToUpload = [] # loop time.sleep delay in seconds @@ -241,11 +243,6 @@ class OnionrCommunicatorDaemon: logger.debug('Heartbeat. Node running for %s.' % humanreadabletime.human_readable_time(self.getUptime())) self.decrementThreadCount('heartbeat') - def announce(self, peer): - '''Announce to peers our address''' - if announcenode.announce_node(self) == False: - logger.warn('Could not introduce node.', terminal=True) - def runCheck(self): if run_file_exists(self): logger.debug('Status check; looks good.') diff --git a/onionr/communicator/uploadqueue/__init__.py b/onionr/communicator/uploadqueue/__init__.py index 0143425e..e6d19b81 100644 --- a/onionr/communicator/uploadqueue/__init__.py +++ b/onionr/communicator/uploadqueue/__init__.py @@ -1,9 +1,9 @@ -''' +""" Onionr - Private P2P Communication Class to remember blocks that need to be uploaded and not shared on startup/shutdown -''' -''' +""" +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -16,7 +16,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' +""" import atexit import json @@ -33,14 +33,14 @@ def _add_to_hidden_blocks(cache): localcommand.local_command('waitforshare/' + bl, post=True) class UploadQueue: - ''' + """ Saves and loads block upload info from json file - ''' + """ def __init__(self, communicator: 'OnionrCommunicatorDaemon'): - '''Start the UploadQueue object, loading left over uploads into queue + """Start the UploadQueue object, loading left over uploads into queue and registering save shutdown function - ''' + """ self.communicator = communicator cache = deadsimplekv.DeadSimpleKV(UPLOAD_MEMORY_FILE) self.store_obj = cache @@ -54,7 +54,7 @@ class UploadQueue: atexit.register(self.save) def save(self): - '''Saves to disk on shutdown or if called manually''' + """Saves to disk on shutdown or if called manually""" bl: list = self.communicator.blocksToUpload self.store_obj.put('uploads', bl) self.store_obj.flush() diff --git a/onionr/communicatorutils/uploadblocks.py b/onionr/communicatorutils/uploadblocks/__init__.py similarity index 97% rename from onionr/communicatorutils/uploadblocks.py rename to onionr/communicatorutils/uploadblocks/__init__.py index 647b4943..19b43c9b 100755 --- a/onionr/communicatorutils/uploadblocks.py +++ b/onionr/communicatorutils/uploadblocks/__init__.py @@ -24,6 +24,9 @@ import onionrblockapi as block from onionrutils import localcommand, stringvalidators, basicrequests from communicator import onlinepeers import onionrcrypto + +from . import session + def upload_blocks_from_communicator(comm_inst): # when inserting a block, we try to upload it to a few peers to add some deniability TIMER_NAME = "upload_blocks_from_communicator" @@ -42,7 +45,7 @@ def upload_blocks_from_communicator(comm_inst): if peer in triedPeers: continue triedPeers.append(peer) - url = 'http://' + peer + '/upload' + url = 'http://%s/upload' % (peer,) try: #data = {'block': block.Block(bl).getRaw()} data = block.Block(bl).getRaw() diff --git a/onionr/communicatorutils/uploadblocks/session.py b/onionr/communicatorutils/uploadblocks/session.py new file mode 100644 index 00000000..24415975 --- /dev/null +++ b/onionr/communicatorutils/uploadblocks/session.py @@ -0,0 +1,51 @@ +""" + Onionr - Private P2P Communication + + Virtual upload "sessions" for blocks +""" +""" + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" +from typing import Union + +from onionrutils import stringvalidators +from onionrutils import bytesconverter +from onionrutils import epoch +from utils import reconstructhash + +class UploadSession: + """Manages statistics for an Onionr block upload session + + accepting a block hash (incl. unpadded) as an argument""" + def __init__(self, block_hash: Union[str, bytes]): + block_hash = bytesconverter.bytes_to_str(block_hash) + block_hash = reconstructhash.reconstruct_hash(block_hash) + if not stringvalidators.validate_hash(block_hash): raise ValueError + + self.block_hash = reconstructhash.deconstruct_hash(block_hash) + self.total_fail_count: int = 0 + self.total_success_count: int = 0 + self.peer_fails = {} + + def fail_peer(self, peer): + try: + self.peer_fails[peer] += 1 + except KeyError: + self.peer_fails[peer] = 0 + + def fail(self): + self.total_fail_count += 1 + + def success(self): + self.total_success_count += 1 diff --git a/onionr/communicatorutils/uploadblocks/sessionmanager.py b/onionr/communicatorutils/uploadblocks/sessionmanager.py new file mode 100644 index 00000000..e69de29b diff --git a/onionr/etc/onionrvalues.py b/onionr/etc/onionrvalues.py index 6fb225bc..66075c0a 100755 --- a/onionr/etc/onionrvalues.py +++ b/onionr/etc/onionrvalues.py @@ -24,7 +24,7 @@ ONIONR_TAGLINE = 'Private P2P Communication - GPLv3 - https://Onionr.net' ONIONR_VERSION = '0.0.0' # for debugging and stuff ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION) API_VERSION = '0' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you. -MIN_PY_VERSION = 6 +MIN_PY_VERSION = 7 DEVELOPMENT_MODE = True MAX_BLOCK_TYPE_LENGTH = 15 MAX_BLOCK_CLOCK_SKEW = 120 diff --git a/tests/test_peerprofiles.py b/tests/test_peerprofiles.py index f8c0d57c..c1877bfd 100644 --- a/tests/test_peerprofiles.py +++ b/tests/test_peerprofiles.py @@ -52,7 +52,7 @@ class TestPeerProfiles(unittest.TestCase): p.addScore(1) self.assertEqual(p.score, keydb.transportinfo.get_address_info(p.address, 'success')) - def test_inc_score_with_sync_Delay(self): + def test_inc_score_with_sync_delay(self): p = peerprofiles.PeerProfiles(test_peers.pop()) s = 0 for x in range(2): diff --git a/tests/test_upload_session.py b/tests/test_upload_session.py new file mode 100644 index 00000000..d05f43b6 --- /dev/null +++ b/tests/test_upload_session.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +import sys, os +sys.path.append(".") +sys.path.append("onionr/") +import unittest, uuid +TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' +print("Test directory:", TEST_DIR) +os.environ["ONIONR_HOME"] = TEST_DIR +import hashlib +from communicatorutils import uploadblocks + +def hash_generator(): + hasher = hashlib.sha3_256() + hasher.update(os.urandom(15)) + return hasher.hexdigest() + +test_hashes = [] +for x in range(100): test_hashes.append(hash_generator()) + +class UploadSessionTest(unittest.TestCase): + def test_init_fail(self): + s = test_hashes.pop() + s = uploadblocks.session.UploadSession(s) + self.assertEqual(s.total_fail_count, 0) + + def test_init_success(self): + s = test_hashes.pop() + s = uploadblocks.session.UploadSession(s) + self.assertEqual(s.total_success_count, 0) + + def test_invalid(self): + invalid = [None, 1, -1, 0, 'ab43c5b8c7b9b037d4f02fa6bc77dbb522bfcbcd7e8ea2953bf2252c6e9232a8b', lambda: None, True, False] + for x in invalid: + self.assertRaises((ValueError, AttributeError), uploadblocks.session.UploadSession, x) + +unittest.main()