bumped min python version to 3.7, added upload sessions to use in preventing infinite upload fails

This commit is contained in:
Kevin Froman 2019-09-16 02:50:13 -05:00
parent 376b2cc2d6
commit 4b8fe7eeb3
8 changed files with 105 additions and 18 deletions

View File

@ -24,7 +24,8 @@ import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as p
from . import onlinepeers, uploadqueue from . import onlinepeers, uploadqueue
from communicatorutils import servicecreator, onionrcommunicatortimers from communicatorutils import servicecreator, onionrcommunicatortimers
from communicatorutils import downloadblocks, lookupblocks, lookupadders from communicatorutils import downloadblocks, lookupblocks, lookupadders
from communicatorutils import servicecreator, connectnewpeers, uploadblocks from communicatorutils import servicecreator, connectnewpeers
from communicatorutils import uploadblocks
from communicatorutils import daemonqueuehandler, announcenode, deniableinserts from communicatorutils import daemonqueuehandler, announcenode, deniableinserts
from communicatorutils import cooldownpeer, housekeeping, netcheck from communicatorutils import cooldownpeer, housekeeping, netcheck
from onionrutils import localcommand, epoch from onionrutils import localcommand, epoch
@ -50,6 +51,7 @@ class OnionrCommunicatorDaemon:
# initialize core with Tor socks port being 3rd argument # initialize core with Tor socks port being 3rd argument
self.proxyPort = shared_state.get(NetController).socksPort self.proxyPort = shared_state.get(NetController).socksPort
# Upload information, list of blocks to upload
self.blocksToUpload = [] self.blocksToUpload = []
# loop time.sleep delay in seconds # loop time.sleep delay in seconds
@ -241,11 +243,6 @@ class OnionrCommunicatorDaemon:
logger.debug('Heartbeat. Node running for %s.' % humanreadabletime.human_readable_time(self.getUptime())) logger.debug('Heartbeat. Node running for %s.' % humanreadabletime.human_readable_time(self.getUptime()))
self.decrementThreadCount('heartbeat') self.decrementThreadCount('heartbeat')
def announce(self, peer):
'''Announce to peers our address'''
if announcenode.announce_node(self) == False:
logger.warn('Could not introduce node.', terminal=True)
def runCheck(self): def runCheck(self):
if run_file_exists(self): if run_file_exists(self):
logger.debug('Status check; looks good.') logger.debug('Status check; looks good.')

View File

@ -1,9 +1,9 @@
''' """
Onionr - Private P2P Communication Onionr - Private P2P Communication
Class to remember blocks that need to be uploaded and not shared on startup/shutdown Class to remember blocks that need to be uploaded and not shared on startup/shutdown
''' """
''' """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or the Free Software Foundation, either version 3 of the License, or
@ -16,7 +16,7 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' """
import atexit import atexit
import json import json
@ -33,14 +33,14 @@ def _add_to_hidden_blocks(cache):
localcommand.local_command('waitforshare/' + bl, post=True) localcommand.local_command('waitforshare/' + bl, post=True)
class UploadQueue: class UploadQueue:
''' """
Saves and loads block upload info from json file Saves and loads block upload info from json file
''' """
def __init__(self, communicator: 'OnionrCommunicatorDaemon'): def __init__(self, communicator: 'OnionrCommunicatorDaemon'):
'''Start the UploadQueue object, loading left over uploads into queue """Start the UploadQueue object, loading left over uploads into queue
and registering save shutdown function and registering save shutdown function
''' """
self.communicator = communicator self.communicator = communicator
cache = deadsimplekv.DeadSimpleKV(UPLOAD_MEMORY_FILE) cache = deadsimplekv.DeadSimpleKV(UPLOAD_MEMORY_FILE)
self.store_obj = cache self.store_obj = cache
@ -54,7 +54,7 @@ class UploadQueue:
atexit.register(self.save) atexit.register(self.save)
def save(self): def save(self):
'''Saves to disk on shutdown or if called manually''' """Saves to disk on shutdown or if called manually"""
bl: list = self.communicator.blocksToUpload bl: list = self.communicator.blocksToUpload
self.store_obj.put('uploads', bl) self.store_obj.put('uploads', bl)
self.store_obj.flush() self.store_obj.flush()

View File

@ -24,6 +24,9 @@ import onionrblockapi as block
from onionrutils import localcommand, stringvalidators, basicrequests from onionrutils import localcommand, stringvalidators, basicrequests
from communicator import onlinepeers from communicator import onlinepeers
import onionrcrypto import onionrcrypto
from . import session
def upload_blocks_from_communicator(comm_inst): def upload_blocks_from_communicator(comm_inst):
# when inserting a block, we try to upload it to a few peers to add some deniability # when inserting a block, we try to upload it to a few peers to add some deniability
TIMER_NAME = "upload_blocks_from_communicator" TIMER_NAME = "upload_blocks_from_communicator"
@ -42,7 +45,7 @@ def upload_blocks_from_communicator(comm_inst):
if peer in triedPeers: if peer in triedPeers:
continue continue
triedPeers.append(peer) triedPeers.append(peer)
url = 'http://' + peer + '/upload' url = 'http://%s/upload' % (peer,)
try: try:
#data = {'block': block.Block(bl).getRaw()} #data = {'block': block.Block(bl).getRaw()}
data = block.Block(bl).getRaw() data = block.Block(bl).getRaw()

View File

@ -0,0 +1,51 @@
"""
Onionr - Private P2P Communication
Virtual upload "sessions" for blocks
"""
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from typing import Union
from onionrutils import stringvalidators
from onionrutils import bytesconverter
from onionrutils import epoch
from utils import reconstructhash
class UploadSession:
"""Manages statistics for an Onionr block upload session
accepting a block hash (incl. unpadded) as an argument"""
def __init__(self, block_hash: Union[str, bytes]):
block_hash = bytesconverter.bytes_to_str(block_hash)
block_hash = reconstructhash.reconstruct_hash(block_hash)
if not stringvalidators.validate_hash(block_hash): raise ValueError
self.block_hash = reconstructhash.deconstruct_hash(block_hash)
self.total_fail_count: int = 0
self.total_success_count: int = 0
self.peer_fails = {}
def fail_peer(self, peer):
try:
self.peer_fails[peer] += 1
except KeyError:
self.peer_fails[peer] = 0
def fail(self):
self.total_fail_count += 1
def success(self):
self.total_success_count += 1

View File

@ -24,7 +24,7 @@ ONIONR_TAGLINE = 'Private P2P Communication - GPLv3 - https://Onionr.net'
ONIONR_VERSION = '0.0.0' # for debugging and stuff ONIONR_VERSION = '0.0.0' # for debugging and stuff
ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION) ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
API_VERSION = '0' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you. API_VERSION = '0' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
MIN_PY_VERSION = 6 MIN_PY_VERSION = 7
DEVELOPMENT_MODE = True DEVELOPMENT_MODE = True
MAX_BLOCK_TYPE_LENGTH = 15 MAX_BLOCK_TYPE_LENGTH = 15
MAX_BLOCK_CLOCK_SKEW = 120 MAX_BLOCK_CLOCK_SKEW = 120

View File

@ -52,7 +52,7 @@ class TestPeerProfiles(unittest.TestCase):
p.addScore(1) p.addScore(1)
self.assertEqual(p.score, keydb.transportinfo.get_address_info(p.address, 'success')) self.assertEqual(p.score, keydb.transportinfo.get_address_info(p.address, 'success'))
def test_inc_score_with_sync_Delay(self): def test_inc_score_with_sync_delay(self):
p = peerprofiles.PeerProfiles(test_peers.pop()) p = peerprofiles.PeerProfiles(test_peers.pop())
s = 0 s = 0
for x in range(2): for x in range(2):

View File

@ -0,0 +1,36 @@
#!/usr/bin/env python3
import sys, os
sys.path.append(".")
sys.path.append("onionr/")
import unittest, uuid
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
import hashlib
from communicatorutils import uploadblocks
def hash_generator():
hasher = hashlib.sha3_256()
hasher.update(os.urandom(15))
return hasher.hexdigest()
test_hashes = []
for x in range(100): test_hashes.append(hash_generator())
class UploadSessionTest(unittest.TestCase):
def test_init_fail(self):
s = test_hashes.pop()
s = uploadblocks.session.UploadSession(s)
self.assertEqual(s.total_fail_count, 0)
def test_init_success(self):
s = test_hashes.pop()
s = uploadblocks.session.UploadSession(s)
self.assertEqual(s.total_success_count, 0)
def test_invalid(self):
invalid = [None, 1, -1, 0, 'ab43c5b8c7b9b037d4f02fa6bc77dbb522bfcbcd7e8ea2953bf2252c6e9232a8b', lambda: None, True, False]
for x in invalid:
self.assertRaises((ValueError, AttributeError), uploadblocks.session.UploadSession, x)
unittest.main()