From 01f9b9b470939e51d45340bc56c134bf6c2a5026 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 27 Dec 2019 01:53:18 -0600 Subject: [PATCH] work on block insertion mixing --- .../__init__.py} | 26 +++--- .../uploadblocks/__init__.py | 1 - .../uploadblocks/mixmate/__init__.py | 8 ++ .../uploadblocks/sessionmanager.py | 5 +- src/etc/onionrvalues.py | 1 + src/httpapi/miscclientapi/endpoints.py | 82 +++++++++++++------ src/onionrblocks/__init__.py | 4 +- src/onionrblocks/insert/__init__.py | 4 + .../{insert.py => insert/main.py} | 2 +- src/onionrblocks/insert/timeinsert.py | 51 ++++++++++++ tests/test_template.py | 11 +++ tests/test_timeinsert.py | 28 +++++++ 12 files changed, 183 insertions(+), 40 deletions(-) rename src/communicatorutils/{daemonqueuehandler.py => daemonqueuehandler/__init__.py} (81%) create mode 100644 src/onionrblocks/insert/__init__.py rename src/onionrblocks/{insert.py => insert/main.py} (99%) create mode 100644 src/onionrblocks/insert/timeinsert.py create mode 100644 tests/test_template.py create mode 100644 tests/test_timeinsert.py diff --git a/src/communicatorutils/daemonqueuehandler.py b/src/communicatorutils/daemonqueuehandler/__init__.py similarity index 81% rename from src/communicatorutils/daemonqueuehandler.py rename to src/communicatorutils/daemonqueuehandler/__init__.py index 9cddc793..01a8a606 100755 --- a/src/communicatorutils/daemonqueuehandler.py +++ b/src/communicatorutils/daemonqueuehandler/__init__.py @@ -5,10 +5,12 @@ Handle daemon queue commands in the communicator import logger from onionrplugins import onionrevents as events from onionrutils import localcommand +from communicatorutils.uploadblocks import mixmate from coredb import daemonqueue import filepaths -from . import restarttor -from communicatorutils.uploadblocks import mixmate + +from .. import restarttor + """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -29,11 +31,12 @@ def handle_daemon_commands(comm_inst): cmd = daemonqueue.daemon_queue() response = '' if cmd is not False: - events.event('daemon_command', data = {'cmd' : cmd}) + events.event('daemon_command', data={'cmd': cmd}) if cmd[0] == 'shutdown': comm_inst.shutdown = True elif cmd[0] == 'runtimeTest': - comm_inst.shared_state.get_by_string("OnionrRunTestManager").run_tests() + comm_inst.shared_state.get_by_string( + "OnionrRunTestManager").run_tests() elif cmd[0] == 'remove_from_insert_list': try: comm_inst.generating_blocks.remove(cmd[1]) @@ -44,7 +47,7 @@ def handle_daemon_commands(comm_inst): comm_inst.announce(cmd[1]) else: logger.debug("No nodes connected. Will not introduce node.") - elif cmd[0] == 'runCheck': # deprecated + elif cmd[0] == 'runCheck': # deprecated logger.debug('Status check; looks good.') open(filepaths.run_check_file + '.runcheck', 'w+').close() elif cmd[0] == 'connectedPeers': @@ -65,18 +68,21 @@ def handle_daemon_commands(comm_inst): elif cmd[0] == 'uploadBlock': comm_inst.blocksToUpload.append(cmd[1]) elif cmd[0] == 'uploadEvent': + localcommand.local_command('/waitforshare/' + cmd[1], post=True, + maxWait=5) try: mixmate.block_mixer(comm_inst.blocksToUpload, cmd[1]) except ValueError: - pass - else: - localcommand.local_command('/waitforshare/' + cmd[1], post=True, maxWait=5) + comm_inst.blocksToUpload.append(cmd[1]) else: - logger.debug('Received daemon queue command unable to be handled: %s' % (cmd[0],)) + logger.debug( + 'Received daemon queue cmd with no handler: %s' % (cmd[0],)) if cmd[0] not in ('', None): if response != '': - localcommand.local_command('queueResponseAdd/' + cmd[4], post=True, postData={'data': response}) + localcommand.local_command('queueResponseAdd/' + cmd[4], + post=True, + postData={'data': response}) response = '' comm_inst.decrementThreadCount('handle_daemon_commands') diff --git a/src/communicatorutils/uploadblocks/__init__.py b/src/communicatorutils/uploadblocks/__init__.py index d0a53108..35493b64 100755 --- a/src/communicatorutils/uploadblocks/__init__.py +++ b/src/communicatorutils/uploadblocks/__init__.py @@ -15,7 +15,6 @@ import onionrcrypto from communicator import onlinepeers if TYPE_CHECKING: from communicator import OnionrCommunicatorDaemon - """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/communicatorutils/uploadblocks/mixmate/__init__.py b/src/communicatorutils/uploadblocks/mixmate/__init__.py index ce50be7e..8c91929c 100644 --- a/src/communicatorutils/uploadblocks/mixmate/__init__.py +++ b/src/communicatorutils/uploadblocks/mixmate/__init__.py @@ -38,6 +38,14 @@ def block_mixer(upload_list: List[onionrtypes.BlockHash], to the said block list """ bl = onionrblockapi.Block(block_to_mix) + + try: + bl.bmetadata['dly'] + except (KeyError, TypeError): + pass + else: + raise ValueError + if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE: raise ValueError diff --git a/src/communicatorutils/uploadblocks/sessionmanager.py b/src/communicatorutils/uploadblocks/sessionmanager.py index 81faa668..43ee5279 100644 --- a/src/communicatorutils/uploadblocks/sessionmanager.py +++ b/src/communicatorutils/uploadblocks/sessionmanager.py @@ -106,7 +106,10 @@ class BlockUploadSessionManager: if (sess.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT: sessions_to_delete.append(sess) for sess in sessions_to_delete: - self.sessions.remove(session) + try: + self.sessions.remove(session) + except ValueError: + pass # TODO cleanup to one round of search # Remove the blocks from the sessions, upload list, # and waitforshare list diff --git a/src/etc/onionrvalues.py b/src/etc/onionrvalues.py index c4c30b65..7f2c975f 100755 --- a/src/etc/onionrvalues.py +++ b/src/etc/onionrvalues.py @@ -43,6 +43,7 @@ DATABASE_LOCK_TIMEOUT = 60 # Block creation anonymization requirements MIN_BLOCK_UPLOAD_PEER_PERCENT = 0.1 +MIN_SHARE_WAIT_DELAY_SECS = 5 # Begin OnionrValues migrated values """Make announce take a few seconds (on average) to compute to discourage excessive node announcements""" diff --git a/src/httpapi/miscclientapi/endpoints.py b/src/httpapi/miscclientapi/endpoints.py index e92bfe05..a0fa5d72 100644 --- a/src/httpapi/miscclientapi/endpoints.py +++ b/src/httpapi/miscclientapi/endpoints.py @@ -1,9 +1,33 @@ -''' - Onionr - Private P2P Communication +"""Onionr - Private P2P Communication. - Misc client API endpoints too small to need their own file and that need access to the client api inst -''' -''' +Misc client API endpoints too small to need their own file +and that need access to the client api inst +""" +from typing import TYPE_CHECKING +from secrets import randbelow + +import os +import subprocess + +from flask import Response, Blueprint, request, send_from_directory, abort +from gevent import spawn +from gevent import sleep +import unpaddedbase32 + +from httpapi import apiutils +import logger +import onionrcrypto +import config +from netcontroller import NetController +from serializeddata import SerializedData +from onionrutils import mnemonickeys +from onionrutils import bytesconverter +from etc import onionrvalues +from utils import reconstructhash +from onionrcommands import restartonionr +if TYPE_CHECKING: + from onionrtypes import BlockHash +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -16,26 +40,14 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' -import os -import subprocess - -from flask import Response, Blueprint, request, send_from_directory, abort -import unpaddedbase32 - -from httpapi import apiutils -import onionrcrypto, config -from netcontroller import NetController -from serializeddata import SerializedData -from onionrutils import mnemonickeys -from onionrutils import bytesconverter -from etc import onionrvalues -from utils import reconstructhash -from onionrcommands import restartonionr +""" pub_key = onionrcrypto.pub_key.replace('=', '') -SCRIPT_NAME = os.path.dirname(os.path.realpath(__file__)) + f'/../../../{onionrvalues.SCRIPT_NAME}' +SCRIPT_NAME = os.path.dirname(os.path.realpath(__file__)) + \ + f'/../../../{onionrvalues.SCRIPT_NAME}' + + class PrivateEndpoints: def __init__(self, client_api): @@ -84,12 +96,30 @@ class PrivateEndpoints: @private_endpoints_bp.route('/waitforshare/', methods=['post']) def waitforshare(name): - '''Used to prevent the **public** api from sharing blocks we just created''' - if not name.isalnum(): raise ValueError('block hash needs to be alpha numeric') + """Prevent the **public** api from sharing blocks we just created""" + def _delay_wait_for_share_block_removal(block: 'BlockHash'): + min_w = onionrvalues.MIN_SHARE_WAIT_DELAY_SECS + # Delay at least min but otherwise getBlocks timer + rand 10s + delay_before_remove = max( + min_w, + randbelow + (config.get + ( + 'timers.getBlocks', + default=10) + randbelow(11))) + sleep(delay_before_remove) + try: + client_api.publicAPI.hideBlocks.remove(name) + except ValueError: + logger.warn( + f'Failed to remove {name} from waitforshare') + + if not name.isalnum(): + raise ValueError('block hash needs to be alnum') name = reconstructhash.reconstruct_hash(name) if name in client_api.publicAPI.hideBlocks: - client_api.publicAPI.hideBlocks.remove(name) - return Response("removed") + spawn(_delay_wait_for_share_block_removal) + return Response("will be removed") else: client_api.publicAPI.hideBlocks.append(name) return Response("added") diff --git a/src/onionrblocks/__init__.py b/src/onionrblocks/__init__.py index 93e9fd3f..91a6b1a1 100644 --- a/src/onionrblocks/__init__.py +++ b/src/onionrblocks/__init__.py @@ -1,3 +1,5 @@ from . import insert +from .insert import time_insert -insert = insert.insert_block \ No newline at end of file +insert = insert.insert_block +time_insert = time_insert \ No newline at end of file diff --git a/src/onionrblocks/insert/__init__.py b/src/onionrblocks/insert/__init__.py new file mode 100644 index 00000000..e516f6b6 --- /dev/null +++ b/src/onionrblocks/insert/__init__.py @@ -0,0 +1,4 @@ +from . import main, timeinsert + +insert_block = main.insert_block +time_insert = timeinsert.time_insert \ No newline at end of file diff --git a/src/onionrblocks/insert.py b/src/onionrblocks/insert/main.py similarity index 99% rename from src/onionrblocks/insert.py rename to src/onionrblocks/insert/main.py index 29300b03..88a6c8fe 100644 --- a/src/onionrblocks/insert.py +++ b/src/onionrblocks/insert/main.py @@ -22,7 +22,7 @@ import json from onionrutils import bytesconverter, epoch import filepaths, onionrstorage -from . import storagecounter +from .. import storagecounter from onionrplugins import onionrevents as events from etc import powchoice, onionrvalues import config, onionrcrypto as crypto, onionrexceptions diff --git a/src/onionrblocks/insert/timeinsert.py b/src/onionrblocks/insert/timeinsert.py new file mode 100644 index 00000000..41d8fcaa --- /dev/null +++ b/src/onionrblocks/insert/timeinsert.py @@ -0,0 +1,51 @@ +"""Onionr - Private P2P Communication. + +Wrapper to insert blocks with variable delay +""" +from . import main +""" + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + + +def time_insert(*args, **kwargs): + """Block insert wrapper to allow for insertions independent of mixmate. + + Takes exact args as insert_block, with additional keyword: + delay=n; where n=seconds to tell initial nodes to delay share for. + + defaults to 0 or previously set value in current block meta + """ + try: + kwargs['meta'] + except KeyError: + kwargs['meta'] = {} + + try: + delay = int(kwargs['meta']['dly']) + except KeyError: + delay = 0 + try: + delay = kwargs['delay'] + del kwargs['delay'] + except KeyError: + delay = 0 + + # Ensure delay >=0 + if delay < 0: + raise ValueError('delay cannot be less than 0') + + kwargs['meta']['dly'] = delay + + return main.insert_block(*args, **kwargs) diff --git a/tests/test_template.py b/tests/test_template.py new file mode 100644 index 00000000..406b09de --- /dev/null +++ b/tests/test_template.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +import unittest, sys +sys.path.append(".") +sys.path.append("src/") + + +class TestTemplate(unittest.TestCase): + def test_my_test(self): + self.assertTrue(True) + +unittest.main() diff --git a/tests/test_timeinsert.py b/tests/test_timeinsert.py new file mode 100644 index 00000000..3d637020 --- /dev/null +++ b/tests/test_timeinsert.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import unittest, sys +sys.path.append(".") +sys.path.append("src/") + +from onionrblocks import time_insert +from onionrblocks import onionrblockapi + +class TestTimeInsert(unittest.TestCase): + def test_time_insert_none(self): + bl = time_insert('test') + self.assertTrue(bl) + bl = onionrblockapi.Block(bl) + self.assertIs(bl.bmetadata['dly'], 0) + + def test_time_insert_10(self): + bl = time_insert('test', delay=10) + self.assertTrue(bl) + bl = onionrblockapi.Block(bl) + self.assertIs(bl.bmetadata['dly'], 10) + + def test_negative(self): + self.assertRaises(ValueError, time_insert, 'test', delay=-1) + self.assertRaises(ValueError, time_insert, 'test', delay=-10) + + + +unittest.main()