From 1b37264eb766f7de8f985b47a58a9a7d38443429 Mon Sep 17 00:00:00 2001 From: Kevin F Date: Sun, 1 May 2022 00:45:26 -0500 Subject: [PATCH] Work on gossip tests and cleaned up api server some --- run_tests.sh | 4 ++ src/apiservers/__init__.py | 2 +- src/apiservers/private/__init__.py | 6 +- .../private/register_private_blueprints.py | 12 ---- src/gossip/client/__init__.py | 4 +- src/gossip/client/dandelionstem/__init__.py | 2 +- src/gossip/server/__init__.py | 6 +- src/gossip/server/acceptstem.py | 2 +- src/httpapi/addblock/__init__.py | 44 ++++++++++++ src/httpapi/apiutils/shutdown.py | 7 -- src/httpapi/daemoneventsapi/__init__.py | 55 --------------- src/httpapi/miscclientapi/endpoints.py | 11 --- src/httpapi/security/client.py | 6 -- src/onionrcommands/daemonlaunch/__init__.py | 15 ++--- static-data/default-plugins/tor/bootstrap.py | 1 - tests/gossip-unittests/test_put_block_mny.py | 67 +++++++++++++++++++ tests/gossip-unittests/test_put_block_one.py | 2 +- .../test_server_diffuse_one.py | 2 +- 18 files changed, 136 insertions(+), 112 deletions(-) create mode 100644 src/httpapi/addblock/__init__.py delete mode 100644 src/httpapi/daemoneventsapi/__init__.py create mode 100644 tests/gossip-unittests/test_put_block_mny.py diff --git a/run_tests.sh b/run_tests.sh index 1ebee923..d019019d 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -13,6 +13,10 @@ for f in tests/*.py; do python3 "$f" || close # if needed let "ran++" done +for f in tests/gossip-unittests/*.py; do + python3 "$f" || close # if needed + let "ran++" +done echo "ran $ran unittests. Unittest Time: $SECONDS" ran=0; diff --git a/src/apiservers/__init__.py b/src/apiservers/__init__.py index f9926150..6b78e13b 100755 --- a/src/apiservers/__init__.py +++ b/src/apiservers/__init__.py @@ -6,4 +6,4 @@ Private is meant for controlling and accessing this node from . import private -ClientAPI = private.PrivateAPI +private_api = private.private_api diff --git a/src/apiservers/private/__init__.py b/src/apiservers/private/__init__.py index e42bc435..bef974dc 100644 --- a/src/apiservers/private/__init__.py +++ b/src/apiservers/private/__init__.py @@ -65,7 +65,7 @@ class PrivateAPI: else: self.host = httpapi.apiutils.setbindip.set_bind_IP( private_API_host_file) - logger.info('Running api on %s:%s' % (self.host, self.bindPort)) + logger.info(f'Running API on {self.host}:{self.bindPort}', terminal=True) self.httpServer = '' self.queueResponse = {} @@ -75,7 +75,6 @@ class PrivateAPI: def start(self): """Start client gevent API web server with flask client app.""" - waitforsetvar.wait_for_set_var(self, "_too_many") fd_handler = httpapi.fdsafehandler.FDSafeHandler self.httpServer = WSGIServer((self.host, self.bindPort), @@ -116,3 +115,6 @@ class PrivateAPI: decrypt=decrypt, raw=raw, headerOnly=headerOnly) + + +private_api = PrivateAPI() \ No newline at end of file diff --git a/src/apiservers/private/register_private_blueprints.py b/src/apiservers/private/register_private_blueprints.py index fb68b229..8d68032e 100644 --- a/src/apiservers/private/register_private_blueprints.py +++ b/src/apiservers/private/register_private_blueprints.py @@ -41,16 +41,4 @@ def register_private_blueprints(private_api, app): app.register_blueprint(private_sse_blueprint) app.register_blueprint(fileoffsetreader.offset_reader_api) - def _add_events_bp(): - while True: - try: - private_api._too_many - break - except AttributeError: - sleep(0.2) - app.register_blueprint( - private_api._too_many.get_by_string('DaemonEventsBP').flask_bp) - - Thread(target=_add_events_bp, daemon=True).start() - return app diff --git a/src/gossip/client/__init__.py b/src/gossip/client/__init__.py index ea089e0a..4860e62f 100644 --- a/src/gossip/client/__init__.py +++ b/src/gossip/client/__init__.py @@ -68,7 +68,7 @@ def gossip_client(): # transport plugin handles the new peer add_onionr_thread( get_new_peers, - 1200, initial_sleep=5) + 120, initial_sleep=5) # Start a new thread to stream blocks from peers @@ -76,8 +76,6 @@ def gossip_client(): dandelion_phase = DandelionPhase(DANDELION_EPOCH_LENGTH) while True: - sleep(5) - continue while not len(gossip_peer_set): sleep(0.2) if dandelion_phase.remaining_time() <= 10: diff --git a/src/gossip/client/dandelionstem/__init__.py b/src/gossip/client/dandelionstem/__init__.py index 5e538a2f..fe5c23d7 100644 --- a/src/gossip/client/dandelionstem/__init__.py +++ b/src/gossip/client/dandelionstem/__init__.py @@ -98,7 +98,7 @@ async def stem_out(d_phase: 'DandelionPhase'): peer_sockets.append(await _setup_edge(gossip_peer_set, tried_edges)) except NotEnoughEdges: # No possible edges at this point (edges < OUTBOUND_DANDELION_EDGE) - logger.warn("Not able to build enough peers for stemout.", + logger.warn("Not able to build enough tunnels for stemout.", terminal=True) break else: diff --git a/src/gossip/server/__init__.py b/src/gossip/server/__init__.py index 5f15ac30..c46cf87f 100644 --- a/src/gossip/server/__init__.py +++ b/src/gossip/server/__init__.py @@ -96,9 +96,13 @@ def gossip_server(): logger.debug( "Inbound edge timed out when steming blocks to us", terminal=True) + except asyncio.exceptions.IncompleteReadError: + logger.debug( + "Inbound edge timed out (Incomplete Read) when steming blocks to us", + terminal=True) except Exception: logger.warn( - f"Err getting\n{traceback.format_exc()}", + f"Err acceptind stem blocks\n{traceback.format_exc()}", terminal=True) # Subtract dandelion edge, make sure >=0 inbound_dandelion_edge_count[0] = \ diff --git a/src/gossip/server/acceptstem.py b/src/gossip/server/acceptstem.py index 2fe24685..0e50b73f 100644 --- a/src/gossip/server/acceptstem.py +++ b/src/gossip/server/acceptstem.py @@ -12,7 +12,7 @@ from ..blockqueues import gossip_block_queues block_size_digits = len(str(BLOCK_MAX_SIZE)) -base_wait_timeout = 30 +base_wait_timeout = 120 if TYPE_CHECKING: from asyncio import StreamWriter, StreamReader diff --git a/src/httpapi/addblock/__init__.py b/src/httpapi/addblock/__init__.py new file mode 100644 index 00000000..bb19020f --- /dev/null +++ b/src/httpapi/addblock/__init__.py @@ -0,0 +1,44 @@ +"""Onionr - Private P2P Communication. + +Serialized APIs +""" + +from asyncio.log import logger +import secrets +from flask import Blueprint, Response, request + +from onionrblocks import Block + +import logger +from gossip import blockqueues +from gossip.constants import BLOCK_ID_SIZE + +""" +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" +blockapi = Blueprint('blockapi', __name__) + + +stream_to_use = secrets.randbits(1) + +# Add a block that we generated (or received from a transport like LAN/sneakernet) +@blockapi.route('/addvdfblock', methods=['POST']) +def block_serialized(): + req_data = request.data + block_id = req_data[:BLOCK_ID_SIZE] + block_data = req_data[BLOCK_ID_SIZE:] + blockqueues.gossip_block_queues[stream_to_use].put( + Block(block_id, block_data, auto_verify=False)) + logger.info("Added block" + block_id, terminal=True) + return "ok" \ No newline at end of file diff --git a/src/httpapi/apiutils/shutdown.py b/src/httpapi/apiutils/shutdown.py index 6e5a085d..9f96c1f9 100644 --- a/src/httpapi/apiutils/shutdown.py +++ b/src/httpapi/apiutils/shutdown.py @@ -28,10 +28,3 @@ def shutdown(client_api_inst): except AttributeError: pass return Response("bye") - - -@shutdown_bp.route('/shutdownclean') -def shutdown_clean(): - # good for calling from other clients - g.too_many.get_by_string("DeadSimpleKV").put('shutdown', True) - return Response("bye") diff --git a/src/httpapi/daemoneventsapi/__init__.py b/src/httpapi/daemoneventsapi/__init__.py deleted file mode 100644 index feb61141..00000000 --- a/src/httpapi/daemoneventsapi/__init__.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Onionr - Private P2P Communication. - -Event driven interface to trigger events in communicator -""" -from typing import Callable - -from flask import Blueprint, request, Response, abort -from werkzeug.exceptions import BadRequest -from gevent import spawn - -""" -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - - -class DaemonEventsBP: - def __init__(self): - """Create DaemonEvents instance, intended to be a singleton. - - Attributes: - listeners: callables that are called when a new event is added. - The callables name should match the event name - _too_many: TooManyObjects instance set by external code - """ - event_BP = Blueprint('event_BP', __name__) - self.listeners = set([]) - self.flask_bp = event_BP - event_BP = self.flask_bp - - @event_BP.route('/daemon-event/', methods=['POST']) - def daemon_event_handler(name): - handler: Callable - - try: - json_data = request.get_json(force=True) - except BadRequest: - json_data = {} - for handler in self.listeners: - if handler.__name__ == name: - return Response(handler(**json_data)) - abort(404) - - def register_listener(self, listener: Callable): - self.listeners.add(listener) diff --git a/src/httpapi/miscclientapi/endpoints.py b/src/httpapi/miscclientapi/endpoints.py index e4c7e8d1..f57bc2e1 100644 --- a/src/httpapi/miscclientapi/endpoints.py +++ b/src/httpapi/miscclientapi/endpoints.py @@ -68,7 +68,6 @@ class PrivateEndpoints: return Response("pong!") - @private_endpoints_bp.route('/shutdown') def shutdown(): return apiutils.shutdown.shutdown(client_api) @@ -110,13 +109,3 @@ class PrivateEndpoints: def get_os_system(): return Response(platform.system().lower()) - - @private_endpoints_bp.route('/getgeneratingblocks') - def get_generating_blocks() -> Response: - return Response( - ','.join( - g.too_many.get_by_string('DeadSimpleKV').get( - 'generating_blocks' - )) - ) - diff --git a/src/httpapi/security/client.py b/src/httpapi/security/client.py index d05f7f41..2fd3b7d6 100644 --- a/src/httpapi/security/client.py +++ b/src/httpapi/security/client.py @@ -71,12 +71,6 @@ class ClientAPISecurity: f'Possible DNS rebinding attack by {request.host}') abort(403) - # Add shared objects - try: - g.too_many = self.client_api._too_many - except KeyError: - g.too_many = None - # Static files for Onionr sites if request.path.startswith('/site/'): return diff --git a/src/onionrcommands/daemonlaunch/__init__.py b/src/onionrcommands/daemonlaunch/__init__.py index 7dffd4db..2dba869d 100755 --- a/src/onionrcommands/daemonlaunch/__init__.py +++ b/src/onionrcommands/daemonlaunch/__init__.py @@ -7,6 +7,7 @@ import queue import sys import platform import signal +from threading import Thread from stem.connection import IncorrectPassword import toomanyobjs @@ -28,7 +29,6 @@ import onionrvalues from onionrutils import cleanup from onionrcrypto import getourkeypair import runtests -from httpapi import daemoneventsapi from .. import version from .killdaemon import kill_daemon # noqa from .showlogo import show_logo @@ -95,15 +95,11 @@ def daemon(): # Initialize the quasi-global variables setup_kv(shared_state.get(DeadSimpleKV)) - shared_state.get(daemoneventsapi.DaemonEventsBP) - # Init run time tester # (ensures Onionr is running right, for testing purposes) # Run time tests are not normally run shared_state.get(runtests.OnionrRunTestManager) - # initialize clientAPI but dont start it yet - shared_state.get(apiservers.ClientAPI) shared_state.share_object() # share the parent object to the threads @@ -117,10 +113,10 @@ def daemon(): events.event('init', threaded=False) events.event('daemon_start') - gossip.start_gossip_threads() + Thread(target=gossip.start_gossip_threads, daemon=True).start() try: - shared_state.get(apiservers.ClientAPI).start() + apiservers.private_api.start() except KeyboardInterrupt: pass @@ -149,8 +145,9 @@ def start(override: bool = False): except psutil.NoSuchProcess: proc = "" if not proc.startswith("python"): - logger.info( - f"Detected stale run file, deleting {filepaths.lock_file}", terminal=True) + logger.warn( + f"Detected stale run file, deleting {filepaths.lock_file}", + terminal=True) try: os.remove(filepaths.lock_file) except FileNotFoundError: diff --git a/static-data/default-plugins/tor/bootstrap.py b/static-data/default-plugins/tor/bootstrap.py index 181b1140..225363b0 100644 --- a/static-data/default-plugins/tor/bootstrap.py +++ b/static-data/default-plugins/tor/bootstrap.py @@ -43,4 +43,3 @@ def on_bootstrap(api, data): args=[TorPeer(socks_address, socks_port, address)], daemon=True).start() - diff --git a/tests/gossip-unittests/test_put_block_mny.py b/tests/gossip-unittests/test_put_block_mny.py new file mode 100644 index 00000000..7289056e --- /dev/null +++ b/tests/gossip-unittests/test_put_block_mny.py @@ -0,0 +1,67 @@ +import os, uuid +from queue import Empty +TEST_DIR = 'testdata/%s-%s' % (str(uuid.uuid4())[:5], os.path.basename(__file__)) + '/' +print("Test directory:", TEST_DIR) +os.environ["ONIONR_HOME"] = TEST_DIR + +from time import sleep +from threading import Thread +import asyncio +import unittest +import sys +sys.path.append(".") +sys.path.append("src/") + +from ordered_set import OrderedSet +import onionrblocks + +import blockdb +from gossip.server import gossip_server +from gossip.blockqueues import gossip_block_queues +from filepaths import gossip_server_socket_file + + +BLOCK_MAX_SIZE = 1024 * 2000 +BLOCK_MAX_SIZE_LEN = len(str(BLOCK_MAX_SIZE)) +BLOCK_ID_SIZE = 128 +BLOCK_STREAM_OFFSET_DIGITS = 8 + + +class OnionrServerPutBlocksTest(unittest.TestCase): + + + def test_put_blocks(self): + + Thread(target=gossip_server, daemon=True).start() + sleep(0.01) + + blocks = [] + for _ in range(10): + bl = onionrblocks.blockcreator.create_anonvdf_block( + b"my test block" + os.urandom(16), b"txt", 2800) + blockdb.add_block_to_db(bl) + blocks.append(bl) + + async def blocks_put_client(): + reader, writer = await asyncio.open_unix_connection( + gossip_server_socket_file) + writer.write(int(5).to_bytes(1, 'big')) + await writer.drain() + + for bl in blocks: + writer.write(bl.id) + writer.write( + str(len(bl.raw)).zfill(BLOCK_MAX_SIZE_LEN).encode('utf-8')) + writer.write(bl.raw) + await writer.drain() + + sleep(0.03) + try: + self.assertEqual(gossip_block_queues[0].get_nowait().raw, bl.raw) + except Empty: + self.assertEqual(gossip_block_queues[1].get_nowait().raw, bl.raw) + + + asyncio.run(blocks_put_client()) + +unittest.main() diff --git a/tests/gossip-unittests/test_put_block_one.py b/tests/gossip-unittests/test_put_block_one.py index 72daea4f..b3337951 100644 --- a/tests/gossip-unittests/test_put_block_one.py +++ b/tests/gossip-unittests/test_put_block_one.py @@ -1,6 +1,6 @@ import os, uuid from queue import Empty -TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' +TEST_DIR = 'testdata/%s-%s' % (str(uuid.uuid4())[:6], os.path.basename(__file__)) + '/' print("Test directory:", TEST_DIR) os.environ["ONIONR_HOME"] = TEST_DIR diff --git a/tests/gossip-unittests/test_server_diffuse_one.py b/tests/gossip-unittests/test_server_diffuse_one.py index dce76b8f..c2957dce 100644 --- a/tests/gossip-unittests/test_server_diffuse_one.py +++ b/tests/gossip-unittests/test_server_diffuse_one.py @@ -1,5 +1,5 @@ import os, uuid -TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' +TEST_DIR = 'testdata/%s-%s' % (str(uuid.uuid4())[:6], os.path.basename(__file__)) + '/' print("Test directory:", TEST_DIR) os.environ["ONIONR_HOME"] = TEST_DIR