From 9d546432f5c6b3056394f91332c98ef26b1ba000 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Thu, 4 Feb 2021 01:52:36 +0000 Subject: [PATCH] added traceback handler --- src/anonvdf-block-creator.py | 3 +- .../private/register_private_blueprints.py | 2 +- src/blockcreatorqueue/__init__.py | 4 +- src/blockio/subprocgenerate.py | 2 + src/onionrcommands/daemonlaunch/__init__.py | 40 ++++++++++++++++++- src/runtests/__init__.py | 2 +- src/runtests/housekeeping.py | 7 ++-- src/runtests/ownnode.py | 2 + src/runtests/sneakernettest.py | 4 +- static-data/base64-code-whitelist.txt | 10 +++++ .../torgossip/commandhandlers.py | 1 - .../default-plugins/torgossip/commands.py | 1 + static-data/default-plugins/torgossip/main.py | 16 +++++--- .../default-plugins/torgossip/peerdb.py | 3 +- .../default-plugins/torgossip/runtest.py | 31 +++++++++----- static-data/default_config.json | 3 ++ 16 files changed, 102 insertions(+), 29 deletions(-) diff --git a/src/anonvdf-block-creator.py b/src/anonvdf-block-creator.py index 4ac1f08c..232feed7 100755 --- a/src/anonvdf-block-creator.py +++ b/src/anonvdf-block-creator.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +from base64 import b85decode import sys import os @@ -13,7 +14,7 @@ metadata = json.loads(sys.argv[1]) block_type = sys.argv[2] ttl = int(sys.argv[3]) -data = sys.stdin.read() +data = b85decode(sys.stdin.read()) with os.fdopen(sys.stdout.fileno(), 'wb') as stdout: bl = blockcreator.create_anonvdf_block(data, block_type, ttl, **metadata) diff --git a/src/apiservers/private/register_private_blueprints.py b/src/apiservers/private/register_private_blueprints.py index d3abcbd0..40039ddf 100644 --- a/src/apiservers/private/register_private_blueprints.py +++ b/src/apiservers/private/register_private_blueprints.py @@ -56,6 +56,6 @@ def register_private_blueprints(private_api, app): app.register_blueprint( private_api._too_many.get_by_string('DaemonEventsBP').flask_bp) - Thread(target=_add_events_bp).start() + Thread(target=_add_events_bp, name='Private blueprints adder').start() return app diff --git a/src/blockcreatorqueue/__init__.py b/src/blockcreatorqueue/__init__.py index 4b007963..abc622d5 100644 --- a/src/blockcreatorqueue/__init__.py +++ b/src/blockcreatorqueue/__init__.py @@ -77,5 +77,7 @@ class BlockCreatorQueue: *self.additional_callback_func_args, **self.additional_callback_func_kwargs) - Thread(target=_do_create, daemon=True).start() + Thread( + target=_do_create, daemon=True, + name="BlockCreatorQueue block creation").start() return digest diff --git a/src/blockio/subprocgenerate.py b/src/blockio/subprocgenerate.py index a59c1ded..58d8b648 100644 --- a/src/blockio/subprocgenerate.py +++ b/src/blockio/subprocgenerate.py @@ -1,3 +1,4 @@ +from base64 import b85encode import os import subprocess @@ -10,6 +11,7 @@ _DIR = os.path.dirname(os.path.realpath(__file__)) + '/../' def vdf_block(data, data_type, ttl, **metadata): + data = b85encode(data) generated = subprocess.Popen( [ f'{_DIR}anonvdf-block-creator.py', diff --git a/src/onionrcommands/daemonlaunch/__init__.py b/src/onionrcommands/daemonlaunch/__init__.py index d92d9b3f..7dc0e77c 100755 --- a/src/onionrcommands/daemonlaunch/__init__.py +++ b/src/onionrcommands/daemonlaunch/__init__.py @@ -7,6 +7,8 @@ import sys import platform import signal from threading import Thread +from threading import enumerate as thread_enumerate +import traceback from stem.connection import IncorrectPassword import stem @@ -44,7 +46,7 @@ from sneakernet import sneakernet_import_thread from onionrstatistics.devreporting import statistics_reporter from setupkvvars import setup_kv from communicatorutils.housekeeping import clean_blocks_not_meeting_pow -from blockcreatorqueue import BlockCreatorQueue, PassToSafeDB +from blockcreatorqueue import PassToSafeDB from .spawndaemonthreads import spawn_client_threads from .loadsafedb import load_safe_db """ @@ -144,7 +146,43 @@ def daemon(): logger.info( f"Recieved sigterm in child process or fork, exiting. PID: {pid}") sys.exit(0) + + def _handle_sigusr1(sig, frame): + traceback_file = identifyhome.identify_home() + "/traceback" + id2name = dict([(th.ident, th.name) for th in thread_enumerate()]) + code = [] + for thread_id, stack in sys._current_frames().items(): + code.append( + "\n# Thread: %s(%d)" % + (id2name.get(thread_id, ""), thread_id)) + for filename, lineno, name, line in traceback.extract_stack(stack): + code.append( + 'File: "%s", line %d, in %s' % (filename, lineno, name)) + if line: + code.append(" %s" % (line.strip())) + with open(traceback_file, 'w') as tb_f: + tb_f.write("\n".join(code)) + logger.info( + f"Wrote traceback of all main process threads to {traceback_file}", + terminal=True) + + def _sigusr1_thrower(): + wait_for_write_pipe = identifyhome.identify_home() + \ + "/activate-traceback" + try: + os.mkfifo(wait_for_write_pipe) + except FileExistsError: + pass + with open(wait_for_write_pipe, "r") as f: + f.read() + os.kill(os.getpid(), signal.SIGUSR1) + signal.signal(signal.SIGTERM, _handle_sig_term) + signal.signal(signal.SIGUSR1, _handle_sigusr1) + + Thread( + target=_sigusr1_thrower, + daemon=True, name="siguser1 wait and throw").start() # Determine if Onionr is in offline mode. # When offline, Onionr can only use LAN and disk transport diff --git a/src/runtests/__init__.py b/src/runtests/__init__.py index cc802f4c..8a71632e 100644 --- a/src/runtests/__init__.py +++ b/src/runtests/__init__.py @@ -84,7 +84,7 @@ class OnionrRunTestManager: logger.error(last.__name__ + ' failed assertions', terminal=True) except Exception as e: logger.error(last.__name__ + ' failed with non-asserting exception') - logger.error(str(e)) + logger.error(repr(e)) else: ep = str(epoch.get_epoch()) logger.info(f'All runtime tests passed at {ep}', terminal=True) diff --git a/src/runtests/housekeeping.py b/src/runtests/housekeeping.py index 94b9076c..755d769b 100644 --- a/src/runtests/housekeeping.py +++ b/src/runtests/housekeeping.py @@ -1,15 +1,14 @@ -import os - from gevent import sleep from oldblocks import insert -import logger +import config from coredb.blockmetadb import get_block_list -from onionrutils import epoch def test_inserted_housekeeping(testmanager): """Tests that inserted blocks are proprely deleted""" + if config.get('runtests.skip_slow', False): + return bl = insert('testdata', expire=12) wait_seconds = 132 # Wait two minutes plus expire time count = 0 diff --git a/src/runtests/ownnode.py b/src/runtests/ownnode.py index 069b6cf7..8273b6ed 100644 --- a/src/runtests/ownnode.py +++ b/src/runtests/ownnode.py @@ -27,6 +27,8 @@ from onionrutils import localcommand def test_own_node(test_manager): if config.get('general.security_level', 0) > 0 or not config.get('transports.tor', True): return + if config.get('runtests.skip_slow', False): + return socks_port = localcommand.local_command('/gettorsocks') if config.get('general.security_level', 0) > 0: return diff --git a/src/runtests/sneakernettest.py b/src/runtests/sneakernettest.py index 4af036fa..9da20257 100644 --- a/src/runtests/sneakernettest.py +++ b/src/runtests/sneakernettest.py @@ -9,10 +9,12 @@ from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT from onionrstorage.removeblock import remove_block from onionrstorage import deleteBlock from coredb.blockmetadb import get_block_list -from utils import bettersleep +import config from gevent import sleep def test_sneakernet_import(test_manager): + if not config.get('transports.lan', False): + return in_db = lambda b: b in get_block_list() bl = insert(os.urandom(10)) assert in_db(bl) diff --git a/static-data/base64-code-whitelist.txt b/static-data/base64-code-whitelist.txt index cfc77904..b92f3755 100644 --- a/static-data/base64-code-whitelist.txt +++ b/static-data/base64-code-whitelist.txt @@ -68,3 +68,13 @@ ZABkAWwAWgBkAGQBbAFaAWQAZAJsAm0DWgMBAGQAZANsBG0FWgUBAGQAZARsBm0HWgcBAGQAZAVsCG0J ZABkAWwAWgBkAGQBbAFaAWQAZAJsAm0DWgMBAGQAZANsBG0FWgUBAGQAZARsBm0HWgcBAGQAZAVsCG0JWgkBAGQAZAZsCm0LWgsBAGQAZAFsCFoIZAdkCIQAWgxkAVMA ZABaAGQBZAJsAW0CWgIBAGQBZANsA20EWgQBAGQBZARsBVoFZAFkBGwBWgFkAWQFbAZtB1oHAQBkBmQHhABaCGUJZAicAWQJZAqEBFoKZQlkCJwBZAtkDIQEWgtkDWQOhABaDGQEUwA= ZABaAGQBZAJsAW0CWgIBAGQBZANsA20EWgQBAGQBZARsBW0GWgYBAGQBZAVsB1oHZAFkBWwDWgNkAWQGbAhtCVoJAQBkB2QIhABaCmULZAmcAWQKZAuEBFoMZAxkDYQAWg1lC2QJnAFkDmQPhARaDmQQZBGEAFoPZAVTAA== +ZABaAGQBZAJsAVoBZQGgAmUBagNkA6ECAQBkAWQCbARaBGQBZAJsBVoFZAFkBGwGbQdaBwEAZQRqCKAJZAFlBWoIoAplBWoIoAtlDKEBoQGhAgEAZAVaDWQBZAZsDm0PWg8BAGQBZAdsEG0RWhEBAGQBZAhsEm0TWhMBAGQLZAlkCoQBWhRkAlMA +ZABaAGQBZAJsAVoBZAFkAmwCWgJkAWQCbANaA2QBZAJsBFoEZAFkA2wFbQZaBgEAZAFkAmwHWgdlAmoIoAlkAWUBagigCmUBagigC2UMoQGhAaECAQBkAWQEbA1tDloOAQBkAWQCbA9aD2QFZAaEAFoQZAJTAA== +ZABaAGQBZAJsAW0CWgIBAGQBZANsA20EWgQBAGQBZARsBW0GWgYBAGQBZAVsB1oHZAFkBWwDWgNkAWQGbAhtCVoJAQBkB2UKZAicAmQJZAqEBFoLZAtkDIQAWgxlCmQNnAFkDmQPhARaDWQQZBGEAFoOZQpkDZwBZBJkE4QEWg9kFGQVhABaEGQFUwA= +ZABaAGQBZAJsAVoBZQGgAmUBagNkA6ECAQBkAWQCbARaBGQBZAJsBVoFZAFkBGwGbQdaBwEAZQRqCKAJZAFlBWoIoAplBWoIoAtlDKEBoQGhAgEAZAVaDXooZAFkBmwObQ9aDwEAZAFkB2wQbRFaEQEAZAFkCGwSbRNaEwEAVwBuLgQAZRRrCnKuAQBaFQEAehBlFmUXZRWDAYMBAQBXADUAZAJaFVsVWABZAG4CWABkC2QJZAqEAVoYZAJTAA== +ZABaAGQBZAJsAW0CWgIBAGQBZANsA20EWgQBAGQBZARsBVoFZAFkBGwBWgFkAWQFbAZtB1oHAQBkBmUIZAecAmQIZAmEBFoJZApkC4QAWgplCGQMnAFkDWQOhARaC2QPZBCEAFoMZQhkDJwBZBFkEoQEWg1kE2QUhABaDmQEUwA= +ZABaAGQBZAJsAW0CWgIBAGQBZANsA20EWgRtBVoFAQBkAWQEbAZtBloGAQBkAWQFbAdtCFoIAQBkAWQGbAlaCUcAZAdkCIQAZAiDAloKZAZTAA== +ZABkAWwAWgBkAGQBbAFaAWQAZAJsAm0DWgMBAGQAZANsBG0FWgUBAGQAZARsBm0HWgcBAGQAZAVsCG0JWgkBAGQAZAZsCm0LWgsBAGQAZAdsDG0NWg0BAGQAZAFsCloKZAhkCYQAWg5kCmQLhABaD2QBUwA= +ZABkAWwAWgBkAGQBbAFaAWQAZAFsAloCZABkAmwDbQRaBAEAZABkA2wFbQZaBgEAZABkBGwHbQhaCAEAZABkBWwJbQpaCgEAZABkBmwLbQxaDAEAZABkAWwJWglkB2QIhABaDWQJZAqEAFoOZAFTAA== +ZABTAA== +ZQBGAGQAUwA= diff --git a/static-data/default-plugins/torgossip/commandhandlers.py b/static-data/default-plugins/torgossip/commandhandlers.py index 6aa0221f..fec6ef48 100644 --- a/static-data/default-plugins/torgossip/commandhandlers.py +++ b/static-data/default-plugins/torgossip/commandhandlers.py @@ -2,7 +2,6 @@ Handle commands for the torgossip server """ -from typing import type_check_only from onionrblocks import generators from onionrblocks.generators import anonvdf import blockio diff --git a/static-data/default-plugins/torgossip/commands.py b/static-data/default-plugins/torgossip/commands.py index a2fe686d..1d111ca1 100644 --- a/static-data/default-plugins/torgossip/commands.py +++ b/static-data/default-plugins/torgossip/commands.py @@ -20,6 +20,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . """ + class GossipCommands(IntEnum): PING = 1, CHECK_HAS_BLOCK = 2, diff --git a/static-data/default-plugins/torgossip/main.py b/static-data/default-plugins/torgossip/main.py index f660946c..a921bb17 100755 --- a/static-data/default-plugins/torgossip/main.py +++ b/static-data/default-plugins/torgossip/main.py @@ -25,17 +25,21 @@ along with this program. If not, see . """ plugin_name = 'torgossip' -from server import start_server -from peerdb import Peers -from runtest import torgossip_runtest +try: + from server import start_server + from peerdb import TorGossipPeers + from runtest import torgossip_runtest +except Exception as e: + print(repr(e)) + def on_init(api, data=None): shared_state = data - - shared_state.get(Peers) - shared_state.get_by_string( "OnionrRunTestManager").plugin_tests.append(torgossip_runtest) + shared_state.get(TorGossipPeers) + + Thread(target=start_server, daemon=True, args=[shared_state]).start() diff --git a/static-data/default-plugins/torgossip/peerdb.py b/static-data/default-plugins/torgossip/peerdb.py index c4a76eb6..f4b1a499 100644 --- a/static-data/default-plugins/torgossip/peerdb.py +++ b/static-data/default-plugins/torgossip/peerdb.py @@ -2,10 +2,9 @@ Torgossip peer safedb interface """ -from typing import TYPE_CHECKING from base64 import b32decode from struct import unpack, pack -frome time import time +from time import time from utils.identifyhome import identify_home import safedb diff --git a/static-data/default-plugins/torgossip/runtest.py b/static-data/default-plugins/torgossip/runtest.py index e89eb6e3..7062b8ff 100644 --- a/static-data/default-plugins/torgossip/runtest.py +++ b/static-data/default-plugins/torgossip/runtest.py @@ -1,6 +1,7 @@ import socket import os -from threading import local +import secrets +from base64 import b32encode from utils import identifyhome from onionrblocks import blockcreator @@ -8,6 +9,11 @@ from blockio import subprocgenerate from onionrutils import localcommand import blockio + +def _fake_onion(): + return b32encode(os.urandom(34)).decode('utf-8') + ".onion" + + def torgossip_runtest(test_manager): @@ -20,12 +26,13 @@ def torgossip_runtest(test_manager): blockio.store_block(bl, shared_state.get_by_string("SafeDB")) tsts = b'' - for i in range(3): - bl = subprocgenerate.vdf_block(b"test" + os.urandom(3), "tst", 100) - tsts += bl.id - blockio.store_block(bl, shared_state.get_by_string("SafeDB")) - bl_new = blockcreator.create_anonvdf_block(b"test5", "txt", 10) + for i in range(3): + bl2 = subprocgenerate.vdf_block(b"what" + os.urandom(4), "tbt", 100) + tsts += bl2.id + blockio.store_block(bl2, shared_state.get_by_string("SafeDB")) + + bl_new = blockcreator.create_anonvdf_block(b"what", "txt", 10) with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: @@ -41,8 +48,10 @@ def torgossip_runtest(test_manager): assert bl.id in s.recv(10000) # test getting a block that doesn't exist - s.sendall(b'5' + os.urandom(64)) - assert s.recv(64) == b"0" + s.sendall(b'5' + int(secrets.randbits(64)).to_bytes(64, 'little')) + + #print(len(s.recv(64))) + assert s.recv(64)[0] == ord(b'0') # test getting a block that does exist s.sendall(b'5' + bl.id) @@ -63,8 +72,10 @@ def torgossip_runtest(test_manager): assert s.recv(1000) == tsts[64*2:] # test peer list - - shared_state.get_by_string('TorGossipPeers').add_peer() + #fake_peer = _fake_onion() + #shared_state.get_by_string('TorGossipPeers').add_peer(fake_peer) + #s.sendall(b'71') + #assert s.recv(100) == fake_peer s.sendall(b'9') assert s.recv(64) == b"BYE" diff --git a/static-data/default_config.json b/static-data/default_config.json index de1d4486..0e3a31b2 100755 --- a/static-data/default_config.json +++ b/static-data/default_config.json @@ -49,6 +49,9 @@ "disabled": [], "enabled": [] }, + "runtests": { + "skip_slow": false + }, "security": { "encrypt_database": false },