From 7fba65c459c307d7cb0edd1d4c41935c59475451 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 21 Feb 2021 23:22:18 +0000 Subject: [PATCH] work on torgossip --- src/apiservers/private/__init__.py | 2 + src/blockio/store/__init__.py | 2 +- src/blockio/subprocgenerate.py | 20 ++- src/httpapi/__init__.py | 35 ++-- src/httpapi/serializedapi/__init__.py | 18 +- src/httpapi/wrappedfunctions/__init__.py | 11 ++ src/runtests/__init__.py | 4 +- src/runtests/wrappedfunctionstest.py | 29 +++ static-data/default-plugins/circles/main.py | 169 ++---------------- .../default-plugins/torgossip/runtest.py | 2 +- 10 files changed, 117 insertions(+), 175 deletions(-) create mode 100644 src/httpapi/wrappedfunctions/__init__.py create mode 100644 src/runtests/wrappedfunctionstest.py diff --git a/src/apiservers/private/__init__.py b/src/apiservers/private/__init__.py index 8c0571a4..1c4bed9f 100644 --- a/src/apiservers/private/__init__.py +++ b/src/apiservers/private/__init__.py @@ -2,6 +2,7 @@ This file handles all incoming http requests to the client, using Flask """ +import http from typing import Dict import hmac @@ -77,6 +78,7 @@ class PrivateAPI: """Start client gevent API web server with flask client app.""" waitforsetvar.wait_for_set_var(self, "_too_many") fd_handler = httpapi.fdsafehandler.FDSafeHandler + self._too_many.add(httpapi.wrappedfunctions.SubProcVDFGenerator(self._too_many)) self.publicAPI = self._too_many.get( # pylint: disable=E1101 public.PublicAPI) self.httpServer = WSGIServer((self.host, self.bindPort), diff --git a/src/blockio/store/__init__.py b/src/blockio/store/__init__.py index 42cd93cb..d7be4929 100644 --- a/src/blockio/store/__init__.py +++ b/src/blockio/store/__init__.py @@ -23,7 +23,7 @@ along with this program. If not, see . """ -def store_block(block: 'Kasten', safe_db: 'SafeDB'): +def store_block(block: 'Kasten', safe_db: 'SafeDB', own_block=False): # This does not handle validation of blocks # safe_db is initialized by the daemon when starting normally # so any other commands need to initialize it seperately diff --git a/src/blockio/subprocgenerate.py b/src/blockio/subprocgenerate.py index 58d8b648..b009a013 100644 --- a/src/blockio/subprocgenerate.py +++ b/src/blockio/subprocgenerate.py @@ -1,16 +1,23 @@ from base64 import b85encode import os import subprocess +import threading import ujson as json - import kasten + +from onionrplugins import onionrevents +from blockio import store_block from onionrblocks.generators.anonvdf import AnonVDFGenerator _DIR = os.path.dirname(os.path.realpath(__file__)) + '/../' def vdf_block(data, data_type, ttl, **metadata): + try: + data = data.encode('utf-8') + except AttributeError: + pass data = b85encode(data) generated = subprocess.Popen( [ @@ -25,3 +32,14 @@ def vdf_block(data, data_type, ttl, **metadata): generated[:64], generated[64:], AnonVDFGenerator, auto_check_generator=True) + +def gen_and_store_vdf_block(shared_state, *args, **kwargs): + safe_db = shared_state.get_by_string('SafeDB') + k = vdf_block(*args, **kwargs) + store_block( + k, + safe_db, + own_block=True + ) + onionrevents.event('blockcreated', data=shared_state, threaded=True) + diff --git a/src/httpapi/__init__.py b/src/httpapi/__init__.py index 101a20c6..ee69a8aa 100755 --- a/src/httpapi/__init__.py +++ b/src/httpapi/__init__.py @@ -1,25 +1,28 @@ """ - Onionr - Private P2P Communication +Onionr - Private P2P Communication - This file registers plugin's flask blueprints for the client http server -""" -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . +Register plugin's flask blueprints for the client http server """ import onionrplugins import config +from . import wrappedfunctions +""" +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + + def load_plugin_blueprints(flaskapp, blueprint: str = 'flask_blueprint'): """Iterate enabled plugins and load any http endpoints they have""" config.reload() diff --git a/src/httpapi/serializedapi/__init__.py b/src/httpapi/serializedapi/__init__.py index 83c939e9..6dae5140 100644 --- a/src/httpapi/serializedapi/__init__.py +++ b/src/httpapi/serializedapi/__init__.py @@ -2,6 +2,8 @@ view and interact with onionr sites """ +import traceback + from flask import Blueprint, Response, request, abort, g import ujson as json """ @@ -33,9 +35,10 @@ def serialized(name: str) -> Response: resp = str(resp) return Response(resp, content_type='application/octet-stream') except Exception as e: - return Response(repr(e), content_type='text/plain', status=500) + return Response(traceback.format_exc(e), content_type='text/plain', status=500) initial = g.too_many.get_by_string(name.split('.')[0]) + print('initial', initial) for c, i in enumerate(name.split('.')): if i and c != 0: attr = getattr(initial, i) @@ -44,11 +47,16 @@ def serialized(name: str) -> Response: try: js = request.get_json(force=True) print('json', js, type(js)) - data = json.loads(js) - args = data['args'] - del data['args'] + if not isinstance(js, dict): + data = json.loads(js) + args = data['args'] + del data['args'] + else: + data = js + args = js['args'] + del js['args'] except (TypeError, ValueError) as e: - print(repr(e)) + print(traceback.format_exc()) data = {} args = [] print('data', data) diff --git a/src/httpapi/wrappedfunctions/__init__.py b/src/httpapi/wrappedfunctions/__init__.py new file mode 100644 index 00000000..56eda24b --- /dev/null +++ b/src/httpapi/wrappedfunctions/__init__.py @@ -0,0 +1,11 @@ +from blockio import subprocgenerate + + +class SubProcVDFGenerator: + def __init__(self, shared_state): + self.shared_state = shared_state + + def gen_and_store_vdf_block(self, *args, **kwargs): + + return subprocgenerate.gen_and_store_vdf_block( + self.shared_state, *args, **kwargs) diff --git a/src/runtests/__init__.py b/src/runtests/__init__.py index 8f90d35e..4fabee69 100644 --- a/src/runtests/__init__.py +++ b/src/runtests/__init__.py @@ -19,6 +19,7 @@ from .lanservertest import test_lan_server from .sneakernettest import test_sneakernet_import from .dnsrebindingtest import test_dns_rebinding from .serviceonlinetest import test_service_online +from .wrappedfunctionstest import test_vdf_create_and_store """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -46,7 +47,8 @@ RUN_TESTS = [uicheck.check_ui, test_lan_server, test_sneakernet_import, test_dns_rebinding, - test_service_online + test_service_online, + test_vdf_create_and_store ] SUCCESS_FILE = os.path.dirname(os.path.realpath(__file__)) + '/../../tests/runtime-result.txt' diff --git a/src/runtests/wrappedfunctionstest.py b/src/runtests/wrappedfunctionstest.py new file mode 100644 index 00000000..ca1e6106 --- /dev/null +++ b/src/runtests/wrappedfunctionstest.py @@ -0,0 +1,29 @@ +from time import sleep +from secrets import token_hex + +import blockio +import logger +from onionrutils.localcommand import local_command +from blockio import list_all_blocks + + +def test_vdf_create_and_store(testmanager): + # data, data_type, ttl, **metadata + db = testmanager._too_many.get_by_string('SafeDB') + bls = list_all_blocks(db) + b_data = "test" + token_hex(5) + res = local_command( + '/serialized/SubProcVDFGenerator.gen_and_store_vdf_block', post=True, post_data={"args": [b_data, "txt", 6000]}, is_json=True) + + print(res) + + while len(list_all_blocks(db)) == len(bls): + sleep(1) + for i in list_all_blocks(db): + i = bytes(i) + if blockio.load_block(i, db).get_packed().decode('utf-8') == b_data: + break + else: + logger.error("Block was not generated", terminal=True) + raise ValueError + diff --git a/static-data/default-plugins/circles/main.py b/static-data/default-plugins/circles/main.py index 7515173f..d2b840be 100755 --- a/static-data/default-plugins/circles/main.py +++ b/static-data/default-plugins/circles/main.py @@ -5,172 +5,41 @@ This default plugin handles "flow" messages """ import sys import os -import deadsimplekv as simplekv -from utils import identifyhome, reconstructhash -from coredb import blockmetadb -import threading -import time -import locale -from oldblocks.onionrblockapi import Block -import logger -import oldblocks -from onionrutils import escapeansi, epoch, bytesconverter +import locale locale.setlocale(locale.LC_ALL, '') + +from blockio.subprocgenerate import gen_and_store_vdf_block + sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) # import after path insert import flowapi # noqa """ - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. - You should have received a copy of the GNU General Public License - along with this program. If not, see . +You should have received a copy of the GNU General Public License +along with this program. If not, see . """ flask_blueprint = flowapi.flask_blueprint security_whitelist = ['circles.circlesstatic', 'circles.circlesindex'] plugin_name = 'circles' -PLUGIN_VERSION = '0.1.0' - -EXPIRE_TIME = 43200 - -class OnionrFlow: - def __init__(self): - self.alreadyOutputed = [] - self.flowRunning = False - self.channel = "" - return - - def start(self): - logger.warn( - "Please note: everything said here is public, " + - "even if a random channel name is used.", terminal=True) - message = "" - self.flowRunning = True - try: - self.channel = logger.readline( - "Enter a channel name or none for default:").strip() - except (KeyboardInterrupt, EOFError): - self.flowRunning = False - newThread = threading.Thread(target=self.showOutput, daemon=True) - newThread.start() - while self.flowRunning: - if self.channel == "": - self.channel = "global" - try: - message = logger.readline(f'\nInsert message into {plugin_name}:').strip().replace( - '\n', '\\n').replace('\r', '\\r') - except EOFError: - pass - except KeyboardInterrupt: - self.flowRunning = False - else: - if message == "q": - self.flowRunning = False - expireTime = epoch.get_epoch() + EXPIRE_TIME - if len(message) > 0: - logger.info('Inserting message as block...', terminal=True) - oldblocks.insert(message, header='brd', - expire=expireTime, - meta = { - 'ch': self.channel}) - - logger.info(f"{plugin_name} is exiting, goodbye", terminal=True) - return - - def showOutput(self): - while isinstance(self.channel, type(None)) and self.flowRunning: - time.sleep(1) - try: - while self.flowRunning: - for block in blockmetadb.get_blocks_by_type('brd'): - if block in self.alreadyOutputed: - continue - block = Block(block) - b_hash = bytesconverter.bytes_to_str(block.getHash()) - if block.getMetadata('ch') != self.channel: - continue - if not self.flowRunning: - break - logger.info('\n------------------------', - prompt=False, terminal=True) - content = block.getContent() - # Escape new lines, remove trailing whitespace, and escape ansi sequences - content = escapeansi.escape_ANSI(content.replace( - b'\n', b'\\n').replace(b'\r', b'\\r').strip().decode('utf-8')) - logger.info(block.getDate().strftime( - "%m/%d %H:%M") + ' - ' + - logger.colors.reset + content, - prompt=False, terminal=True) - self.alreadyOutputed.append(b_hash) - time.sleep(5) - except KeyboardInterrupt: - self.flowRunning = False +PLUGIN_VERSION = '1.0.0' def on_circles_cmd(api, data=None): - OnionrFlow().start() + message = "" + while message != "-q": + message = input("Enter message") + gen_and_store_vdf_block() -def on_circlesend_cmd(api, data=None): - err_msg = "Second arg is board name, third is quoted message" - try: - sys.argv[2] - except IndexError: - logger.error(err_msg, terminal=True) - try: - sys.argv[3] - except IndexError: - logger.error(err_msg, terminal=True) - - bl = oldblocks.insert(sys.argv[3], header='brd', - expire=(EXPIRE_TIME + epoch.get_epoch()), - meta={'ch': sys.argv[2]}) - print(bl) - - - -def on_softreset(api, data=None): - try: - os.remove(identifyhome.identify_home() + '/board-index.cache.json') - logger.info('Cleared Circles board cache') - except FileNotFoundError: - pass - - -def on_processblocks(api, data=None): - metadata = data['block'].bmetadata # Get the block metadata - if data['type'] != 'brd': - return - - b_hash = reconstructhash.deconstruct_hash( - data['block'].hash) # Get the 0-truncated block hash - board_cache = simplekv.DeadSimpleKV(identifyhome.identify_home( - ) + '/board-index.cache.json', flush_on_exit=False) # get the board index cache - board_cache.refresh() - # Validate the channel name is sane for caching - try: - ch = metadata['ch'] - except KeyError: - ch = 'global' - ch_len = len(ch) - if ch_len == 0: - ch = 'global' - elif ch_len > 12: - return - - existing_posts = board_cache.get(ch) - if existing_posts is None: - existing_posts = [] - existing_posts.append(data['block'].hash) - board_cache.put(ch, existing_posts) - board_cache.flush() diff --git a/static-data/default-plugins/torgossip/runtest.py b/static-data/default-plugins/torgossip/runtest.py index 1fe2a74b..aa134b37 100644 --- a/static-data/default-plugins/torgossip/runtest.py +++ b/static-data/default-plugins/torgossip/runtest.py @@ -21,7 +21,7 @@ def _shrink_peer_address(peer): return peer def torgossip_runtest(test_manager): - + return s_file = identifyhome.identify_home() + "/torgossip.sock" bl_test = blockcreator.create_anonvdf_block(b"test", "txt", 10)