Browse Source

Work on gossip tests and cleaned up api server some

newtransport
Kevin F 4 weeks ago
parent
commit
1b37264eb7
  1. 4
      run_tests.sh
  2. 2
      src/apiservers/__init__.py
  3. 6
      src/apiservers/private/__init__.py
  4. 12
      src/apiservers/private/register_private_blueprints.py
  5. 4
      src/gossip/client/__init__.py
  6. 2
      src/gossip/client/dandelionstem/__init__.py
  7. 6
      src/gossip/server/__init__.py
  8. 2
      src/gossip/server/acceptstem.py
  9. 44
      src/httpapi/addblock/__init__.py
  10. 7
      src/httpapi/apiutils/shutdown.py
  11. 55
      src/httpapi/daemoneventsapi/__init__.py
  12. 11
      src/httpapi/miscclientapi/endpoints.py
  13. 6
      src/httpapi/security/client.py
  14. 15
      src/onionrcommands/daemonlaunch/__init__.py
  15. 1
      static-data/default-plugins/tor/bootstrap.py
  16. 67
      tests/gossip-unittests/test_put_block_mny.py
  17. 2
      tests/gossip-unittests/test_put_block_one.py
  18. 2
      tests/gossip-unittests/test_server_diffuse_one.py

4
run_tests.sh

@ -13,6 +13,10 @@ for f in tests/*.py; do @@ -13,6 +13,10 @@ for f in tests/*.py; do
python3 "$f" || close # if needed
let "ran++"
done
for f in tests/gossip-unittests/*.py; do
python3 "$f" || close # if needed
let "ran++"
done
echo "ran $ran unittests. Unittest Time: $SECONDS"
ran=0;

2
src/apiservers/__init__.py

@ -6,4 +6,4 @@ Private is meant for controlling and accessing this node @@ -6,4 +6,4 @@ Private is meant for controlling and accessing this node
from . import private
ClientAPI = private.PrivateAPI
private_api = private.private_api

6
src/apiservers/private/__init__.py

@ -65,7 +65,7 @@ class PrivateAPI: @@ -65,7 +65,7 @@ class PrivateAPI:
else:
self.host = httpapi.apiutils.setbindip.set_bind_IP(
private_API_host_file)
logger.info('Running api on %s:%s' % (self.host, self.bindPort))
logger.info(f'Running API on {self.host}:{self.bindPort}', terminal=True)
self.httpServer = ''
self.queueResponse = {}
@ -75,7 +75,6 @@ class PrivateAPI: @@ -75,7 +75,6 @@ class PrivateAPI:
def start(self):
"""Start client gevent API web server with flask client app."""
waitforsetvar.wait_for_set_var(self, "_too_many")
fd_handler = httpapi.fdsafehandler.FDSafeHandler
self.httpServer = WSGIServer((self.host, self.bindPort),
@ -116,3 +115,6 @@ class PrivateAPI: @@ -116,3 +115,6 @@ class PrivateAPI:
decrypt=decrypt,
raw=raw,
headerOnly=headerOnly)
private_api = PrivateAPI()

12
src/apiservers/private/register_private_blueprints.py

@ -41,16 +41,4 @@ def register_private_blueprints(private_api, app): @@ -41,16 +41,4 @@ def register_private_blueprints(private_api, app):
app.register_blueprint(private_sse_blueprint)
app.register_blueprint(fileoffsetreader.offset_reader_api)
def _add_events_bp():
while True:
try:
private_api._too_many
break
except AttributeError:
sleep(0.2)
app.register_blueprint(
private_api._too_many.get_by_string('DaemonEventsBP').flask_bp)
Thread(target=_add_events_bp, daemon=True).start()
return app

4
src/gossip/client/__init__.py

@ -68,7 +68,7 @@ def gossip_client(): @@ -68,7 +68,7 @@ def gossip_client():
# transport plugin handles the new peer
add_onionr_thread(
get_new_peers,
1200, initial_sleep=5)
120, initial_sleep=5)
# Start a new thread to stream blocks from peers
@ -76,8 +76,6 @@ def gossip_client(): @@ -76,8 +76,6 @@ def gossip_client():
dandelion_phase = DandelionPhase(DANDELION_EPOCH_LENGTH)
while True:
sleep(5)
continue
while not len(gossip_peer_set):
sleep(0.2)
if dandelion_phase.remaining_time() <= 10:

2
src/gossip/client/dandelionstem/__init__.py

@ -98,7 +98,7 @@ async def stem_out(d_phase: 'DandelionPhase'): @@ -98,7 +98,7 @@ async def stem_out(d_phase: 'DandelionPhase'):
peer_sockets.append(await _setup_edge(gossip_peer_set, tried_edges))
except NotEnoughEdges:
# No possible edges at this point (edges < OUTBOUND_DANDELION_EDGE)
logger.warn("Not able to build enough peers for stemout.",
logger.warn("Not able to build enough tunnels for stemout.",
terminal=True)
break
else:

6
src/gossip/server/__init__.py

@ -96,9 +96,13 @@ def gossip_server(): @@ -96,9 +96,13 @@ def gossip_server():
logger.debug(
"Inbound edge timed out when steming blocks to us",
terminal=True)
except asyncio.exceptions.IncompleteReadError:
logger.debug(
"Inbound edge timed out (Incomplete Read) when steming blocks to us",
terminal=True)
except Exception:
logger.warn(
f"Err getting\n{traceback.format_exc()}",
f"Err acceptind stem blocks\n{traceback.format_exc()}",
terminal=True)
# Subtract dandelion edge, make sure >=0
inbound_dandelion_edge_count[0] = \

2
src/gossip/server/acceptstem.py

@ -12,7 +12,7 @@ from ..blockqueues import gossip_block_queues @@ -12,7 +12,7 @@ from ..blockqueues import gossip_block_queues
block_size_digits = len(str(BLOCK_MAX_SIZE))
base_wait_timeout = 30
base_wait_timeout = 120
if TYPE_CHECKING:
from asyncio import StreamWriter, StreamReader

44
src/httpapi/addblock/__init__.py

@ -0,0 +1,44 @@ @@ -0,0 +1,44 @@
"""Onionr - Private P2P Communication.
Serialized APIs
"""
from asyncio.log import logger
import secrets
from flask import Blueprint, Response, request
from onionrblocks import Block
import logger
from gossip import blockqueues
from gossip.constants import BLOCK_ID_SIZE
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
blockapi = Blueprint('blockapi', __name__)
stream_to_use = secrets.randbits(1)
# Add a block that we generated (or received from a transport like LAN/sneakernet)
@blockapi.route('/addvdfblock', methods=['POST'])
def block_serialized():
req_data = request.data
block_id = req_data[:BLOCK_ID_SIZE]
block_data = req_data[BLOCK_ID_SIZE:]
blockqueues.gossip_block_queues[stream_to_use].put(
Block(block_id, block_data, auto_verify=False))
logger.info("Added block" + block_id, terminal=True)
return "ok"

7
src/httpapi/apiutils/shutdown.py

@ -28,10 +28,3 @@ def shutdown(client_api_inst): @@ -28,10 +28,3 @@ def shutdown(client_api_inst):
except AttributeError:
pass
return Response("bye")
@shutdown_bp.route('/shutdownclean')
def shutdown_clean():
# good for calling from other clients
g.too_many.get_by_string("DeadSimpleKV").put('shutdown', True)
return Response("bye")

55
src/httpapi/daemoneventsapi/__init__.py

@ -1,55 +0,0 @@ @@ -1,55 +0,0 @@
"""Onionr - Private P2P Communication.
Event driven interface to trigger events in communicator
"""
from typing import Callable
from flask import Blueprint, request, Response, abort
from werkzeug.exceptions import BadRequest
from gevent import spawn
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
class DaemonEventsBP:
def __init__(self):
"""Create DaemonEvents instance, intended to be a singleton.
Attributes:
listeners: callables that are called when a new event is added.
The callables name should match the event name
_too_many: TooManyObjects instance set by external code
"""
event_BP = Blueprint('event_BP', __name__)
self.listeners = set([])
self.flask_bp = event_BP
event_BP = self.flask_bp
@event_BP.route('/daemon-event/<name>', methods=['POST'])
def daemon_event_handler(name):
handler: Callable
try:
json_data = request.get_json(force=True)
except BadRequest:
json_data = {}
for handler in self.listeners:
if handler.__name__ == name:
return Response(handler(**json_data))
abort(404)
def register_listener(self, listener: Callable):
self.listeners.add(listener)

11
src/httpapi/miscclientapi/endpoints.py

@ -68,7 +68,6 @@ class PrivateEndpoints: @@ -68,7 +68,6 @@ class PrivateEndpoints:
return Response("pong!")
@private_endpoints_bp.route('/shutdown')
def shutdown():
return apiutils.shutdown.shutdown(client_api)
@ -110,13 +109,3 @@ class PrivateEndpoints: @@ -110,13 +109,3 @@ class PrivateEndpoints:
def get_os_system():
return Response(platform.system().lower())
@private_endpoints_bp.route('/getgeneratingblocks')
def get_generating_blocks() -> Response:
return Response(
','.join(
g.too_many.get_by_string('DeadSimpleKV').get(
'generating_blocks'
))
)

6
src/httpapi/security/client.py

@ -71,12 +71,6 @@ class ClientAPISecurity: @@ -71,12 +71,6 @@ class ClientAPISecurity:
f'Possible DNS rebinding attack by {request.host}')
abort(403)
# Add shared objects
try:
g.too_many = self.client_api._too_many
except KeyError:
g.too_many = None
# Static files for Onionr sites
if request.path.startswith('/site/'):
return

15
src/onionrcommands/daemonlaunch/__init__.py

@ -7,6 +7,7 @@ import queue @@ -7,6 +7,7 @@ import queue
import sys
import platform
import signal
from threading import Thread
from stem.connection import IncorrectPassword
import toomanyobjs
@ -28,7 +29,6 @@ import onionrvalues @@ -28,7 +29,6 @@ import onionrvalues
from onionrutils import cleanup
from onionrcrypto import getourkeypair
import runtests
from httpapi import daemoneventsapi
from .. import version
from .killdaemon import kill_daemon # noqa
from .showlogo import show_logo
@ -95,15 +95,11 @@ def daemon(): @@ -95,15 +95,11 @@ def daemon():
# Initialize the quasi-global variables
setup_kv(shared_state.get(DeadSimpleKV))
shared_state.get(daemoneventsapi.DaemonEventsBP)
# Init run time tester
# (ensures Onionr is running right, for testing purposes)
# Run time tests are not normally run
shared_state.get(runtests.OnionrRunTestManager)
# initialize clientAPI but dont start it yet
shared_state.get(apiservers.ClientAPI)
shared_state.share_object() # share the parent object to the threads
@ -117,10 +113,10 @@ def daemon(): @@ -117,10 +113,10 @@ def daemon():
events.event('init', threaded=False)
events.event('daemon_start')
gossip.start_gossip_threads()
Thread(target=gossip.start_gossip_threads, daemon=True).start()
try:
shared_state.get(apiservers.ClientAPI).start()
apiservers.private_api.start()
except KeyboardInterrupt:
pass
@ -149,8 +145,9 @@ def start(override: bool = False): @@ -149,8 +145,9 @@ def start(override: bool = False):
except psutil.NoSuchProcess:
proc = ""
if not proc.startswith("python"):
logger.info(
f"Detected stale run file, deleting {filepaths.lock_file}", terminal=True)
logger.warn(
f"Detected stale run file, deleting {filepaths.lock_file}",
terminal=True)
try:
os.remove(filepaths.lock_file)
except FileNotFoundError:

1
static-data/default-plugins/tor/bootstrap.py

@ -43,4 +43,3 @@ def on_bootstrap(api, data): @@ -43,4 +43,3 @@ def on_bootstrap(api, data):
args=[TorPeer(socks_address, socks_port, address)],
daemon=True).start()

67
tests/gossip-unittests/test_put_block_mny.py

@ -0,0 +1,67 @@ @@ -0,0 +1,67 @@
import os, uuid
from queue import Empty
TEST_DIR = 'testdata/%s-%s' % (str(uuid.uuid4())[:5], os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
from time import sleep
from threading import Thread
import asyncio
import unittest
import sys
sys.path.append(".")
sys.path.append("src/")
from ordered_set import OrderedSet
import onionrblocks
import blockdb
from gossip.server import gossip_server
from gossip.blockqueues import gossip_block_queues
from filepaths import gossip_server_socket_file
BLOCK_MAX_SIZE = 1024 * 2000
BLOCK_MAX_SIZE_LEN = len(str(BLOCK_MAX_SIZE))
BLOCK_ID_SIZE = 128
BLOCK_STREAM_OFFSET_DIGITS = 8
class OnionrServerPutBlocksTest(unittest.TestCase):
def test_put_blocks(self):
Thread(target=gossip_server, daemon=True).start()
sleep(0.01)
blocks = []
for _ in range(10):
bl = onionrblocks.blockcreator.create_anonvdf_block(
b"my test block" + os.urandom(16), b"txt", 2800)
blockdb.add_block_to_db(bl)
blocks.append(bl)
async def blocks_put_client():
reader, writer = await asyncio.open_unix_connection(
gossip_server_socket_file)
writer.write(int(5).to_bytes(1, 'big'))
await writer.drain()
for bl in blocks:
writer.write(bl.id)
writer.write(
str(len(bl.raw)).zfill(BLOCK_MAX_SIZE_LEN).encode('utf-8'))
writer.write(bl.raw)
await writer.drain()
sleep(0.03)
try:
self.assertEqual(gossip_block_queues[0].get_nowait().raw, bl.raw)
except Empty:
self.assertEqual(gossip_block_queues[1].get_nowait().raw, bl.raw)
asyncio.run(blocks_put_client())
unittest.main()

2
tests/gossip-unittests/test_put_block_one.py

@ -1,6 +1,6 @@ @@ -1,6 +1,6 @@
import os, uuid
from queue import Empty
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
TEST_DIR = 'testdata/%s-%s' % (str(uuid.uuid4())[:6], os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR

2
tests/gossip-unittests/test_server_diffuse_one.py

@ -1,5 +1,5 @@ @@ -1,5 +1,5 @@
import os, uuid
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
TEST_DIR = 'testdata/%s-%s' % (str(uuid.uuid4())[:6], os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR

Loading…
Cancel
Save