diff --git a/run_tests.sh b/run_tests.sh
index 1ebee923..d019019d 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -13,6 +13,10 @@ for f in tests/*.py; do
python3 "$f" || close # if needed
let "ran++"
done
+for f in tests/gossip-unittests/*.py; do
+ python3 "$f" || close # if needed
+ let "ran++"
+done
echo "ran $ran unittests. Unittest Time: $SECONDS"
ran=0;
diff --git a/src/apiservers/__init__.py b/src/apiservers/__init__.py
index f9926150..6b78e13b 100755
--- a/src/apiservers/__init__.py
+++ b/src/apiservers/__init__.py
@@ -6,4 +6,4 @@ Private is meant for controlling and accessing this node
from . import private
-ClientAPI = private.PrivateAPI
+private_api = private.private_api
diff --git a/src/apiservers/private/__init__.py b/src/apiservers/private/__init__.py
index e42bc435..bef974dc 100644
--- a/src/apiservers/private/__init__.py
+++ b/src/apiservers/private/__init__.py
@@ -65,7 +65,7 @@ class PrivateAPI:
else:
self.host = httpapi.apiutils.setbindip.set_bind_IP(
private_API_host_file)
- logger.info('Running api on %s:%s' % (self.host, self.bindPort))
+ logger.info(f'Running API on {self.host}:{self.bindPort}', terminal=True)
self.httpServer = ''
self.queueResponse = {}
@@ -75,7 +75,6 @@ class PrivateAPI:
def start(self):
"""Start client gevent API web server with flask client app."""
- waitforsetvar.wait_for_set_var(self, "_too_many")
fd_handler = httpapi.fdsafehandler.FDSafeHandler
self.httpServer = WSGIServer((self.host, self.bindPort),
@@ -116,3 +115,6 @@ class PrivateAPI:
decrypt=decrypt,
raw=raw,
headerOnly=headerOnly)
+
+
+private_api = PrivateAPI()
\ No newline at end of file
diff --git a/src/apiservers/private/register_private_blueprints.py b/src/apiservers/private/register_private_blueprints.py
index fb68b229..8d68032e 100644
--- a/src/apiservers/private/register_private_blueprints.py
+++ b/src/apiservers/private/register_private_blueprints.py
@@ -41,16 +41,4 @@ def register_private_blueprints(private_api, app):
app.register_blueprint(private_sse_blueprint)
app.register_blueprint(fileoffsetreader.offset_reader_api)
- def _add_events_bp():
- while True:
- try:
- private_api._too_many
- break
- except AttributeError:
- sleep(0.2)
- app.register_blueprint(
- private_api._too_many.get_by_string('DaemonEventsBP').flask_bp)
-
- Thread(target=_add_events_bp, daemon=True).start()
-
return app
diff --git a/src/gossip/client/__init__.py b/src/gossip/client/__init__.py
index ea089e0a..4860e62f 100644
--- a/src/gossip/client/__init__.py
+++ b/src/gossip/client/__init__.py
@@ -68,7 +68,7 @@ def gossip_client():
# transport plugin handles the new peer
add_onionr_thread(
get_new_peers,
- 1200, initial_sleep=5)
+ 120, initial_sleep=5)
# Start a new thread to stream blocks from peers
@@ -76,8 +76,6 @@ def gossip_client():
dandelion_phase = DandelionPhase(DANDELION_EPOCH_LENGTH)
while True:
- sleep(5)
- continue
while not len(gossip_peer_set):
sleep(0.2)
if dandelion_phase.remaining_time() <= 10:
diff --git a/src/gossip/client/dandelionstem/__init__.py b/src/gossip/client/dandelionstem/__init__.py
index 5e538a2f..fe5c23d7 100644
--- a/src/gossip/client/dandelionstem/__init__.py
+++ b/src/gossip/client/dandelionstem/__init__.py
@@ -98,7 +98,7 @@ async def stem_out(d_phase: 'DandelionPhase'):
peer_sockets.append(await _setup_edge(gossip_peer_set, tried_edges))
except NotEnoughEdges:
# No possible edges at this point (edges < OUTBOUND_DANDELION_EDGE)
- logger.warn("Not able to build enough peers for stemout.",
+ logger.warn("Not able to build enough tunnels for stemout.",
terminal=True)
break
else:
diff --git a/src/gossip/server/__init__.py b/src/gossip/server/__init__.py
index 5f15ac30..c46cf87f 100644
--- a/src/gossip/server/__init__.py
+++ b/src/gossip/server/__init__.py
@@ -96,9 +96,13 @@ def gossip_server():
logger.debug(
"Inbound edge timed out when steming blocks to us",
terminal=True)
+ except asyncio.exceptions.IncompleteReadError:
+ logger.debug(
+ "Inbound edge timed out (Incomplete Read) when steming blocks to us",
+ terminal=True)
except Exception:
logger.warn(
- f"Err getting\n{traceback.format_exc()}",
+ f"Err acceptind stem blocks\n{traceback.format_exc()}",
terminal=True)
# Subtract dandelion edge, make sure >=0
inbound_dandelion_edge_count[0] = \
diff --git a/src/gossip/server/acceptstem.py b/src/gossip/server/acceptstem.py
index 2fe24685..0e50b73f 100644
--- a/src/gossip/server/acceptstem.py
+++ b/src/gossip/server/acceptstem.py
@@ -12,7 +12,7 @@ from ..blockqueues import gossip_block_queues
block_size_digits = len(str(BLOCK_MAX_SIZE))
-base_wait_timeout = 30
+base_wait_timeout = 120
if TYPE_CHECKING:
from asyncio import StreamWriter, StreamReader
diff --git a/src/httpapi/addblock/__init__.py b/src/httpapi/addblock/__init__.py
new file mode 100644
index 00000000..bb19020f
--- /dev/null
+++ b/src/httpapi/addblock/__init__.py
@@ -0,0 +1,44 @@
+"""Onionr - Private P2P Communication.
+
+Serialized APIs
+"""
+
+from asyncio.log import logger
+import secrets
+from flask import Blueprint, Response, request
+
+from onionrblocks import Block
+
+import logger
+from gossip import blockqueues
+from gossip.constants import BLOCK_ID_SIZE
+
+"""
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+"""
+blockapi = Blueprint('blockapi', __name__)
+
+
+stream_to_use = secrets.randbits(1)
+
+# Add a block that we generated (or received from a transport like LAN/sneakernet)
+@blockapi.route('/addvdfblock', methods=['POST'])
+def block_serialized():
+ req_data = request.data
+ block_id = req_data[:BLOCK_ID_SIZE]
+ block_data = req_data[BLOCK_ID_SIZE:]
+ blockqueues.gossip_block_queues[stream_to_use].put(
+ Block(block_id, block_data, auto_verify=False))
+ logger.info("Added block" + block_id, terminal=True)
+ return "ok"
\ No newline at end of file
diff --git a/src/httpapi/apiutils/shutdown.py b/src/httpapi/apiutils/shutdown.py
index 6e5a085d..9f96c1f9 100644
--- a/src/httpapi/apiutils/shutdown.py
+++ b/src/httpapi/apiutils/shutdown.py
@@ -28,10 +28,3 @@ def shutdown(client_api_inst):
except AttributeError:
pass
return Response("bye")
-
-
-@shutdown_bp.route('/shutdownclean')
-def shutdown_clean():
- # good for calling from other clients
- g.too_many.get_by_string("DeadSimpleKV").put('shutdown', True)
- return Response("bye")
diff --git a/src/httpapi/daemoneventsapi/__init__.py b/src/httpapi/daemoneventsapi/__init__.py
deleted file mode 100644
index feb61141..00000000
--- a/src/httpapi/daemoneventsapi/__init__.py
+++ /dev/null
@@ -1,55 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Event driven interface to trigger events in communicator
-"""
-from typing import Callable
-
-from flask import Blueprint, request, Response, abort
-from werkzeug.exceptions import BadRequest
-from gevent import spawn
-
-"""
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see .
-"""
-
-
-class DaemonEventsBP:
- def __init__(self):
- """Create DaemonEvents instance, intended to be a singleton.
-
- Attributes:
- listeners: callables that are called when a new event is added.
- The callables name should match the event name
- _too_many: TooManyObjects instance set by external code
- """
- event_BP = Blueprint('event_BP', __name__)
- self.listeners = set([])
- self.flask_bp = event_BP
- event_BP = self.flask_bp
-
- @event_BP.route('/daemon-event/', methods=['POST'])
- def daemon_event_handler(name):
- handler: Callable
-
- try:
- json_data = request.get_json(force=True)
- except BadRequest:
- json_data = {}
- for handler in self.listeners:
- if handler.__name__ == name:
- return Response(handler(**json_data))
- abort(404)
-
- def register_listener(self, listener: Callable):
- self.listeners.add(listener)
diff --git a/src/httpapi/miscclientapi/endpoints.py b/src/httpapi/miscclientapi/endpoints.py
index e4c7e8d1..f57bc2e1 100644
--- a/src/httpapi/miscclientapi/endpoints.py
+++ b/src/httpapi/miscclientapi/endpoints.py
@@ -68,7 +68,6 @@ class PrivateEndpoints:
return Response("pong!")
-
@private_endpoints_bp.route('/shutdown')
def shutdown():
return apiutils.shutdown.shutdown(client_api)
@@ -110,13 +109,3 @@ class PrivateEndpoints:
def get_os_system():
return Response(platform.system().lower())
-
- @private_endpoints_bp.route('/getgeneratingblocks')
- def get_generating_blocks() -> Response:
- return Response(
- ','.join(
- g.too_many.get_by_string('DeadSimpleKV').get(
- 'generating_blocks'
- ))
- )
-
diff --git a/src/httpapi/security/client.py b/src/httpapi/security/client.py
index d05f7f41..2fd3b7d6 100644
--- a/src/httpapi/security/client.py
+++ b/src/httpapi/security/client.py
@@ -71,12 +71,6 @@ class ClientAPISecurity:
f'Possible DNS rebinding attack by {request.host}')
abort(403)
- # Add shared objects
- try:
- g.too_many = self.client_api._too_many
- except KeyError:
- g.too_many = None
-
# Static files for Onionr sites
if request.path.startswith('/site/'):
return
diff --git a/src/onionrcommands/daemonlaunch/__init__.py b/src/onionrcommands/daemonlaunch/__init__.py
index 7dffd4db..2dba869d 100755
--- a/src/onionrcommands/daemonlaunch/__init__.py
+++ b/src/onionrcommands/daemonlaunch/__init__.py
@@ -7,6 +7,7 @@ import queue
import sys
import platform
import signal
+from threading import Thread
from stem.connection import IncorrectPassword
import toomanyobjs
@@ -28,7 +29,6 @@ import onionrvalues
from onionrutils import cleanup
from onionrcrypto import getourkeypair
import runtests
-from httpapi import daemoneventsapi
from .. import version
from .killdaemon import kill_daemon # noqa
from .showlogo import show_logo
@@ -95,15 +95,11 @@ def daemon():
# Initialize the quasi-global variables
setup_kv(shared_state.get(DeadSimpleKV))
- shared_state.get(daemoneventsapi.DaemonEventsBP)
-
# Init run time tester
# (ensures Onionr is running right, for testing purposes)
# Run time tests are not normally run
shared_state.get(runtests.OnionrRunTestManager)
- # initialize clientAPI but dont start it yet
- shared_state.get(apiservers.ClientAPI)
shared_state.share_object() # share the parent object to the threads
@@ -117,10 +113,10 @@ def daemon():
events.event('init', threaded=False)
events.event('daemon_start')
- gossip.start_gossip_threads()
+ Thread(target=gossip.start_gossip_threads, daemon=True).start()
try:
- shared_state.get(apiservers.ClientAPI).start()
+ apiservers.private_api.start()
except KeyboardInterrupt:
pass
@@ -149,8 +145,9 @@ def start(override: bool = False):
except psutil.NoSuchProcess:
proc = ""
if not proc.startswith("python"):
- logger.info(
- f"Detected stale run file, deleting {filepaths.lock_file}", terminal=True)
+ logger.warn(
+ f"Detected stale run file, deleting {filepaths.lock_file}",
+ terminal=True)
try:
os.remove(filepaths.lock_file)
except FileNotFoundError:
diff --git a/static-data/default-plugins/tor/bootstrap.py b/static-data/default-plugins/tor/bootstrap.py
index 181b1140..225363b0 100644
--- a/static-data/default-plugins/tor/bootstrap.py
+++ b/static-data/default-plugins/tor/bootstrap.py
@@ -43,4 +43,3 @@ def on_bootstrap(api, data):
args=[TorPeer(socks_address, socks_port, address)],
daemon=True).start()
-
diff --git a/tests/gossip-unittests/test_put_block_mny.py b/tests/gossip-unittests/test_put_block_mny.py
new file mode 100644
index 00000000..7289056e
--- /dev/null
+++ b/tests/gossip-unittests/test_put_block_mny.py
@@ -0,0 +1,67 @@
+import os, uuid
+from queue import Empty
+TEST_DIR = 'testdata/%s-%s' % (str(uuid.uuid4())[:5], os.path.basename(__file__)) + '/'
+print("Test directory:", TEST_DIR)
+os.environ["ONIONR_HOME"] = TEST_DIR
+
+from time import sleep
+from threading import Thread
+import asyncio
+import unittest
+import sys
+sys.path.append(".")
+sys.path.append("src/")
+
+from ordered_set import OrderedSet
+import onionrblocks
+
+import blockdb
+from gossip.server import gossip_server
+from gossip.blockqueues import gossip_block_queues
+from filepaths import gossip_server_socket_file
+
+
+BLOCK_MAX_SIZE = 1024 * 2000
+BLOCK_MAX_SIZE_LEN = len(str(BLOCK_MAX_SIZE))
+BLOCK_ID_SIZE = 128
+BLOCK_STREAM_OFFSET_DIGITS = 8
+
+
+class OnionrServerPutBlocksTest(unittest.TestCase):
+
+
+ def test_put_blocks(self):
+
+ Thread(target=gossip_server, daemon=True).start()
+ sleep(0.01)
+
+ blocks = []
+ for _ in range(10):
+ bl = onionrblocks.blockcreator.create_anonvdf_block(
+ b"my test block" + os.urandom(16), b"txt", 2800)
+ blockdb.add_block_to_db(bl)
+ blocks.append(bl)
+
+ async def blocks_put_client():
+ reader, writer = await asyncio.open_unix_connection(
+ gossip_server_socket_file)
+ writer.write(int(5).to_bytes(1, 'big'))
+ await writer.drain()
+
+ for bl in blocks:
+ writer.write(bl.id)
+ writer.write(
+ str(len(bl.raw)).zfill(BLOCK_MAX_SIZE_LEN).encode('utf-8'))
+ writer.write(bl.raw)
+ await writer.drain()
+
+ sleep(0.03)
+ try:
+ self.assertEqual(gossip_block_queues[0].get_nowait().raw, bl.raw)
+ except Empty:
+ self.assertEqual(gossip_block_queues[1].get_nowait().raw, bl.raw)
+
+
+ asyncio.run(blocks_put_client())
+
+unittest.main()
diff --git a/tests/gossip-unittests/test_put_block_one.py b/tests/gossip-unittests/test_put_block_one.py
index 72daea4f..b3337951 100644
--- a/tests/gossip-unittests/test_put_block_one.py
+++ b/tests/gossip-unittests/test_put_block_one.py
@@ -1,6 +1,6 @@
import os, uuid
from queue import Empty
-TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
+TEST_DIR = 'testdata/%s-%s' % (str(uuid.uuid4())[:6], os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
diff --git a/tests/gossip-unittests/test_server_diffuse_one.py b/tests/gossip-unittests/test_server_diffuse_one.py
index dce76b8f..c2957dce 100644
--- a/tests/gossip-unittests/test_server_diffuse_one.py
+++ b/tests/gossip-unittests/test_server_diffuse_one.py
@@ -1,5 +1,5 @@
import os, uuid
-TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
+TEST_DIR = 'testdata/%s-%s' % (str(uuid.uuid4())[:6], os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR