Work on gossip system and tor transport

src/apiservers/private/__init__.py

@@ -3,6 +3,8 @@
 This file handles all incoming http requests to the client, using Flask
 """
 from typing import Dict
+from typing import Set
+from typing import TYPE_CHECKING
 import hmac
 
 import flask
@@ -51,6 +53,9 @@ class PrivateAPI:
         self.startTime = epoch.get_epoch()
         app = flask.Flask(__name__)
 
+        self.gossip_block_queue: 'queue.Queue' = None
+        self.gossip_peer_set: Set['Peer'] = None
+
 
         bind_port = int(config.get('client.client.port', 59496))
         self.bindPort = bind_port

src/bigbrother/ministry/ofcommunication.py

@@ -29,6 +29,8 @@ def detect_socket_leaks(socket_event):
     raises exception & logs if not to loopback
     """
     ip_address = socket_event[1][0]
+    if ip_address.startswith('/'):
+        return
 
     # validate is valid ip address (no hostname, etc)
     # raises NetworkLeak if not

src/bigbrother/ministry/ofexec.py

@@ -59,7 +59,10 @@ def block_exec(event, info):
                         'apport/report.py'
                        ]
     whitelisted_source = [
-        'ZABaAGQBZAJsAW0CWgIBAHoGZAFkA2wDWgRXAG4LBABlBXkZAQABAAEAZARaBlkAbgN3AGQFWgZkAWQGbAdtCFoIAQBkAWQDbAlaCWQBZAdsCm0LWgwBAAkAZQmgDWQIZAWhAnM1ZARaBmUIgwBkCRcAWg5kEmQMZQ9kDWUPZgRkDmQPhAVaEGQTZBBkEYQBWhFkA1MA'
+        'ZABaAGQBZAJsAW0CWgIBAHoGZAFkA2wDWgRXAG4LBABlBXkZAQABAAEAZARaBlkAbgN3AGQFWgZkAWQGbAdtCFoIAQBkAWQDbAlaCWQBZAdsCm0LWgwBAAkAZQmgDWQIZAWhAnM1ZARaBmUIgwBkCRcAWg5kEmQMZQ9kDWUPZgRkDmQPhAVaEGQTZBBkEYQBWhFkA1MA',
+        'RwBkAGQBhABkAYMCWgBkAlMA',
+        'ZABaAGQBZAJsAVoBZAFkAmwCWgJkAWQCbANaA2QBZAJsBFoEZAFkAmwFWgVkAWQCbAZaBmQBZAJsB1oIZAFkAmwJWghkAWQCbApaCGQBZAJsC1oIZANaDGQEWg1kBWQCZAJkBmQCZQ1kB2QIZAJmCWQJZAqEAVoOZAVkBmQCZQ1kB2QIZgZkC2QMhAFaD2QCUwA=',
+        'ZABaAGQBZAJsAVoBZAFkAmwCWgJkAWQCbANaA2QBZAJsBFoEZAFkAmwFWgVkAWQCbAZaBmQBZAJsB1oIZAFkAmwJWghkAWQCbApaCGQBZANsC20MWgxtDVoNbQ5aDm0PWg9tEFoQbRFaEQEAZARaEmQFWhNkBmQCZAJkB2QCZRNkCGQJZAJmCWQKZRRkC2UPZRBlFBkAGQBkDGUPZRQZAGQNZRVkDmUPZQ1lFGcBZAJmAhkAGQBkD2UVZBBlFmQRZRZkEmUPZRQZAGQTZQRqF2YUZBRkFYQFWhhkBmQHZAJlE2QIZAlmBmQWZQ5lFGURZRRlEGUUGQBmAhkAZgIZAGQKZRRkDWUVZA5lD2UNZRRnAWQCZgIZABkAZA9lFWQQZRZkEWUWZBNlBGoXZhBkF2QYhAVaGWQCUwA='
     ]
     home = identifyhome.identify_home()
 

src/filepaths/__init__.py

@@ -4,6 +4,9 @@ home = identifyhome.identify_home()
 if not home.endswith('/'): home += '/'
 
 app_root = os.path.dirname(os.path.realpath(__file__)) + '/../../'
+
+gossip_server_socket_file = home + 'gossip-server.sock'
+
 usage_file = home + 'disk-usage.txt'
 block_data_location = home + 'blocks/'
 contacts_location = home + 'contacts/'

src/gossip/__init__.py

@@ -1,9 +1,12 @@
 from typing import TYPE_CHECKING, Set
-from gossip.peer import Peer
+from os import urandom
+import queue
+
 if TYPE_CHECKING:
-    import queue
     from onionrblocks import Block
 
+    from .peer import Peer
+
 from onionrthreads import add_onionr_thread
 import onionrplugins
 
@@ -17,20 +20,27 @@ as well as each of the plugins.
 
 The transports forward incoming requests to the gossip server
 
-When a new peer announcement is recieved an event is fired and the transport plugin that handles it will (or wont)
+When a new peer announcement is recieved an event is fired and the transport
-create a new peer object by connecting to that peer
+plugin that handles it will (or wont) create a new peer object by connecting
+to that peer
 
-When a new block is generated, it is added to a queue in raw form passed to the starter
+When a new block is generated, it is added to a queue in raw form passed to
+the starter
 
+In stem phase, client uploads recieved (stem) blocks to 2 random peers.
+In stem phase, server disables diffusion
 
 """
 
-def start_gossip_threads(peer_set: Set[Peer], block_queue: queue.Queue[Block]):
+
+def start_gossip_threads(
+        peer_set: Set['Peer'], block_queue: queue.Queue['Block']):
     # Peer set is largely handled by the transport plugins
     # There is a unified set so gossip logic is not repeated
+    seed = urandom(32)
 
-    add_onionr_thread(gossip_server, 1, peer_set, block_queue, initial_sleep=0.2)
+    add_onionr_thread(
-    add_onionr_thread(gossip_client, 1, peer_set, block_queue, initial_sleep=0)
+        gossip_server, 1, peer_set, block_queue, seed, initial_sleep=0.2)
+    add_onionr_thread(
+        gossip_client, 1, peer_set, block_queue, seed, initial_sleep=0)
     onionrplugins.events.event('gossip_start', data=peer_set, threaded=True)
-
-

src/gossip/client.py

@@ -1,3 +1,7 @@
+"""Onionr - Private P2P Communication.
+
+Dandelion ++ Gossip client logic
+"""
 from typing import TYPE_CHECKING
 from typing import Set
 
@@ -5,8 +9,28 @@ from queue import Queue
 
 if TYPE_CHECKING:
     from onionrblocks import Block
-    from peer import Peer
+    from .peer import Peer
+
+import onionrplugins
+"""
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
 
 
-def gossip_client(peer_set: Set[Peer], block_queue: Queue[Block]):
+
-    return
+def gossip_client(
+        peer_set: Set['Peer'],
+        block_queue: Queue['Block'],
+        dandelion_seed: bytes):
+    onionrplugins.events.event('')

src/gossip/server.py

@@ -7,6 +7,14 @@ if TYPE_CHECKING:
     from onionrblocks import Block
     from peer import Peer
 
+from filepaths import gossip_server_socket_file
 
-def gossip_server(peer_set: Set[Peer], block_queue: Queue[Block]):
+import asyncio
+
+
+
+def gossip_server(
+        peer_set: Set['Peer'],
+        block_queue: Queue['Block'],
+        dandelion_seed: bytes):
     return

src/onionrcommands/daemonlaunch/__init__.py

@@ -3,6 +3,7 @@
 launch the api servers and communicator
 """
 import os
+import queue
 import sys
 import platform
 import signal
@@ -118,7 +119,12 @@ def daemon():
     events.event('init', threaded=False)
     events.event('daemon_start')
 
-    gossip.start_gossip_threads(shared_state.get(DeadSimpleKV)['peers'], shared_state.get(DeadSimpleKV)['block_queue'])
+    shared_state.get(apiservers.ClientAPI).gossip_peer_set = set()
+    shared_state.get(apiservers.ClientAPI).gossip_block_queue = queue.Queue()
+
+    gossip.start_gossip_threads(
+        shared_state.get(apiservers.ClientAPI).gossip_peer_set,
+        shared_state.get(apiservers.ClientAPI).gossip_block_queue)
 
     try:
         shared_state.get(apiservers.ClientAPI).start()

src/onionrplugins/onionrevents.py

@@ -5,6 +5,8 @@ Deals with configuration management.
 """
 from threading import Thread
 
+import traceback
+
 import config, logger
 import onionrplugins as plugins
 from . import onionrpluginapi as pluginapi
@@ -42,8 +44,9 @@ def __event_caller(event_name, data = {}):
             logger.warn('Disabling nonexistant plugin "%s"...' % plugin, terminal=True)
             plugins.disable(plugin, stop_event = False)
         except Exception as e:
-            logger.warn('Event "%s" failed for plugin "%s".' % (event_name, plugin), terminal=True)
+
-            logger.debug((event_name + ' - ' + plugin + ' - ' + str(e)), terminal=True)
+            logger.error('Event "%s" failed for plugin "%s".' % (event_name, plugin), terminal=True)
+            logger.error('\n' + traceback.format_exc(), terminal=True)
 
 def event(event_name, data = {}, threaded = True):
     """Call an event on all plugins (if defined)"""
@@ -59,7 +62,6 @@ def call(plugin, event_name, data = None, pluginapi = None):
     """Call an event on a plugin if one is defined"""
 
     if not plugin is None:
-        try:
         attribute = 'on_' + str(event_name).lower()
         if pluginapi is None:
             pluginapi = get_pluginapi(data)
@@ -67,8 +69,6 @@ def call(plugin, event_name, data = None, pluginapi = None):
             return getattr(plugin, attribute)(pluginapi, data)
 
         return True
-        except Exception as e:
+
-            #logger.error(str(e), terminal=True)
-            return False
     else:
         return True

src/onionrutils/basicrequests.py

@@ -1,111 +0,0 @@
-'''Onionr - Private P2P Communication.
-
-    Do HTTP GET or POST requests through a proxy
-'''
-from ipaddress import IPv4Address
-from urllib.parse import urlparse
-
-import requests, streamedrequests
-import logger, onionrexceptions
-import onionrvalues
-from . import localcommand
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-
-
-def do_post_request(url, data={}, port=0, proxyType='tor', max_size=10000, content_type: str = ''):
-    '''Do a POST request through a local tor or i2p instance.'''
-    if proxyType == 'tor':
-        if port == 0:
-            port = localcommand.local_command('/gettorsocks')
-        proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
-    elif proxyType == 'i2p':
-        proxies = {'http': 'http://127.0.0.1:4444'}
-    elif proxyType == 'lan':
-        address = urlparse(url).hostname
-        if IPv4Address(address).is_private and not IPv4Address(address).is_loopback:
-            proxies = {}
-        else:
-            return
-    else:
-        return
-    headers = {'User-Agent': 'PyOnionr', 'Connection':'close'}
-    if len(content_type) > 0:
-        headers['Content-Type'] = content_type
-    try:
-        proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
-        #r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
-        r = streamedrequests.post(url, post_data=data, request_headers=headers, proxy=proxies, connect_timeout=15, stream_timeout=30, max_size=max_size, allow_redirects=False)
-        retData = r[1]
-    except KeyboardInterrupt:
-        raise KeyboardInterrupt
-    except requests.exceptions.RequestException as e:
-        logger.debug('Error: %s' % str(e))
-        retData = False
-    return retData
-
-
-def do_get_request(url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False, max_size=5242880, connect_timeout=15):
-    '''
-    Do a get request through a local tor or i2p instance
-    '''
-    API_VERSION = onionrvalues.API_VERSION
-    retData = False
-    if proxyType == 'tor':
-        if port == 0:
-            port = localcommand.local_command('/gettorsocks')
-        proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
-    elif proxyType == 'i2p':
-        proxies = {'http': 'http://127.0.0.1:4444'}
-    elif proxyType == 'lan':
-        address = urlparse(url).hostname
-        if IPv4Address(address).is_private and not IPv4Address(address).is_loopback:
-            proxies = None
-        else:
-            return
-    else:
-        return
-    headers = {'User-Agent': 'PyOnionr', 'Connection':'close'}
-    response_headers = dict()
-    try:
-        if not proxies is None:
-            proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
-        r = streamedrequests.get(url, request_headers=headers, allow_redirects=False, proxy=proxies, connect_timeout=connect_timeout, stream_timeout=120, max_size=max_size)
-        # Check server is using same API version as us
-        if not ignoreAPI:
-            try:
-                response_headers = r[0].headers
-                if r[0].headers['X-API'] != str(API_VERSION):
-                    raise onionrexceptions.InvalidAPIVersion
-            except KeyError:
-                raise onionrexceptions.InvalidAPIVersion
-        retData = r[1]
-    except KeyboardInterrupt:
-        raise KeyboardInterrupt
-    except ValueError as e:
-        pass
-    except onionrexceptions.InvalidAPIVersion:
-        if 'X-API' in response_headers:
-            logger.debug('Using API version %s. Cannot communicate with node\'s API version of %s.' % (API_VERSION, response_headers['X-API']))
-        else:
-            logger.debug('Using API version %s. API version was not sent with the request.' % API_VERSION)
-    except requests.exceptions.RequestException as e:
-        if not 'ConnectTimeoutError' in str(e) and not 'Request rejected or failed' in str(e):
-            logger.debug('Error: %s' % str(e))
-        retData = False
-    if returnHeaders:
-        return (retData, response_headers)
-    else:
-        return retData

src/setupkvvars/__init__.py

@@ -28,8 +28,6 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 def setup_kv(shared_vars: 'DeadSimpleKV'):
     """Init initial pseudo-globals."""
-    shared_vars.put("peers", set())
-    shared_vars.put("block_queue", queue.Queue())
     shared_vars.put('shutdown', False)
     shared_vars.put('generating_blocks', [])
     shared_vars.put('startTime', epoch.get_epoch())

static-data/default-plugins/tor/.env

@@ -0,0 +1 @@
+PYTHONPATH=./venv/bin/python310:../../src/:./

static-data/default-plugins/tor/main.py

@@ -6,39 +6,85 @@ This default plugin handles "flow" messages
 import sys
 import os
 import locale
+from typing import Set, TYPE_CHECKING
+import base64
 
+from stem.control import Controller
+
+import logger
+from utils import readstatic
+import config
+from filepaths import gossip_server_socket_file
+
+
+from gossip.peer import Peer
+import onionrcrypto
 
 locale.setlocale(locale.LC_ALL, '')
 sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
 # import after path insert
+import starttor
+from torfilepaths import control_socket
 
 """
-    This program is free software: you can redistribute it and/or modify
+This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
+it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
+the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
+(at your option) any later version.
 
-    This program is distributed in the hope that it will be useful,
+This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
+but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
+GNU General Public License for more details.
 
-    You should have received a copy of the GNU General Public License
+You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
 """
 
-#flask_blueprint = flowapi.flask_blueprint
-#security_whitelist = ['circles.circlesstatic', 'circles.circlesindex']
 
 plugin_name = 'tor'
 PLUGIN_VERSION = '0.0.0'
 
 
+bootstrap_file = f'{os.path.dirname(os.path.realpath(__file__))}/bootstrap.txt'
+
 class OnionrTor:
     def __init__(self):
         return
 
 
 def on_init(api, data=None):
-    print("plugin init")
+    logger.info(
-    return
+        f"Tor Transport Plugin v{PLUGIN_VERSION} enabled", terminal=True)
+
+
+def on_gossip_start(api, data: Set[Peer] = None):
+    # We don't do gossip logic
+    try:
+        with open(bootstrap_file, 'r') as bootstrap_file_obj:
+            bootstrap_nodes = bootstrap_file_obj.read().split(',')
+    except FileNotFoundError:
+        bootstrap_nodes = []
+    #for node in bootstrap_nodes:
+    starttor.start_tor()
+
+    with Controller.from_socket_file(control_socket) as controller:
+        controller.authenticate()
+        logger.info(f"Tor socks is listening on {controller.get_listeners('SOCKS')}", terminal=True)
+        key = config.get('tor.key')
+        new_address = ''
+        if not key:
+            add_onion_resp = controller.create_ephemeral_hidden_service(
+                {'80': f'unix:{gossip_server_socket_file}'},
+                key_content='BEST', key_type='NEW')
+            config.set('tor.key', add_onion_resp.private_key, savefile=True)
+            new_address = 'Generated '
+        else:
+            add_onion_resp = controller.create_ephemeral_hidden_service(
+                {'80': f'unix:{gossip_server_socket_file}'},
+                key_content=key, key_type='ED25519-V3')
+        logger.info(
+            f'{new_address}Tor transport address {add_onion_resp.service_id}' +
+            '.onion',
+            terminal=True)
+

static-data/default-plugins/tor/starttor.py

@@ -0,0 +1,18 @@
+import stem.process
+
+from utils.identifyhome import identify_home
+
+from torfilepaths import control_socket
+from torfilepaths import tor_data_dir
+
+def start_tor():
+
+    tor_process = stem.process.launch_tor_with_config(
+        config={
+            'SocksPort': 'auto OnionTrafficOnly',
+            'DataDirectory': tor_data_dir,
+            'ControlSocket': control_socket,
+        },
+        completion_percent=50,
+        take_ownership=True
+    )

static-data/default-plugins/tor/torfilepaths.py

@@ -0,0 +1,3 @@
+from utils.identifyhome import identify_home
+control_socket = f'{identify_home()}/torcontrol.sock'
+tor_data_dir = f'{identify_home()}/tordata'

static-data/default-plugins/tor/torrc

@@ -0,0 +1,2 @@
+OnionTrafficOnly 1
+