Work on gossip system and tor transport

This commit is contained in:
Kevin F 2022-02-14 17:47:54 -06:00
parent 713aeb199d
commit e5b396fc11
16 changed files with 174 additions and 156 deletions

View File

@ -3,6 +3,8 @@
This file handles all incoming http requests to the client, using Flask This file handles all incoming http requests to the client, using Flask
""" """
from typing import Dict from typing import Dict
from typing import Set
from typing import TYPE_CHECKING
import hmac import hmac
import flask import flask
@ -51,6 +53,9 @@ class PrivateAPI:
self.startTime = epoch.get_epoch() self.startTime = epoch.get_epoch()
app = flask.Flask(__name__) app = flask.Flask(__name__)
self.gossip_block_queue: 'queue.Queue' = None
self.gossip_peer_set: Set['Peer'] = None
bind_port = int(config.get('client.client.port', 59496)) bind_port = int(config.get('client.client.port', 59496))
self.bindPort = bind_port self.bindPort = bind_port

View File

@ -29,6 +29,8 @@ def detect_socket_leaks(socket_event):
raises exception & logs if not to loopback raises exception & logs if not to loopback
""" """
ip_address = socket_event[1][0] ip_address = socket_event[1][0]
if ip_address.startswith('/'):
return
# validate is valid ip address (no hostname, etc) # validate is valid ip address (no hostname, etc)
# raises NetworkLeak if not # raises NetworkLeak if not

View File

@ -59,7 +59,10 @@ def block_exec(event, info):
'apport/report.py' 'apport/report.py'
] ]
whitelisted_source = [ whitelisted_source = [
'ZABaAGQBZAJsAW0CWgIBAHoGZAFkA2wDWgRXAG4LBABlBXkZAQABAAEAZARaBlkAbgN3AGQFWgZkAWQGbAdtCFoIAQBkAWQDbAlaCWQBZAdsCm0LWgwBAAkAZQmgDWQIZAWhAnM1ZARaBmUIgwBkCRcAWg5kEmQMZQ9kDWUPZgRkDmQPhAVaEGQTZBBkEYQBWhFkA1MA' 'ZABaAGQBZAJsAW0CWgIBAHoGZAFkA2wDWgRXAG4LBABlBXkZAQABAAEAZARaBlkAbgN3AGQFWgZkAWQGbAdtCFoIAQBkAWQDbAlaCWQBZAdsCm0LWgwBAAkAZQmgDWQIZAWhAnM1ZARaBmUIgwBkCRcAWg5kEmQMZQ9kDWUPZgRkDmQPhAVaEGQTZBBkEYQBWhFkA1MA',
'RwBkAGQBhABkAYMCWgBkAlMA',
'ZABaAGQBZAJsAVoBZAFkAmwCWgJkAWQCbANaA2QBZAJsBFoEZAFkAmwFWgVkAWQCbAZaBmQBZAJsB1oIZAFkAmwJWghkAWQCbApaCGQBZAJsC1oIZANaDGQEWg1kBWQCZAJkBmQCZQ1kB2QIZAJmCWQJZAqEAVoOZAVkBmQCZQ1kB2QIZgZkC2QMhAFaD2QCUwA=',
'ZABaAGQBZAJsAVoBZAFkAmwCWgJkAWQCbANaA2QBZAJsBFoEZAFkAmwFWgVkAWQCbAZaBmQBZAJsB1oIZAFkAmwJWghkAWQCbApaCGQBZANsC20MWgxtDVoNbQ5aDm0PWg9tEFoQbRFaEQEAZARaEmQFWhNkBmQCZAJkB2QCZRNkCGQJZAJmCWQKZRRkC2UPZRBlFBkAGQBkDGUPZRQZAGQNZRVkDmUPZQ1lFGcBZAJmAhkAGQBkD2UVZBBlFmQRZRZkEmUPZRQZAGQTZQRqF2YUZBRkFYQFWhhkBmQHZAJlE2QIZAlmBmQWZQ5lFGURZRRlEGUUGQBmAhkAZgIZAGQKZRRkDWUVZA5lD2UNZRRnAWQCZgIZABkAZA9lFWQQZRZkEWUWZBNlBGoXZhBkF2QYhAVaGWQCUwA='
] ]
home = identifyhome.identify_home() home = identifyhome.identify_home()

View File

@ -4,6 +4,9 @@ home = identifyhome.identify_home()
if not home.endswith('/'): home += '/' if not home.endswith('/'): home += '/'
app_root = os.path.dirname(os.path.realpath(__file__)) + '/../../' app_root = os.path.dirname(os.path.realpath(__file__)) + '/../../'
gossip_server_socket_file = home + 'gossip-server.sock'
usage_file = home + 'disk-usage.txt' usage_file = home + 'disk-usage.txt'
block_data_location = home + 'blocks/' block_data_location = home + 'blocks/'
contacts_location = home + 'contacts/' contacts_location = home + 'contacts/'

View File

@ -1,9 +1,12 @@
from typing import TYPE_CHECKING, Set from typing import TYPE_CHECKING, Set
from gossip.peer import Peer from os import urandom
import queue
if TYPE_CHECKING: if TYPE_CHECKING:
import queue
from onionrblocks import Block from onionrblocks import Block
from .peer import Peer
from onionrthreads import add_onionr_thread from onionrthreads import add_onionr_thread
import onionrplugins import onionrplugins
@ -17,20 +20,27 @@ as well as each of the plugins.
The transports forward incoming requests to the gossip server The transports forward incoming requests to the gossip server
When a new peer announcement is recieved an event is fired and the transport plugin that handles it will (or wont) When a new peer announcement is recieved an event is fired and the transport
create a new peer object by connecting to that peer plugin that handles it will (or wont) create a new peer object by connecting
to that peer
When a new block is generated, it is added to a queue in raw form passed to the starter When a new block is generated, it is added to a queue in raw form passed to
the starter
In stem phase, client uploads recieved (stem) blocks to 2 random peers.
In stem phase, server disables diffusion
""" """
def start_gossip_threads(peer_set: Set[Peer], block_queue: queue.Queue[Block]):
def start_gossip_threads(
peer_set: Set['Peer'], block_queue: queue.Queue['Block']):
# Peer set is largely handled by the transport plugins # Peer set is largely handled by the transport plugins
# There is a unified set so gossip logic is not repeated # There is a unified set so gossip logic is not repeated
seed = urandom(32)
add_onionr_thread(gossip_server, 1, peer_set, block_queue, initial_sleep=0.2) add_onionr_thread(
add_onionr_thread(gossip_client, 1, peer_set, block_queue, initial_sleep=0) gossip_server, 1, peer_set, block_queue, seed, initial_sleep=0.2)
add_onionr_thread(
gossip_client, 1, peer_set, block_queue, seed, initial_sleep=0)
onionrplugins.events.event('gossip_start', data=peer_set, threaded=True) onionrplugins.events.event('gossip_start', data=peer_set, threaded=True)

View File

@ -1,3 +1,7 @@
"""Onionr - Private P2P Communication.
Dandelion ++ Gossip client logic
"""
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from typing import Set from typing import Set
@ -5,8 +9,28 @@ from queue import Queue
if TYPE_CHECKING: if TYPE_CHECKING:
from onionrblocks import Block from onionrblocks import Block
from peer import Peer from .peer import Peer
import onionrplugins
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def gossip_client(peer_set: Set[Peer], block_queue: Queue[Block]):
return def gossip_client(
peer_set: Set['Peer'],
block_queue: Queue['Block'],
dandelion_seed: bytes):
onionrplugins.events.event('')

View File

@ -7,6 +7,14 @@ if TYPE_CHECKING:
from onionrblocks import Block from onionrblocks import Block
from peer import Peer from peer import Peer
from filepaths import gossip_server_socket_file
def gossip_server(peer_set: Set[Peer], block_queue: Queue[Block]): import asyncio
def gossip_server(
peer_set: Set['Peer'],
block_queue: Queue['Block'],
dandelion_seed: bytes):
return return

View File

@ -3,6 +3,7 @@
launch the api servers and communicator launch the api servers and communicator
""" """
import os import os
import queue
import sys import sys
import platform import platform
import signal import signal
@ -118,7 +119,12 @@ def daemon():
events.event('init', threaded=False) events.event('init', threaded=False)
events.event('daemon_start') events.event('daemon_start')
gossip.start_gossip_threads(shared_state.get(DeadSimpleKV)['peers'], shared_state.get(DeadSimpleKV)['block_queue']) shared_state.get(apiservers.ClientAPI).gossip_peer_set = set()
shared_state.get(apiservers.ClientAPI).gossip_block_queue = queue.Queue()
gossip.start_gossip_threads(
shared_state.get(apiservers.ClientAPI).gossip_peer_set,
shared_state.get(apiservers.ClientAPI).gossip_block_queue)
try: try:
shared_state.get(apiservers.ClientAPI).start() shared_state.get(apiservers.ClientAPI).start()

View File

@ -5,6 +5,8 @@ Deals with configuration management.
""" """
from threading import Thread from threading import Thread
import traceback
import config, logger import config, logger
import onionrplugins as plugins import onionrplugins as plugins
from . import onionrpluginapi as pluginapi from . import onionrpluginapi as pluginapi
@ -42,8 +44,9 @@ def __event_caller(event_name, data = {}):
logger.warn('Disabling nonexistant plugin "%s"...' % plugin, terminal=True) logger.warn('Disabling nonexistant plugin "%s"...' % plugin, terminal=True)
plugins.disable(plugin, stop_event = False) plugins.disable(plugin, stop_event = False)
except Exception as e: except Exception as e:
logger.warn('Event "%s" failed for plugin "%s".' % (event_name, plugin), terminal=True)
logger.debug((event_name + ' - ' + plugin + ' - ' + str(e)), terminal=True) logger.error('Event "%s" failed for plugin "%s".' % (event_name, plugin), terminal=True)
logger.error('\n' + traceback.format_exc(), terminal=True)
def event(event_name, data = {}, threaded = True): def event(event_name, data = {}, threaded = True):
"""Call an event on all plugins (if defined)""" """Call an event on all plugins (if defined)"""
@ -59,16 +62,13 @@ def call(plugin, event_name, data = None, pluginapi = None):
"""Call an event on a plugin if one is defined""" """Call an event on a plugin if one is defined"""
if not plugin is None: if not plugin is None:
try: attribute = 'on_' + str(event_name).lower()
attribute = 'on_' + str(event_name).lower() if pluginapi is None:
if pluginapi is None: pluginapi = get_pluginapi(data)
pluginapi = get_pluginapi(data) if hasattr(plugin, attribute):
if hasattr(plugin, attribute): return getattr(plugin, attribute)(pluginapi, data)
return getattr(plugin, attribute)(pluginapi, data)
return True
return True
except Exception as e:
#logger.error(str(e), terminal=True)
return False
else: else:
return True return True

View File

@ -1,111 +0,0 @@
'''Onionr - Private P2P Communication.
Do HTTP GET or POST requests through a proxy
'''
from ipaddress import IPv4Address
from urllib.parse import urlparse
import requests, streamedrequests
import logger, onionrexceptions
import onionrvalues
from . import localcommand
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
def do_post_request(url, data={}, port=0, proxyType='tor', max_size=10000, content_type: str = ''):
'''Do a POST request through a local tor or i2p instance.'''
if proxyType == 'tor':
if port == 0:
port = localcommand.local_command('/gettorsocks')
proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
elif proxyType == 'i2p':
proxies = {'http': 'http://127.0.0.1:4444'}
elif proxyType == 'lan':
address = urlparse(url).hostname
if IPv4Address(address).is_private and not IPv4Address(address).is_loopback:
proxies = {}
else:
return
else:
return
headers = {'User-Agent': 'PyOnionr', 'Connection':'close'}
if len(content_type) > 0:
headers['Content-Type'] = content_type
try:
proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
#r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
r = streamedrequests.post(url, post_data=data, request_headers=headers, proxy=proxies, connect_timeout=15, stream_timeout=30, max_size=max_size, allow_redirects=False)
retData = r[1]
except KeyboardInterrupt:
raise KeyboardInterrupt
except requests.exceptions.RequestException as e:
logger.debug('Error: %s' % str(e))
retData = False
return retData
def do_get_request(url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False, max_size=5242880, connect_timeout=15):
'''
Do a get request through a local tor or i2p instance
'''
API_VERSION = onionrvalues.API_VERSION
retData = False
if proxyType == 'tor':
if port == 0:
port = localcommand.local_command('/gettorsocks')
proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
elif proxyType == 'i2p':
proxies = {'http': 'http://127.0.0.1:4444'}
elif proxyType == 'lan':
address = urlparse(url).hostname
if IPv4Address(address).is_private and not IPv4Address(address).is_loopback:
proxies = None
else:
return
else:
return
headers = {'User-Agent': 'PyOnionr', 'Connection':'close'}
response_headers = dict()
try:
if not proxies is None:
proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
r = streamedrequests.get(url, request_headers=headers, allow_redirects=False, proxy=proxies, connect_timeout=connect_timeout, stream_timeout=120, max_size=max_size)
# Check server is using same API version as us
if not ignoreAPI:
try:
response_headers = r[0].headers
if r[0].headers['X-API'] != str(API_VERSION):
raise onionrexceptions.InvalidAPIVersion
except KeyError:
raise onionrexceptions.InvalidAPIVersion
retData = r[1]
except KeyboardInterrupt:
raise KeyboardInterrupt
except ValueError as e:
pass
except onionrexceptions.InvalidAPIVersion:
if 'X-API' in response_headers:
logger.debug('Using API version %s. Cannot communicate with node\'s API version of %s.' % (API_VERSION, response_headers['X-API']))
else:
logger.debug('Using API version %s. API version was not sent with the request.' % API_VERSION)
except requests.exceptions.RequestException as e:
if not 'ConnectTimeoutError' in str(e) and not 'Request rejected or failed' in str(e):
logger.debug('Error: %s' % str(e))
retData = False
if returnHeaders:
return (retData, response_headers)
else:
return retData

View File

@ -28,8 +28,6 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
def setup_kv(shared_vars: 'DeadSimpleKV'): def setup_kv(shared_vars: 'DeadSimpleKV'):
"""Init initial pseudo-globals.""" """Init initial pseudo-globals."""
shared_vars.put("peers", set())
shared_vars.put("block_queue", queue.Queue())
shared_vars.put('shutdown', False) shared_vars.put('shutdown', False)
shared_vars.put('generating_blocks', []) shared_vars.put('generating_blocks', [])
shared_vars.put('startTime', epoch.get_epoch()) shared_vars.put('startTime', epoch.get_epoch())

View File

@ -0,0 +1 @@
PYTHONPATH=./venv/bin/python310:../../src/:./

View File

@ -6,39 +6,85 @@ This default plugin handles "flow" messages
import sys import sys
import os import os
import locale import locale
from typing import Set, TYPE_CHECKING
import base64
from stem.control import Controller
import logger
from utils import readstatic
import config
from filepaths import gossip_server_socket_file
from gossip.peer import Peer
import onionrcrypto
locale.setlocale(locale.LC_ALL, '') locale.setlocale(locale.LC_ALL, '')
sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
# import after path insert # import after path insert
import starttor
from torfilepaths import control_socket
""" """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or the Free Software Foundation, either version 3 of the License, or
(at your option) any later version. (at your option) any later version.
This program is distributed in the hope that it will be useful, This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details. GNU General Public License for more details.
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
""" """
#flask_blueprint = flowapi.flask_blueprint
#security_whitelist = ['circles.circlesstatic', 'circles.circlesindex']
plugin_name = 'tor' plugin_name = 'tor'
PLUGIN_VERSION = '0.0.0' PLUGIN_VERSION = '0.0.0'
bootstrap_file = f'{os.path.dirname(os.path.realpath(__file__))}/bootstrap.txt'
class OnionrTor: class OnionrTor:
def __init__(self): def __init__(self):
return return
def on_init(api, data=None): def on_init(api, data=None):
print("plugin init") logger.info(
return f"Tor Transport Plugin v{PLUGIN_VERSION} enabled", terminal=True)
def on_gossip_start(api, data: Set[Peer] = None):
# We don't do gossip logic
try:
with open(bootstrap_file, 'r') as bootstrap_file_obj:
bootstrap_nodes = bootstrap_file_obj.read().split(',')
except FileNotFoundError:
bootstrap_nodes = []
#for node in bootstrap_nodes:
starttor.start_tor()
with Controller.from_socket_file(control_socket) as controller:
controller.authenticate()
logger.info(f"Tor socks is listening on {controller.get_listeners('SOCKS')}", terminal=True)
key = config.get('tor.key')
new_address = ''
if not key:
add_onion_resp = controller.create_ephemeral_hidden_service(
{'80': f'unix:{gossip_server_socket_file}'},
key_content='BEST', key_type='NEW')
config.set('tor.key', add_onion_resp.private_key, savefile=True)
new_address = 'Generated '
else:
add_onion_resp = controller.create_ephemeral_hidden_service(
{'80': f'unix:{gossip_server_socket_file}'},
key_content=key, key_type='ED25519-V3')
logger.info(
f'{new_address}Tor transport address {add_onion_resp.service_id}' +
'.onion',
terminal=True)

View File

@ -0,0 +1,18 @@
import stem.process
from utils.identifyhome import identify_home
from torfilepaths import control_socket
from torfilepaths import tor_data_dir
def start_tor():
tor_process = stem.process.launch_tor_with_config(
config={
'SocksPort': 'auto OnionTrafficOnly',
'DataDirectory': tor_data_dir,
'ControlSocket': control_socket,
},
completion_percent=50,
take_ownership=True
)

View File

@ -0,0 +1,3 @@
from utils.identifyhome import identify_home
control_socket = f'{identify_home()}/torcontrol.sock'
tor_data_dir = f'{identify_home()}/tordata'

View File

@ -0,0 +1,2 @@
OnionTrafficOnly 1