Work on gossip system and tor transport

This commit is contained in:
Kevin F 2022-02-14 17:47:54 -06:00
parent 713aeb199d
commit e5b396fc11
16 changed files with 174 additions and 156 deletions

View File

@ -3,6 +3,8 @@
This file handles all incoming http requests to the client, using Flask
"""
from typing import Dict
from typing import Set
from typing import TYPE_CHECKING
import hmac
import flask
@ -51,6 +53,9 @@ class PrivateAPI:
self.startTime = epoch.get_epoch()
app = flask.Flask(__name__)
self.gossip_block_queue: 'queue.Queue' = None
self.gossip_peer_set: Set['Peer'] = None
bind_port = int(config.get('client.client.port', 59496))
self.bindPort = bind_port

View File

@ -29,6 +29,8 @@ def detect_socket_leaks(socket_event):
raises exception & logs if not to loopback
"""
ip_address = socket_event[1][0]
if ip_address.startswith('/'):
return
# validate is valid ip address (no hostname, etc)
# raises NetworkLeak if not

View File

@ -59,7 +59,10 @@ def block_exec(event, info):
'apport/report.py'
]
whitelisted_source = [
'ZABaAGQBZAJsAW0CWgIBAHoGZAFkA2wDWgRXAG4LBABlBXkZAQABAAEAZARaBlkAbgN3AGQFWgZkAWQGbAdtCFoIAQBkAWQDbAlaCWQBZAdsCm0LWgwBAAkAZQmgDWQIZAWhAnM1ZARaBmUIgwBkCRcAWg5kEmQMZQ9kDWUPZgRkDmQPhAVaEGQTZBBkEYQBWhFkA1MA'
'ZABaAGQBZAJsAW0CWgIBAHoGZAFkA2wDWgRXAG4LBABlBXkZAQABAAEAZARaBlkAbgN3AGQFWgZkAWQGbAdtCFoIAQBkAWQDbAlaCWQBZAdsCm0LWgwBAAkAZQmgDWQIZAWhAnM1ZARaBmUIgwBkCRcAWg5kEmQMZQ9kDWUPZgRkDmQPhAVaEGQTZBBkEYQBWhFkA1MA',
'RwBkAGQBhABkAYMCWgBkAlMA',
'ZABaAGQBZAJsAVoBZAFkAmwCWgJkAWQCbANaA2QBZAJsBFoEZAFkAmwFWgVkAWQCbAZaBmQBZAJsB1oIZAFkAmwJWghkAWQCbApaCGQBZAJsC1oIZANaDGQEWg1kBWQCZAJkBmQCZQ1kB2QIZAJmCWQJZAqEAVoOZAVkBmQCZQ1kB2QIZgZkC2QMhAFaD2QCUwA=',
'ZABaAGQBZAJsAVoBZAFkAmwCWgJkAWQCbANaA2QBZAJsBFoEZAFkAmwFWgVkAWQCbAZaBmQBZAJsB1oIZAFkAmwJWghkAWQCbApaCGQBZANsC20MWgxtDVoNbQ5aDm0PWg9tEFoQbRFaEQEAZARaEmQFWhNkBmQCZAJkB2QCZRNkCGQJZAJmCWQKZRRkC2UPZRBlFBkAGQBkDGUPZRQZAGQNZRVkDmUPZQ1lFGcBZAJmAhkAGQBkD2UVZBBlFmQRZRZkEmUPZRQZAGQTZQRqF2YUZBRkFYQFWhhkBmQHZAJlE2QIZAlmBmQWZQ5lFGURZRRlEGUUGQBmAhkAZgIZAGQKZRRkDWUVZA5lD2UNZRRnAWQCZgIZABkAZA9lFWQQZRZkEWUWZBNlBGoXZhBkF2QYhAVaGWQCUwA='
]
home = identifyhome.identify_home()

View File

@ -4,6 +4,9 @@ home = identifyhome.identify_home()
if not home.endswith('/'): home += '/'
app_root = os.path.dirname(os.path.realpath(__file__)) + '/../../'
gossip_server_socket_file = home + 'gossip-server.sock'
usage_file = home + 'disk-usage.txt'
block_data_location = home + 'blocks/'
contacts_location = home + 'contacts/'

View File

@ -1,9 +1,12 @@
from typing import TYPE_CHECKING, Set
from gossip.peer import Peer
if TYPE_CHECKING:
from os import urandom
import queue
if TYPE_CHECKING:
from onionrblocks import Block
from .peer import Peer
from onionrthreads import add_onionr_thread
import onionrplugins
@ -17,20 +20,27 @@ as well as each of the plugins.
The transports forward incoming requests to the gossip server
When a new peer announcement is recieved an event is fired and the transport plugin that handles it will (or wont)
create a new peer object by connecting to that peer
When a new peer announcement is recieved an event is fired and the transport
plugin that handles it will (or wont) create a new peer object by connecting
to that peer
When a new block is generated, it is added to a queue in raw form passed to the starter
When a new block is generated, it is added to a queue in raw form passed to
the starter
In stem phase, client uploads recieved (stem) blocks to 2 random peers.
In stem phase, server disables diffusion
"""
def start_gossip_threads(peer_set: Set[Peer], block_queue: queue.Queue[Block]):
def start_gossip_threads(
peer_set: Set['Peer'], block_queue: queue.Queue['Block']):
# Peer set is largely handled by the transport plugins
# There is a unified set so gossip logic is not repeated
seed = urandom(32)
add_onionr_thread(gossip_server, 1, peer_set, block_queue, initial_sleep=0.2)
add_onionr_thread(gossip_client, 1, peer_set, block_queue, initial_sleep=0)
add_onionr_thread(
gossip_server, 1, peer_set, block_queue, seed, initial_sleep=0.2)
add_onionr_thread(
gossip_client, 1, peer_set, block_queue, seed, initial_sleep=0)
onionrplugins.events.event('gossip_start', data=peer_set, threaded=True)

View File

@ -1,3 +1,7 @@
"""Onionr - Private P2P Communication.
Dandelion ++ Gossip client logic
"""
from typing import TYPE_CHECKING
from typing import Set
@ -5,8 +9,28 @@ from queue import Queue
if TYPE_CHECKING:
from onionrblocks import Block
from peer import Peer
from .peer import Peer
import onionrplugins
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def gossip_client(peer_set: Set[Peer], block_queue: Queue[Block]):
return
def gossip_client(
peer_set: Set['Peer'],
block_queue: Queue['Block'],
dandelion_seed: bytes):
onionrplugins.events.event('')

View File

@ -7,6 +7,14 @@ if TYPE_CHECKING:
from onionrblocks import Block
from peer import Peer
from filepaths import gossip_server_socket_file
def gossip_server(peer_set: Set[Peer], block_queue: Queue[Block]):
import asyncio
def gossip_server(
peer_set: Set['Peer'],
block_queue: Queue['Block'],
dandelion_seed: bytes):
return

View File

@ -3,6 +3,7 @@
launch the api servers and communicator
"""
import os
import queue
import sys
import platform
import signal
@ -118,7 +119,12 @@ def daemon():
events.event('init', threaded=False)
events.event('daemon_start')
gossip.start_gossip_threads(shared_state.get(DeadSimpleKV)['peers'], shared_state.get(DeadSimpleKV)['block_queue'])
shared_state.get(apiservers.ClientAPI).gossip_peer_set = set()
shared_state.get(apiservers.ClientAPI).gossip_block_queue = queue.Queue()
gossip.start_gossip_threads(
shared_state.get(apiservers.ClientAPI).gossip_peer_set,
shared_state.get(apiservers.ClientAPI).gossip_block_queue)
try:
shared_state.get(apiservers.ClientAPI).start()

View File

@ -5,6 +5,8 @@ Deals with configuration management.
"""
from threading import Thread
import traceback
import config, logger
import onionrplugins as plugins
from . import onionrpluginapi as pluginapi
@ -42,8 +44,9 @@ def __event_caller(event_name, data = {}):
logger.warn('Disabling nonexistant plugin "%s"...' % plugin, terminal=True)
plugins.disable(plugin, stop_event = False)
except Exception as e:
logger.warn('Event "%s" failed for plugin "%s".' % (event_name, plugin), terminal=True)
logger.debug((event_name + ' - ' + plugin + ' - ' + str(e)), terminal=True)
logger.error('Event "%s" failed for plugin "%s".' % (event_name, plugin), terminal=True)
logger.error('\n' + traceback.format_exc(), terminal=True)
def event(event_name, data = {}, threaded = True):
"""Call an event on all plugins (if defined)"""
@ -59,7 +62,6 @@ def call(plugin, event_name, data = None, pluginapi = None):
"""Call an event on a plugin if one is defined"""
if not plugin is None:
try:
attribute = 'on_' + str(event_name).lower()
if pluginapi is None:
pluginapi = get_pluginapi(data)
@ -67,8 +69,6 @@ def call(plugin, event_name, data = None, pluginapi = None):
return getattr(plugin, attribute)(pluginapi, data)
return True
except Exception as e:
#logger.error(str(e), terminal=True)
return False
else:
return True

View File

@ -1,111 +0,0 @@
'''Onionr - Private P2P Communication.
Do HTTP GET or POST requests through a proxy
'''
from ipaddress import IPv4Address
from urllib.parse import urlparse
import requests, streamedrequests
import logger, onionrexceptions
import onionrvalues
from . import localcommand
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
def do_post_request(url, data={}, port=0, proxyType='tor', max_size=10000, content_type: str = ''):
'''Do a POST request through a local tor or i2p instance.'''
if proxyType == 'tor':
if port == 0:
port = localcommand.local_command('/gettorsocks')
proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
elif proxyType == 'i2p':
proxies = {'http': 'http://127.0.0.1:4444'}
elif proxyType == 'lan':
address = urlparse(url).hostname
if IPv4Address(address).is_private and not IPv4Address(address).is_loopback:
proxies = {}
else:
return
else:
return
headers = {'User-Agent': 'PyOnionr', 'Connection':'close'}
if len(content_type) > 0:
headers['Content-Type'] = content_type
try:
proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
#r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
r = streamedrequests.post(url, post_data=data, request_headers=headers, proxy=proxies, connect_timeout=15, stream_timeout=30, max_size=max_size, allow_redirects=False)
retData = r[1]
except KeyboardInterrupt:
raise KeyboardInterrupt
except requests.exceptions.RequestException as e:
logger.debug('Error: %s' % str(e))
retData = False
return retData
def do_get_request(url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False, max_size=5242880, connect_timeout=15):
'''
Do a get request through a local tor or i2p instance
'''
API_VERSION = onionrvalues.API_VERSION
retData = False
if proxyType == 'tor':
if port == 0:
port = localcommand.local_command('/gettorsocks')
proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
elif proxyType == 'i2p':
proxies = {'http': 'http://127.0.0.1:4444'}
elif proxyType == 'lan':
address = urlparse(url).hostname
if IPv4Address(address).is_private and not IPv4Address(address).is_loopback:
proxies = None
else:
return
else:
return
headers = {'User-Agent': 'PyOnionr', 'Connection':'close'}
response_headers = dict()
try:
if not proxies is None:
proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
r = streamedrequests.get(url, request_headers=headers, allow_redirects=False, proxy=proxies, connect_timeout=connect_timeout, stream_timeout=120, max_size=max_size)
# Check server is using same API version as us
if not ignoreAPI:
try:
response_headers = r[0].headers
if r[0].headers['X-API'] != str(API_VERSION):
raise onionrexceptions.InvalidAPIVersion
except KeyError:
raise onionrexceptions.InvalidAPIVersion
retData = r[1]
except KeyboardInterrupt:
raise KeyboardInterrupt
except ValueError as e:
pass
except onionrexceptions.InvalidAPIVersion:
if 'X-API' in response_headers:
logger.debug('Using API version %s. Cannot communicate with node\'s API version of %s.' % (API_VERSION, response_headers['X-API']))
else:
logger.debug('Using API version %s. API version was not sent with the request.' % API_VERSION)
except requests.exceptions.RequestException as e:
if not 'ConnectTimeoutError' in str(e) and not 'Request rejected or failed' in str(e):
logger.debug('Error: %s' % str(e))
retData = False
if returnHeaders:
return (retData, response_headers)
else:
return retData

View File

@ -28,8 +28,6 @@ along with this program. If not, see <https://www.gnu.org/licenses/>.
def setup_kv(shared_vars: 'DeadSimpleKV'):
"""Init initial pseudo-globals."""
shared_vars.put("peers", set())
shared_vars.put("block_queue", queue.Queue())
shared_vars.put('shutdown', False)
shared_vars.put('generating_blocks', [])
shared_vars.put('startTime', epoch.get_epoch())

View File

@ -0,0 +1 @@
PYTHONPATH=./venv/bin/python310:../../src/:./

View File

@ -6,11 +6,25 @@ This default plugin handles "flow" messages
import sys
import os
import locale
from typing import Set, TYPE_CHECKING
import base64
from stem.control import Controller
import logger
from utils import readstatic
import config
from filepaths import gossip_server_socket_file
from gossip.peer import Peer
import onionrcrypto
locale.setlocale(locale.LC_ALL, '')
sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
# import after path insert
import starttor
from torfilepaths import control_socket
"""
This program is free software: you can redistribute it and/or modify
@ -27,18 +41,50 @@ sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
#flask_blueprint = flowapi.flask_blueprint
#security_whitelist = ['circles.circlesstatic', 'circles.circlesindex']
plugin_name = 'tor'
PLUGIN_VERSION = '0.0.0'
bootstrap_file = f'{os.path.dirname(os.path.realpath(__file__))}/bootstrap.txt'
class OnionrTor:
def __init__(self):
return
def on_init(api, data=None):
print("plugin init")
return
logger.info(
f"Tor Transport Plugin v{PLUGIN_VERSION} enabled", terminal=True)
def on_gossip_start(api, data: Set[Peer] = None):
# We don't do gossip logic
try:
with open(bootstrap_file, 'r') as bootstrap_file_obj:
bootstrap_nodes = bootstrap_file_obj.read().split(',')
except FileNotFoundError:
bootstrap_nodes = []
#for node in bootstrap_nodes:
starttor.start_tor()
with Controller.from_socket_file(control_socket) as controller:
controller.authenticate()
logger.info(f"Tor socks is listening on {controller.get_listeners('SOCKS')}", terminal=True)
key = config.get('tor.key')
new_address = ''
if not key:
add_onion_resp = controller.create_ephemeral_hidden_service(
{'80': f'unix:{gossip_server_socket_file}'},
key_content='BEST', key_type='NEW')
config.set('tor.key', add_onion_resp.private_key, savefile=True)
new_address = 'Generated '
else:
add_onion_resp = controller.create_ephemeral_hidden_service(
{'80': f'unix:{gossip_server_socket_file}'},
key_content=key, key_type='ED25519-V3')
logger.info(
f'{new_address}Tor transport address {add_onion_resp.service_id}' +
'.onion',
terminal=True)

View File

@ -0,0 +1,18 @@
import stem.process
from utils.identifyhome import identify_home
from torfilepaths import control_socket
from torfilepaths import tor_data_dir
def start_tor():
tor_process = stem.process.launch_tor_with_config(
config={
'SocksPort': 'auto OnionTrafficOnly',
'DataDirectory': tor_data_dir,
'ControlSocket': control_socket,
},
completion_percent=50,
take_ownership=True
)

View File

@ -0,0 +1,3 @@
from utils.identifyhome import identify_home
control_socket = f'{identify_home()}/torcontrol.sock'
tor_data_dir = f'{identify_home()}/tordata'

View File

@ -0,0 +1,2 @@
OnionTrafficOnly 1