kev/Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

Removed lan and tor transports

Browse Source
This commit is contained in:
Kevin F 2022-01-11 01:20:15 -06:00
parent 2352e066cc
commit 4bd9bd6e9d
88 changed files with 72 additions and 4372 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/apiservers/__init__.py
View File

@ -4,7 +4,6 @@ Public is net-facing server meant for other nodes
Private is meant for controlling and accessing this node Private is meant for controlling and accessing this node
""" """
from . import public, private from . import private
PublicAPI = public.PublicAPI
ClientAPI = private.PrivateAPI ClientAPI = private.PrivateAPI

27
src/apiservers/private/__init__.py
View File

@ -16,20 +16,20 @@ import logger
from etc import waitforsetvar from etc import waitforsetvar
from . import register_private_blueprints from . import register_private_blueprints
import config import config
from .. import public
""" """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or the Free Software Foundation, either version 3 of the License, or
(at your option) any later version. (at your option) any later version.
This program is distributed in the hope that it will be useful, This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details. GNU General Public License for more details.
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
""" """
@ -50,7 +50,7 @@ class PrivateAPI:
self.startTime = epoch.get_epoch() self.startTime = epoch.get_epoch()
app = flask.Flask(__name__) app = flask.Flask(__name__)
bind_port = int(config.get('client.client.port', 59496)) bind_port = int(config.get('client.client.port', 59496))
self.bindPort = bind_port self.bindPort = bind_port
@ -77,8 +77,7 @@ class PrivateAPI:
"""Start client gevent API web server with flask client app.""" """Start client gevent API web server with flask client app."""
waitforsetvar.wait_for_set_var(self, "_too_many") waitforsetvar.wait_for_set_var(self, "_too_many")
fd_handler = httpapi.fdsafehandler.FDSafeHandler fd_handler = httpapi.fdsafehandler.FDSafeHandler
self.publicAPI = self._too_many.get( # pylint: disable=E1101
public.PublicAPI)
self.httpServer = WSGIServer((self.host, self.bindPort), self.httpServer = WSGIServer((self.host, self.bindPort),
self.app, log=None, self.app, log=None,
handler_class=fd_handler) handler_class=fd_handler)

76
src/apiservers/public/__init__.py
View File

@ -1,76 +0,0 @@
"""Onionr - Private P2P Communication.
This file handles all incoming http requests
to the public api server, using Flask
"""
import time
import threading
import flask
from gevent.pywsgi import WSGIServer
from httpapi import apiutils, security, fdsafehandler, miscpublicapi
import logger
import config
import filepaths
from utils import gettransports
from etc import onionrvalues, waitforsetvar
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def _get_tor_adder(pub_api):
transports = []
while len(transports) == 0:
transports = gettransports.get()
time.sleep(0.3)
pub_api.torAdder = transports[0]
class PublicAPI:
"""The new client api server, isolated from the public api."""
def __init__(self):
"""Setup the public api app."""
app = flask.Flask('PublicAPI')
app.config['MAX_CONTENT_LENGTH'] = 5 * 1024 * 1024
self.i2pEnabled = config.get('i2p.host', False)
self.hideBlocks = [] # Blocks to be denied sharing
self.host = apiutils.setbindip.set_bind_IP(
filepaths.public_API_host_file)
threading.Thread(target=_get_tor_adder,
args=[self], daemon=True).start()
self.torAdder = ""
self.bindPort = config.get('client.public.port')
self.lastRequest = 0
# total rec requests to public api since server started
self.hitCount = 0
self.config = config
self.API_VERSION = onionrvalues.API_VERSION
logger.info('Running public api on %s:%s' % (self.host, self.bindPort))
app.register_blueprint(
security.public.PublicAPISecurity(self).public_api_security_bp)
app.register_blueprint(
miscpublicapi.endpoints.PublicEndpoints(self).public_endpoints_bp)
self.app = app
def start(self):
"""Start the Public API server."""
waitforsetvar.wait_for_set_var(self, "_too_many")
self.httpServer = WSGIServer((self.host, self.bindPort),
self.app, log=None,
handler_class=fdsafehandler.FDSafeHandler)
self.httpServer.serve_forever()

15
src/communicator/README.md
View File

@ -1,15 +0,0 @@
# Onionr Communicator
Onionr communicator is the Onionr client. It "connects" to remote Onionr peers and does things such as:
* Finding new peers
* Uploading blocks
* Downloading blocks
* Daemon maintenance/housekeeping
## Files
* \_\_init\_\_.py: Contains the main communicator code. Inits and launches the communicator and sets up the timers
* peeraction.py: contains a function to send commands to remote peers
* bootstrappers.py: adds peers from the bootstrap list to the communicator to try to connect to them
* onlinepeers: management of the online peer pool for the communicator

206
src/communicator/__init__.py
View File

@ -1,206 +0,0 @@
"""Onionr - Private P2P Communication.
This file contains both the OnionrCommunicate class for
communcating with peers and code to operate as a daemon,
getting commands from the command queue database
"""
import time
import config
import logger
import onionrpeers
import onionrplugins as plugins
from . import onlinepeers
from . import uploadqueue
from communicatorutils import downloadblocks
from communicatorutils import lookupblocks
from communicatorutils import lookupadders
from communicatorutils import connectnewpeers
from communicatorutils import uploadblocks
from communicatorutils import announcenode, deniableinserts
from communicatorutils import cooldownpeer
from communicatorutils import housekeeping
from communicatorutils import netcheck
from onionrthreads import add_onionr_thread
from onionrcommands.openwebinterface import get_url
from netcontroller import NetController
from . import bootstrappeers
from . import daemoneventhooks
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
config.reload()
class OnionrCommunicatorDaemon:
def __init__(self, shared_state, developmentMode=None):
if developmentMode is None:
developmentMode = config.get(
'general.dev_mode', False)
# configure logger and stuff
self.config = config
self.shared_state = shared_state # TooManyObjects module
shared_state.add(self)
# populate kv values
self.kv = self.shared_state.get_by_string('DeadSimpleKV')
if config.get('general.offline_mode', False):
self.kv.put('isOnline', False)
# initialize core with Tor socks port being 3rd argument
self.proxyPort = shared_state.get(NetController).socksPort
self.upload_session_manager = self.shared_state.get(
uploadblocks.sessionmanager.BlockUploadSessionManager)
self.shared_state.share_object()
# loop time.sleep delay in seconds
self.delay = 5
# amount of threads running by name, used to prevent too many
self.threadCounts = {}
# Loads in and starts the enabled plugins
plugins.reload()
# extends our upload list and saves our list when Onionr exits
uploadqueue.UploadQueue(self)
add_onionr_thread(
lookupblocks.lookup_blocks_from_communicator,
[self.shared_state], 25, 3)
add_onionr_thread(
downloadblocks.download_blocks_from_communicator,
[self.shared_state],
config.get('timers.getBlocks', 10), 1)
add_onionr_thread(onlinepeers.clear_offline_peer, [self.kv], 58)
add_onionr_thread(
housekeeping.clean_old_blocks, [self.shared_state], 10, 1)
# Discover new peers
add_onionr_thread(
lookupadders.lookup_new_peer_transports_with_communicator,
[shared_state], 60, 3)
# Timer for adjusting which peers
# we actively communicate to at any given time,
# to avoid over-using peers
add_onionr_thread(
cooldownpeer.cooldown_peer, [self.shared_state], 30, 60)
# Timer to read the upload queue and upload the entries to peers
add_onionr_thread(
uploadblocks.upload_blocks_from_communicator,
[self.shared_state], 5, 1)
# This timer creates deniable blocks,
# in an attempt to further obfuscate block insertion metadata
if config.get('general.insert_deniable_blocks', True):
add_onionr_thread(
deniableinserts.insert_deniable_block, [], 180, 10)
if config.get('transports.tor', True):
# Timer to check for connectivity,
# through Tor to various high-profile onion services
add_onionr_thread(netcheck.net_check, [shared_state], 500, 60)
# Announce the public API server transport address
# to other nodes if security level allows
if config.get('general.security_level', 1) == 0 \
and config.get('general.announce_node', True):
# Default to high security level incase config breaks
add_onionr_thread(
announcenode.announce_node, [self.shared_state], 600, 60)
else:
logger.debug('Will not announce node.')
add_onionr_thread(onionrpeers.peer_cleanup, [], 300, 300)
add_onionr_thread(housekeeping.clean_keys, [], 15, 1)
if config.get('general.use_bootstrap_list', True):
bootstrappeers.add_bootstrap_list_to_peer_list(
self.kv, [], db_only=True)
daemoneventhooks.daemon_event_handlers(shared_state)
get_url()
if not config.get('onboarding.done', True):
logger.info(
'First run detected. Run openhome to get setup.',
terminal=True)
while not config.get('onboarding.done', True) and \
not self.shared_state.get_by_string(
'DeadSimpleKV').get('shutdown'):
try:
time.sleep(2)
except KeyboardInterrupt:
self.shared_state.get_by_string(
'DeadSimpleKV').put('shutdown', True)
# Main daemon loop, mainly for calling timers,
# don't do any complex operations here to avoid locking
try:
while not self.shared_state.get_by_string(
'DeadSimpleKV').get('shutdown'):
time.sleep(self.delay)
except KeyboardInterrupt:
self.shared_state.get_by_string(
'DeadSimpleKV').put('shutdown', True)
logger.info(
'Goodbye. (Onionr is cleaning up, and will exit)', terminal=True)
def decrementThreadCount(self, threadName):
"""Decrement amount of a thread name if more than zero.
called when a function meant to be run in a thread ends
"""
try:
if self.threadCounts[threadName] > 0:
self.threadCounts[threadName] -= 1
except KeyError:
pass
def peerCleanup(self):
"""This just calls onionrpeers.cleanupPeers.
Remove dead or bad peers (offline too long, too slow)"""
onionrpeers.peer_cleanup()
self.decrementThreadCount('peerCleanup')
def getPeerProfileInstance(self, peer):
"""Gets a peer profile instance from the list of profiles"""
for i in self.kv.get('peerProfiles'):
# if the peer's profile is already loaded, return that
if i.address == peer:
retData = i
break
else:
# if the peer's profile is not loaded, return a new one.
# connectNewPeer also adds it to the list on connect
retData = onionrpeers.PeerProfiles(peer)
self.kv.get('peerProfiles').append(retData)
return retData
def startCommunicator(shared_state):
OnionrCommunicatorDaemon(shared_state)

36
src/communicator/bootstrappeers.py
View File

@ -1,36 +0,0 @@
"""Onionr - Private P2P Communication.
add bootstrap peers to the communicator peer list
"""
from typing import TYPE_CHECKING
from utils import readstatic, gettransports
from coredb import keydb
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
bootstrap_peers = readstatic.read_static('bootstrap-nodes.txt').split(',')
def add_bootstrap_list_to_peer_list(kv, peerList, db_only=False):
"""Add the bootstrap list to the peer list (no duplicates)."""
for i in bootstrap_peers:
# Add bootstrap peers to peerList (does not save them)
# Don't add them if they're already added or in the offline list
if i not in peerList and i not in kv.get('offlinePeers') \
and i not in gettransports.get() and len(str(i).strip()) > 0:
if not db_only:
peerList.append(i)
keydb.addkeys.add_address(i)

82
src/communicator/daemoneventhooks/__init__.py
View File

@ -1,82 +0,0 @@
"""Onionr - Private P2P Communication.
Hooks to handle daemon events
"""
from threading import Thread
from .removefrominsertqueue import remove_from_insert_queue
from typing import TYPE_CHECKING
from gevent import sleep
from communicatorutils.uploadblocks import mixmate
from communicatorutils import restarttor
if TYPE_CHECKING:
from toomanyobjs import TooMany
from deadsimplekv import DeadSimpleKV
from communicator import OnionrCommunicatorDaemon
from httpapi.daemoneventsapi import DaemonEventsBP
from onionrtypes import BlockHash
from apiservers import PublicAPI
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def daemon_event_handlers(shared_state: 'TooMany'):
def _get_inst(class_name: str):
while True:
try:
return shared_state.get_by_string(class_name)
except KeyError:
sleep(0.2)
comm_inst = _get_inst('OnionrCommunicatorDaemon')
public_api: 'PublicAPI' = _get_inst('PublicAPI')
events_api: 'DaemonEventsBP' = _get_inst('DaemonEventsBP')
kv: 'DeadSimpleKV' = _get_inst('DeadSimpleKV')
def remove_from_insert_queue_wrapper(block_hash: 'BlockHash'):
remove_from_insert_queue(comm_inst, block_hash)
return "removed"
def print_test(text=''):
print("It works!", text)
return f"It works! {text}"
def upload_event(block: 'BlockHash' = ''):
if not block:
raise ValueError
public_api.hideBlocks.append(block)
try:
mixmate.block_mixer(kv.get('blocksToUpload'), block)
except ValueError:
pass
return "removed"
def restart_tor():
restarttor.restart(shared_state)
kv.put('offlinePeers', [])
kv.put('onlinePeers', [])
def test_runtime():
Thread(target=comm_inst.shared_state.get_by_string(
"OnionrRunTestManager").run_tests).start()
events_api.register_listener(remove_from_insert_queue_wrapper)
events_api.register_listener(restart_tor)
events_api.register_listener(print_test)
events_api.register_listener(upload_event)
events_api.register_listener(test_runtime)

33
src/communicator/daemoneventhooks/removefrominsertqueue.py
View File

@ -1,33 +0,0 @@
"""Onionr - P2P Anonymous Storage Network.
Remove block hash from daemon's upload list.
"""
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
from communicator import OnionrCommunicatorDaemon
from onionrtypes import BlockHash
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def remove_from_insert_queue(comm_inst: "OnionrCommunicatorDaemon",
b_hash: "BlockHash"):
"""Remove block hash from daemon's upload list."""
kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV")
try:
kv.get('generating_blocks').remove(b_hash)
except ValueError:
pass

12
src/communicator/onlinepeers/README.md
View File

@ -1,12 +0,0 @@
# Online Peers
Manages a pool of peers to perform actions with. Since Onionr does not maintain socket connections, it holds a list of peers.
## Files
* \_\_init\_\_.py: exposes some functions to interact with the pool
* clearofflinepeer.py: Pop the oldest peer in the offline list
* onlinepeers.py: communicator timer to add new peers to the pool randomly
* pickonlinepeers.py: returns a random peer from the online pool
* removeonlinepeer.py: removes a specified peer from the online pool

6
src/communicator/onlinepeers/__init__.py
View File

@ -1,6 +0,0 @@
from . import clearofflinepeer, onlinepeers, pickonlinepeers, removeonlinepeer
clear_offline_peer = clearofflinepeer.clear_offline_peer
get_online_peers = onlinepeers.get_online_peers
pick_online_peer = pickonlinepeers.pick_online_peer
remove_online_peer = removeonlinepeer.remove_online_peer

35
src/communicator/onlinepeers/clearofflinepeer.py
View File

@ -1,35 +0,0 @@
"""Onionr - Private P2P Communication.
clear offline peer in a communicator instance
"""
from typing import TYPE_CHECKING
import logger
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def clear_offline_peer(kv: 'DeadSimpleKV'):
"""Remove the longest offline peer to retry later."""
try:
removed = kv.get('offlinePeers').pop(0)
except IndexError:
pass
else:
logger.debug('Removed ' + removed +
' from offline list, will try them again.')

63
src/communicator/onlinepeers/onlinepeers.py
View File

@ -1,63 +0,0 @@
"""Onionr - Private P2P Communication.
get online peers in a communicator instance
"""
import time
from typing import TYPE_CHECKING
import config
from etc.humanreadabletime import human_readable_time
from communicatorutils.connectnewpeers import connect_new_peer_to_communicator
import logger
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def get_online_peers(shared_state):
"""Manage the kv.get('onlinePeers') attribute list.
Connect to more peers if we have none connected
"""
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
if config.get('general.offline_mode', False):
return
logger.info('Refreshing peer pool...')
max_peers = int(config.get('peers.max_connect', 10))
needed = max_peers - len(kv.get('onlinePeers'))
last_seen = 'never'
if not isinstance(kv.get('lastNodeSeen'), type(None)):
last_seen = human_readable_time(kv.get('lastNodeSeen'))
for _ in range(needed):
if len(kv.get('onlinePeers')) == 0:
connect_new_peer_to_communicator(shared_state, useBootstrap=True)
else:
connect_new_peer_to_communicator(shared_state)
if kv.get('shutdown'):
break
else:
if len(kv.get('onlinePeers')) == 0:
logger.debug('Couldn\'t connect to any peers.' +
f' Last node seen {last_seen} ago.')
try:
get_online_peers(shared_state)
except RecursionError:
pass
else:
kv.put('lastNodeSeen', time.time())

47
src/communicator/onlinepeers/pickonlinepeers.py
View File

@ -1,47 +0,0 @@
"""
Onionr - Private P2P Communication.
pick online peers in a communicator instance
"""
import secrets
from typing import TYPE_CHECKING
import onionrexceptions
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def pick_online_peer(kv: 'DeadSimpleKV'):
"""Randomly picks peer from pool without bias (using secrets module)."""
ret_data = ''
peer_length = len(kv.get('onlinePeers'))
if peer_length <= 0:
raise onionrexceptions.OnlinePeerNeeded
while True:
peer_length = len(kv.get('onlinePeers'))
try:
# Get a random online peer, securely.
# May get stuck in loop if network is lost
ret_data = kv.get('onlinePeers')[secrets.randbelow(peer_length)]
except IndexError:
pass
else:
break
return ret_data

38
src/communicator/onlinepeers/removeonlinepeer.py
View File

@ -1,38 +0,0 @@
"""Onionr - Private P2P Communication.
remove an online peer from the pool in a communicator instance
"""
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def remove_online_peer(kv, peer):
"""Remove an online peer."""
try:
del kv.get('connectTimes')[peer]
except KeyError:
pass
try:
del kv.get('dbTimestamps')[peer]
except KeyError:
pass
try:
kv.get('onlinePeers').remove(peer)
except ValueError:
pass

78
src/communicator/peeraction.py
View File

@ -1,78 +0,0 @@
"""Onionr - Private P2P Communication.
This file implements logic for performing requests to Onionr peers
"""
from typing import TYPE_CHECKING
import streamedrequests
import logger
from onionrutils import epoch, basicrequests
from coredb import keydb
from . import onlinepeers
from onionrtypes import OnionAddressString
from onionrpeers.peerprofiles import PeerProfiles
from etc.waitforsetvar import wait_for_set_var
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def get_peer_profile(kv, address: OnionAddressString) -> 'PeerProfiles':
profile_inst_list = kv.get('peerProfiles')
for profile in profile_inst_list:
if profile.address == address:
return profile
p = PeerProfiles(address)
profile_inst_list.append(p)
return p
def peer_action(shared_state, peer, action,
returnHeaders=False, max_resp_size=5242880):
"""Perform a get request to a peer."""
penalty_score = -10
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
if len(peer) == 0:
return False
url = 'http://%s/%s' % (peer, action)
try:
ret_data = basicrequests.do_get_request(url, port=kv.get('proxyPort'),
max_size=max_resp_size)
except streamedrequests.exceptions.ResponseLimitReached:
logger.warn(
'Request failed due to max response size being overflowed',
terminal=True)
ret_data = False
penalty_score = -100
# if request failed, (error), mark peer offline
if ret_data is False:
try:
get_peer_profile(kv, peer).addScore(penalty_score)
onlinepeers.remove_online_peer(kv, peer)
keydb.transportinfo.set_address_info(
peer, 'lastConnectAttempt', epoch.get_epoch())
if action != 'ping' and not kv.get('shutdown'):
logger.warn(f'Lost connection to {peer}', terminal=True)
# Will only add a new peer to pool if needed
onlinepeers.get_online_peers(kv)
except ValueError:
pass
else:
peer_profile = get_peer_profile(kv, peer)
peer_profile.update_connect_time()
peer_profile.addScore(1)
# If returnHeaders, returns tuple of data, headers.
# If not, just data string
return ret_data

73
src/communicator/uploadqueue/__init__.py
View File

@ -1,73 +0,0 @@
"""Onionr - Private P2P Communication.
Class to remember blocks that need to be uploaded
and not shared on startup/shutdown
"""
import atexit
import os
from typing import TYPE_CHECKING
import deadsimplekv
import filepaths
from onionrutils import localcommand
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
UPLOAD_MEMORY_FILE = filepaths.upload_list
def _add_to_hidden_blocks(cache):
for bl in cache:
localcommand.local_command('waitforshare/' + bl, post=True)
class UploadQueue:
"""Saves and loads block upload info from json file."""
def __init__(self, communicator: 'OnionrCommunicatorDaemon'):
"""Start the UploadQueue object, loading left over uploads into queue.
register save shutdown function
"""
self.communicator = communicator
cache: deadsimplekv.DeadSimpleKV = deadsimplekv.DeadSimpleKV(
UPLOAD_MEMORY_FILE)
self.kv: "DeadSimpleKV" = \
communicator.shared_state.get_by_string("DeadSimpleKV")
self.store_obj = cache
cache = cache.get('uploads')
if cache is None:
cache = []
_add_to_hidden_blocks(cache)
self.kv.get('blocksToUpload').extend(cache)
atexit.register(self.save)
def save(self):
"""Save to disk on shutdown or if called manually."""
bl: deadsimplekv.DeadSimpleKV = self.kv.get('blocksToUpload')
if len(bl) == 0:
try:
os.remove(UPLOAD_MEMORY_FILE)
except FileNotFoundError:
pass
else:
self.store_obj.put('uploads', bl)
self.store_obj.flush()

33
src/communicatorutils/README.md
View File

@ -1,33 +0,0 @@
# communicatorutils
The files in this submodule handle various subtasks and utilities for the onionr communicator.
## Files:
announcenode.py: Uses a communicator instance to announce our transport address to connected nodes
connectnewpeers.py: takes a communicator instance and has it connect to as many peers as needed, and/or to a new specified peer.
cooldownpeer.py: randomly selects a connected peer in a communicator and disconnects them for the purpose of security and network balancing.
daemonqueuehandler.py: checks for new commands in the daemon queue and processes them accordingly.
deniableinserts.py: insert fake blocks with the communicator for plausible deniability
downloadblocks.py: iterates a communicator instance's block download queue and attempts to download the blocks from online peers
housekeeping.py: cleans old blocks and forward secrecy keys
lookupadders.py: ask connected peers to share their list of peer transport addresses
lookupblocks.py: lookup new blocks from connected peers from the communicator
netcheck.py: check if the node is online based on communicator status and onion server ping results
onionrcommunicataortimers.py: create a timer for a function to be launched on an interval. Control how many possible instances of a timer may be running a function at once and control if the timer should be ran in a thread or not.
proxypicker.py: returns a string name for the appropriate proxy to be used with a particular peer transport address.
servicecreator.py: iterate connection blocks and create new direct connection servers for them.
uploadblocks.py: iterate a communicator's upload queue and upload the blocks to connected peers

0
src/communicatorutils/__init__.py
View File

77
src/communicatorutils/announcenode.py
View File

@ -1,77 +0,0 @@
"""
Onionr - Private P2P Communication.
Use a communicator instance to announce
our transport address to connected nodes
"""
from typing import TYPE_CHECKING
import logger
from onionrutils import basicrequests
from utils import gettransports
from netcontroller import NetController
from communicator import onlinepeers
from coredb import keydb
import onionrexceptions
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def announce_node(shared_state):
"""Announce our node to our peers."""
ret_data = False
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
config = shared_state.get_by_string("OnionrCommunicatorDaemon").config
# Do not let announceCache get too large
if len(kv.get('announceCache')) >= 10000:
kv.get('announceCache').popitem()
if config.get('general.security_level', 0) == 0:
# Announce to random online peers
for i in kv.get('onlinePeers'):
if i not in kv.get('announceCache'):
peer = i
break
else:
try:
peer = onlinepeers.pick_online_peer(kv)
except onionrexceptions.OnlinePeerNeeded:
peer = ""
try:
ourID = gettransports.get()[0]
if not peer:
raise onionrexceptions.OnlinePeerNeeded
except (IndexError, onionrexceptions.OnlinePeerNeeded):
pass
else:
url = 'http://' + peer + '/announce'
data = {'node': ourID}
logger.info('Announcing node to ' + url)
if basicrequests.do_post_request(
url,
data,
port=shared_state.get(NetController).socksPort)\
== 'Success':
logger.info('Successfully introduced node to ' + peer,
terminal=True)
ret_data = True
keydb.transportinfo.set_address_info(peer, 'introduced', 1)
return ret_data

117
src/communicatorutils/connectnewpeers.py
View File

@ -1,117 +0,0 @@
"""Onionr - Private P2P Communication.
Connect a new peer to our communicator instance.
Does so randomly if no peer is specified
"""
import time
import secrets
import onionrexceptions
import logger
import onionrpeers
from utils import networkmerger, gettransports
from onionrutils import stringvalidators, epoch
from communicator import peeraction, bootstrappeers
from coredb import keydb
import config
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def connect_new_peer_to_communicator(shared_state, peer='', useBootstrap=False):
retData = False
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
tried = kv.get('offlinePeers')
transports = gettransports.get()
if peer != '':
if stringvalidators.validate_transport(peer):
peerList = [peer]
else:
raise onionrexceptions.InvalidAddress(
'Will not attempt connection test to invalid address')
else:
peerList = keydb.listkeys.list_adders()
mainPeerList = keydb.listkeys.list_adders()
if not peerList:
peerList = onionrpeers.get_score_sorted_peer_list()
"""
If we don't have enough peers connected or random chance,
select new peers to try
"""
if len(peerList) < 8 or secrets.randbelow(4) == 3:
tryingNew = []
for x in kv.get('newPeers'):
if x not in peerList:
peerList.append(x)
tryingNew.append(x)
for i in tryingNew:
kv.get('newPeers').remove(i)
if len(peerList) == 0 or useBootstrap:
# Avoid duplicating bootstrap addresses in peerList
if config.get('general.use_bootstrap_list', True):
bootstrappeers.add_bootstrap_list_to_peer_list(kv, peerList)
for address in peerList:
address = address.strip()
# Don't connect to our own address
if address in transports:
continue
"""Don't connect to invalid address or
if its already been tried/connected, or if its cooled down
"""
if len(address) == 0 or address in tried \
or address in kv.get('onlinePeers') \
or address in kv.get('cooldownPeer'):
continue
if kv.get('shutdown'):
return
# Ping a peer,
ret = peeraction.peer_action(shared_state, address, 'ping')
if ret == 'pong!':
time.sleep(0.1)
if address not in mainPeerList:
# Add a peer to our list if it isn't already since it connected
networkmerger.mergeAdders(address)
if address not in kv.get('onlinePeers'):
logger.info('Connected to ' + address, terminal=True)
kv.get('onlinePeers').append(address)
kv.get('connectTimes')[address] = epoch.get_epoch()
retData = address
# add peer to profile list if they're not in it
for profile in kv.get('peerProfiles'):
if profile.address == address:
break
else:
kv.get('peerProfiles').append(
onionrpeers.PeerProfiles(address))
try:
del kv.get('plaintextDisabledPeers')[address]
except KeyError:
pass
if peeraction.peer_action(
shared_state, address, 'plaintext') == 'false':
kv.get('plaintextDisabledPeers')[address] = True
break
else:
# Mark a peer as tried if they failed to respond to ping
tried.append(address)
logger.debug('Failed to connect to %s: %s ' % (address, ret))
return retData

60
src/communicatorutils/cooldownpeer.py
View File

@ -1,60 +0,0 @@
"""Onionr - Private P2P Communication.
Select random online peer in a communicator instance and have them "cool down"
"""
from typing import TYPE_CHECKING
from onionrutils import epoch
from communicator import onlinepeers
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def cooldown_peer(shared_state):
"""Randomly add an online peer to cooldown, so we can connect a new one."""
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
config = shared_state.get_by_string("OnionrCommunicatorDaemon").config
online_peer_amount = len(kv.get('onlinePeers'))
minTime = 300
cooldown_time = 600
to_cool = ''
tempConnectTimes = dict(kv.get('connectTimes'))
# Remove peers from cooldown that have been there long enough
tempCooldown = dict(kv.get('cooldownPeer'))
for peer in tempCooldown:
if (epoch.get_epoch() - tempCooldown[peer]) >= cooldown_time:
del kv.get('cooldownPeer')[peer]
# Cool down a peer, if we have max connections alive for long enough
if online_peer_amount >= config.get('peers.max_connect', 10, save=True):
finding = True
while finding:
try:
to_cool = min(tempConnectTimes, key=tempConnectTimes.get)
if (epoch.get_epoch() - tempConnectTimes[to_cool]) < minTime:
del tempConnectTimes[to_cool]
else:
finding = False
except ValueError:
break
else:
onlinepeers.remove_online_peer(kv, to_cool)
kv.get('cooldownPeer')[to_cool] = epoch.get_epoch()

35
src/communicatorutils/deniableinserts.py
View File

@ -1,35 +0,0 @@
"""Onionr - Private P2P Communication.
Use the communicator to insert fake mail messages
"""
import secrets
from etc import onionrvalues
import onionrblocks
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def insert_deniable_block():
"""Insert a fake block to make it more difficult to track real blocks."""
fakePeer = ''
chance = 10
if secrets.randbelow(chance) == (chance - 1):
# This assumes on the libsodium primitives to have key-privacy
fakePeer = onionrvalues.DENIABLE_PEER_ADDRESS
data = secrets.token_hex(secrets.randbelow(5120) + 1)
onionrblocks.insert(data, header='pm', encryptType='asym',
asymPeer=fakePeer, disableForward=True,
meta={'subject': 'foo'})

142
src/communicatorutils/downloadblocks/__init__.py
View File

@ -1,142 +0,0 @@
"""Onionr - Private P2P Communication.
Download blocks using the communicator instance.
"""
from typing import TYPE_CHECKING
from secrets import SystemRandom
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
from deadsimplekv import DeadSimpleKV
from gevent import spawn
import onionrexceptions
import logger
import onionrpeers
from communicator import peeraction
from communicator import onlinepeers
from onionrblocks import blockmetadata
from onionrutils import validatemetadata
from coredb import blockmetadb
from onionrutils.localcommand import local_command
import onionrcrypto
import onionrstorage
from onionrblocks import onionrblacklist
from onionrblocks import storagecounter
from onionrproofs import vdf
from . import shoulddownload
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
storage_counter = storagecounter.StorageCounter()
def download_blocks_from_communicator(shared_state: "TooMany"):
"""Use communicator instance to download blocks in the comms's queue"""
blacklist = onionrblacklist.OnionrBlackList()
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
count: int = 0
metadata_validation_result: bool = False
# Iterate the block queue in the communicator
for blockHash in list(kv.get('blockQueue')):
count += 1
try:
blockPeers = list(kv.get('blockQueue')[blockHash])
except KeyError:
blockPeers = []
removeFromQueue = True
if not shoulddownload.should_download(shared_state, blockHash):
continue
if kv.get('shutdown') or not kv.get('isOnline') or \
storage_counter.is_full():
# Exit loop if shutting down or offline, or disk allocation reached
break
# Do not download blocks being downloaded
if blockHash in kv.get('currentDownloading'):
continue
if len(kv.get('onlinePeers')) == 0:
break
# So we can avoid concurrent downloading in other threads of same block
kv.get('currentDownloading').append(blockHash)
if len(blockPeers) == 0:
try:
peerUsed = onlinepeers.pick_online_peer(kv)
except onionrexceptions.OnlinePeerNeeded:
continue
else:
SystemRandom().shuffle(blockPeers)
peerUsed = blockPeers.pop(0)
if not kv.get('shutdown') and peerUsed.strip() != '':
logger.info(
f"Attempting to download %s from {peerUsed}..." % (blockHash[:12],))
content = peeraction.peer_action(
shared_state, peerUsed,
'getdata/' + blockHash,
max_resp_size=3000000) # block content from random peer
if content is not False and len(content) > 0:
try:
content = content.encode()
except AttributeError:
pass
if vdf.verify_vdf(blockHash, content):
#content = content.decode() # decode here because sha3Hash needs bytes above
metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
metadata = metas[0]
try:
metadata_validation_result = \
validatemetadata.validate_metadata(metadata, metas[2])
except onionrexceptions.PlaintextNotSupported:
logger.debug(f"Not saving {blockHash} due to plaintext not enabled")
removeFromQueue = True
except onionrexceptions.DataExists:
metadata_validation_result = False
if metadata_validation_result: # check if metadata is valid, and verify nonce
save_block(blockHash, data)
else:
if blacklist.inBlacklist(blockHash):
logger.warn(f'Block {blockHash} is blacklisted.')
else:
logger.warn('Metadata for block %s is invalid.' % (blockHash,))
blacklist.addToDB(blockHash)
else:
# if block didn't meet expected hash
tempHash = onionrcrypto.hashers.sha3_hash(content) # lazy hack, TODO use var
try:
tempHash = tempHash.decode()
except AttributeError:
pass
# Punish peer for sharing invalid block (not always malicious, but is bad regardless)
onionrpeers.PeerProfiles(peerUsed).addScore(-50)
if removeFromQueue:
try:
del kv.get('blockQueue')[blockHash] # remove from block queue both if success or false
if count == LOG_SKIP_COUNT:
logger.info('%s blocks remaining in queue' %
[len(kv.get('blockQueue'))], terminal=True)
count = 0
except KeyError:
pass
kv.get('currentDownloading').remove(blockHash)

42
src/communicatorutils/downloadblocks/shoulddownload.py
View File

@ -1,42 +0,0 @@
"""Onionr - Private P2P Communication.
Check if a block should be downloaded
(if we already have it or its blacklisted or not)
"""
from coredb import blockmetadb
from onionrblocks import onionrblacklist
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def should_download(shared_state, block_hash) -> bool:
"""Return bool for if a (assumed to exist) block should be downloaded."""
blacklist = onionrblacklist.OnionrBlackList()
should = True
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
if block_hash in blockmetadb.get_block_list():
# Don't download block we have
should = False
else:
if blacklist.inBlacklist(block_hash):
# Don't download blacklisted block
should = False
if should is False:
# Remove block from communicator queue if it shouldn't be downloaded
try:
del kv.get('blockQueue')[block_hash]
except KeyError:
pass
return should

108
src/communicatorutils/housekeeping.py
View File

@ -1,108 +0,0 @@
"""Onionr - Private P2P Communication.
Cleanup old Onionr blocks and forward secrecy keys using the communicator.
Ran from a communicator timer usually
"""
import sqlite3
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
import logger
from onionrusers import onionrusers
from onionrutils import epoch
from coredb import blockmetadb, dbfiles
import onionrstorage
from onionrstorage import removeblock
from onionrblocks import onionrblacklist
from onionrblocks.storagecounter import StorageCounter
from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
from onionrproofs import hashMeetsDifficulty
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
storage_counter = StorageCounter()
def __remove_from_upload(shared_state, block_hash: str):
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
try:
kv.get('blocksToUpload').remove(block_hash)
except ValueError:
pass
def __purge_block(shared_state, block_hash, add_to_blacklist = True):
blacklist = None
removeblock.remove_block(block_hash)
onionrstorage.deleteBlock(block_hash)
__remove_from_upload(shared_state, block_hash)
if add_to_blacklist:
blacklist = onionrblacklist.OnionrBlackList()
blacklist.addToDB(block_hash)
def clean_old_blocks(shared_state):
"""Delete expired blocks + old blocks if disk allocation is near full"""
blacklist = onionrblacklist.OnionrBlackList()
# Delete expired blocks
for bHash in blockmetadb.expiredblocks.get_expired_blocks():
__purge_block(shared_state, bHash, add_to_blacklist=True)
logger.info('Deleted expired block: %s' % (bHash,))
while storage_counter.is_full():
try:
oldest = blockmetadb.get_block_list()[0]
except IndexError:
break
else:
__purge_block(shared_state, bHash, add_to_blacklist=True)
logger.info('Deleted block because of full storage: %s' % (oldest,))
def clean_keys():
"""Delete expired forward secrecy keys"""
conn = sqlite3.connect(dbfiles.user_id_info_db,
timeout=DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
time = epoch.get_epoch()
deleteKeys = []
for entry in c.execute(
"SELECT * FROM forwardKeys WHERE expire <= ?", (time,)):
logger.debug('Forward key: %s' % entry[1])
deleteKeys.append(entry[1])
for key in deleteKeys:
logger.debug('Deleting forward key %s' % key)
c.execute("DELETE from forwardKeys where forwardKey = ?", (key,))
conn.commit()
conn.close()
onionrusers.deleteExpiredKeys()
def clean_blocks_not_meeting_pow(shared_state):
"""Clean blocks not meeting min send/rec pow. Used if config.json POW changes"""
block_list = blockmetadb.get_block_list()
for block in block_list:
if not hashMeetsDifficulty(block):
logger.warn(
f"Deleting block {block} because it was stored" +
"with a POW level smaller than current.", terminal=True)
__purge_block(shared_state, block)

66
src/communicatorutils/lookupadders.py
View File

@ -1,66 +0,0 @@
"""Onionr - Private P2P Communication.
Lookup new peer transport addresses using the communicator
"""
from typing import TYPE_CHECKING
import logger
from onionrutils import stringvalidators
from communicator import peeraction, onlinepeers
from utils import gettransports
import onionrexceptions
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def lookup_new_peer_transports_with_communicator(shared_state):
logger.info('Looking up new addresses...')
tryAmount = 1
newPeers = []
transports = gettransports.get()
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
for i in range(tryAmount):
# Download new peer address list from random online peers
if len(newPeers) > 10000:
# Don't get new peers if we have too many queued up
break
try:
peer = onlinepeers.pick_online_peer(kv)
newAdders = peeraction.peer_action(shared_state, peer, action='pex')
except onionrexceptions.OnlinePeerNeeded:
continue
try:
newPeers = newAdders.split(',')
except AttributeError:
pass
else:
# Validate new peers are good format and not already in queue
invalid = []
for x in newPeers:
x = x.strip()
if not stringvalidators.validate_transport(x) \
or x in kv.get('newPeers') or x in transports:
# avoid adding if its our address
invalid.append(x)
for x in invalid:
try:
newPeers.remove(x)
except ValueError:
pass
kv.get('newPeers').extend(newPeers)

126
src/communicatorutils/lookupblocks.py
View File

@ -1,126 +0,0 @@
"""Onionr - Private P2P Communication.
Lookup new blocks with the communicator using a random connected peer
"""
from typing import TYPE_CHECKING
from gevent import time
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
import logger
import onionrproofs
from onionrutils import stringvalidators, epoch
from communicator import peeraction, onlinepeers
from coredb.blockmetadb import get_block_list
from utils import reconstructhash
from onionrblocks import onionrblacklist
import onionrexceptions
import config
from etc import onionrvalues
from onionrblocks.storagecounter import StorageCounter
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
blacklist = onionrblacklist.OnionrBlackList()
storage_counter = StorageCounter()
def lookup_blocks_from_communicator(shared_state: 'TooMany'):
logger.info('Looking up new blocks')
tryAmount = 2
newBlocks = ''
# List of existing saved blocks
existingBlocks = get_block_list()
triedPeers = [] # list of peers we've tried this time around
# Max amount of *new* block hashes to have in queue
maxBacklog = 1560
lastLookupTime = 0 # Last time we looked up a particular peer's list
new_block_count = 0
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
for i in range(tryAmount):
# Defined here to reset it each time, time offset is added later
listLookupCommand = 'getblocklist'
if len(kv.get('blockQueue')) >= maxBacklog:
break
if not kv.get('isOnline'):
break
# check if disk allocation is used
if storage_counter.is_full():
logger.debug(
'Not looking up new blocks due to maximum amount of disk used')
break
try:
# select random online peer
peer = onlinepeers.pick_online_peer(kv)
except onionrexceptions.OnlinePeerNeeded:
time.sleep(1)
continue
# if we've already tried all the online peers this time around, stop
if peer in triedPeers:
if len(kv.get('onlinePeers')) == len(triedPeers):
break
else:
continue
triedPeers.append(peer)
# Get the last time we looked up a peer's stamp,
# to only fetch blocks since then.
# Saved in memory only for privacy reasons
try:
lastLookupTime = kv.get('dbTimestamps')[peer]
except KeyError:
lastLookupTime = epoch.get_epoch() - onionrvalues.DEFAULT_EXPIRE
listLookupCommand += '?date=%s' % (lastLookupTime,)
try:
newBlocks = peeraction.peer_action(
shared_state,
peer, listLookupCommand) # get list of new block hashes
except Exception as error:
logger.warn(
f'Could not get new blocks from {peer}.',
error=error)
newBlocks = False
if newBlocks != False: # noqa
# if request was a success
for i in newBlocks.split('\n'):
if stringvalidators.validate_hash(i):
i = reconstructhash.reconstruct_hash(i)
# if newline seperated string is valid hash
# if block does not exist on disk + is not already in queue
if i not in existingBlocks:
if i not in kv.get('blockQueue'):
if onionrproofs.hashMeetsDifficulty(i) and \
not blacklist.inBlacklist(i):
if len(kv.get('blockQueue')) <= 1000000:
# add blocks to download queue
kv.get('blockQueue')[i] = [peer]
new_block_count += 1
kv.get('dbTimestamps')[peer] = \
epoch.get_rounded_epoch(roundS=60)
else:
if peer not in kv.get('blockQueue')[i]:
if len(kv.get('blockQueue')[i]) < 10:
kv.get('blockQueue')[i].append(peer)
if new_block_count > 0:
block_string = ""
if new_block_count > 1:
block_string = "s"
logger.info(
f'Discovered {new_block_count} new block{block_string}',
terminal=True)

64
src/communicatorutils/netcheck.py
View File

@ -1,64 +0,0 @@
"""
Onionr - Private P2P Communication.
Determine if our node is able to use Tor based
on the status of a communicator instance
and the result of pinging onion http servers
"""
import logger
from utils import netutils
from onionrutils import localcommand, epoch
from . import restarttor
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def net_check(shared_state):
"""Check if we are connected to the internet.
or not when we can't connect to any peers
"""
# for detecting if we have received incoming connections recently
rec = False
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
proxy_port = shared_state.get_by_string("NetController").socksPort
if len(kv.get('onlinePeers')) == 0:
try:
if (epoch.get_epoch() - int(localcommand.local_command
('/lastconnect'))) <= 60:
kv.put('isOnline', True)
rec = True
except ValueError:
pass
if not rec and not netutils.check_network(torPort=proxy_port):
if not kv.get('shutdown'):
if not shared_state.get_by_string(
"OnionrCommunicatorDaemon").config.get(
'general.offline_mode', False):
logger.warn('Network check failed, are you connected to ' +
'the Internet, and is Tor working? ' +
'This is usually temporary, but bugs and censorship can cause this to persist, in which case you should report it to beardog [at] mailbox.org', # noqa
terminal=True)
restarttor.restart(shared_state)
kv.put('offlinePeers', [])
kv.put('isOnline', False)
else:
kv.put('isOnline', True)

28
src/communicatorutils/proxypicker.py
View File

@ -1,28 +0,0 @@
"""
Onionr - Private P2P Communication.
Pick a proxy to use based on a peer's address
"""
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def pick_proxy(peer_address):
if peer_address.endswith('.onion'):
return 'tor'
elif peer_address.endswith('.i2p'):
return 'i2p'
raise ValueError(
f"Peer address not ending with acceptable domain: {peer_address}")

28
src/communicatorutils/restarttor.py
View File

@ -1,28 +0,0 @@
"""
Onionr - Private P2P Communication.
Restart Onionr managed Tor
"""
import netcontroller
import config
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def restart(shared_state):
if not config.get('tor.use_existing_tor', False):
net = shared_state.get(netcontroller.NetController)
net.killTor()
net.startTor()

148
src/communicatorutils/uploadblocks/__init__.py
View File

@ -1,148 +0,0 @@
"""Onionr - Private P2P Communication.
Upload blocks in the upload queue to peers from the communicator
"""
from typing import TYPE_CHECKING
from time import sleep
from threading import Thread
from secrets import SystemRandom
from . import sessionmanager
from onionrtypes import UserID
import logger
from communicatorutils import proxypicker
import onionrexceptions
from onionrblocks import onionrblockapi as block
from onionrblocks.blockmetadata.fromdata import get_block_metadata_from_data
from onionrutils import stringvalidators, basicrequests
from onionrutils.validatemetadata import validate_metadata
from communicator import onlinepeers
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
from communicator import OnionrCommunicatorDaemon
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def upload_blocks_from_communicator(shared_state: 'OnionrCommunicatorDaemon'):
"""Accept a communicator instance + upload blocks from its upload queue."""
"""when inserting a block, we try to upload
it to a few peers to add some deniability & increase functionality"""
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
session_manager: sessionmanager.BlockUploadSessionManager
session_manager = shared_state.get(
sessionmanager.BlockUploadSessionManager)
tried_peers: UserID = []
finishedUploads = []
SystemRandom().shuffle(kv.get('blocksToUpload'))
def remove_from_hidden(bl):
sleep(60)
try:
shared_state.get_by_string(
'PublicAPI').hideBlocks.remove(bl)
except ValueError:
pass
if len(kv.get('blocksToUpload')) != 0:
for bl in kv.get('blocksToUpload'):
if not stringvalidators.validate_hash(bl):
logger.warn('Requested to upload invalid block', terminal=True)
return
session = session_manager.add_session(bl)
for _ in range(min(len(kv.get('onlinePeers')), 6)):
try:
peer = onlinepeers.pick_online_peer(kv)
if not block.Block(bl).isEncrypted:
if peer in kv.get('plaintextDisabledPeers'):
logger.info(f"Cannot upload plaintext block to peer that denies it {peer}") # noqa
continue
except onionrexceptions.OnlinePeerNeeded:
continue
try:
session.peer_exists[peer]
continue
except KeyError:
pass
try:
if session.peer_fails[peer] > 3:
continue
except KeyError:
pass
if peer in tried_peers:
continue
tried_peers.append(peer)
url = f'http://{peer}/upload'
try:
data = block.Block(bl).getRaw()
if not data:
logger.warn(
f"Couldn't get data for block in upload list {bl}",
terminal=True)
raise onionrexceptions.NoDataAvailable
try:
def __check_metadata():
metadata = get_block_metadata_from_data(data)[0]
if not validate_metadata(metadata, data):
logger.warn(
f"Metadata for uploading block not valid {bl}")
raise onionrexceptions.InvalidMetadata
__check_metadata()
except onionrexceptions.DataExists:
pass
except( # noqa
onionrexceptions.NoDataAvailable,
onionrexceptions.InvalidMetadata) as _:
finishedUploads.append(bl)
break
proxy_type = proxypicker.pick_proxy(peer)
logger.info(
f"Uploading block {bl[:8]} to {peer}", terminal=True)
resp = basicrequests.do_post_request(
url, data=data, proxyType=proxy_type,
content_type='application/octet-stream')
if resp is not False:
if resp == 'success':
Thread(target=remove_from_hidden,
args=[bl], daemon=True).start()
session.success()
session.peer_exists[peer] = True
elif resp == 'exists':
session.success()
session.peer_exists[peer] = True
else:
session.fail()
session.fail_peer(peer)
shared_state.get_by_string(
'OnionrCommunicatorDaemon').getPeerProfileInstance(
peer).addScore(-5)
logger.warn(
f'Failed to upload {bl[:8]}, reason: {resp}',
terminal=True)
else:
session.fail()
session_manager.clean_session()
for x in finishedUploads:
try:
kv.get('blocksToUpload').remove(x)
shared_state.get_by_string(
'PublicAPI').hideBlocks.remove(x)
except ValueError:
pass

48
src/communicatorutils/uploadblocks/mixmate/__init__.py
View File

@ -1,48 +0,0 @@
"""Onionr - Private P2P Communication.
Perform block mixing
"""
import time
from typing import List
import onionrtypes
from onionrblocks import onionrblockapi
from .pool import UploadPool
from .pool import PoolFullException
from etc import onionrvalues
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
upload_pool = UploadPool(4)
def block_mixer(upload_list: List[onionrtypes.BlockHash],
block_to_mix: onionrtypes.BlockHash):
"""Delay and mix block inserts.
Take a block list and a received/created block and add it
to the said block list
"""
bl = onionrblockapi.Block(block_to_mix)
try:
if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE:
raise ValueError
except TypeError:
pass
if block_to_mix:
upload_list.append(block_to_mix)

71
src/communicatorutils/uploadblocks/mixmate/pool.py
View File

@ -1,71 +0,0 @@
"""Onionr - Private P2P Communication.
Upload pool
"""
from typing import List
from secrets import SystemRandom
import onionrutils
import onionrtypes
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
class PoolFullException(Exception):
"""For when the UploadPool is full.
Raise when a new hash is attempted to be added
"""
class PoolNotReady(Exception):
"""Raise when UploadPool pool access is attempted without it being full."""
class AlreadyInPool(Exception):
"""Raise when a hash already in pool is attempted to be added again."""
class UploadPool:
"""Upload pool for mixing blocks together and delaying uploads."""
def __init__(self, pool_size: int):
"""Create a new pool with a specified max size.
Uses private var and getter to avoid direct adding
"""
self._pool: List[onionrtypes.BlockHash] = []
self._pool_size = pool_size
self.birthday = onionrutils.epoch.get_epoch()
def add_to_pool(self, item: List[onionrtypes.BlockHash]):
"""Add a new hash to the pool. Raise PoolFullException if full."""
if len(self._pool) >= self._pool_size:
raise PoolFullException
if not onionrutils.stringvalidators.validate_hash(item):
raise ValueError
self._pool.append(item)
def get_pool(self) -> List[onionrtypes.BlockHash]:
"""Get the hash pool in secure random order."""
if len(self._pool) != self._pool_size:
raise PoolNotReady
final_pool: List[onionrtypes.BlockHash] = list(self._pool)
SystemRandom().shuffle(final_pool)
self._pool.clear()
self.birthday = onionrutils.epoch.get_epoch()
return final_pool

57
src/communicatorutils/uploadblocks/session.py
View File

@ -1,57 +0,0 @@
"""Onionr - Private P2P Communication.
Virtual upload "sessions" for blocks
"""
from typing import Union, Dict
from onionrtypes import UserID
from onionrutils import stringvalidators
from onionrutils import bytesconverter
from onionrutils import epoch
from utils import reconstructhash
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
class UploadSession:
"""Manage statistics for an Onionr block upload session.
accept a block hash (incl. unpadded) as an argument
"""
def __init__(self, block_hash: Union[str, bytes]):
block_hash = bytesconverter.bytes_to_str(block_hash)
block_hash = reconstructhash.reconstruct_hash(block_hash)
if not stringvalidators.validate_hash(block_hash):
raise ValueError
self.start_time = epoch.get_epoch()
self.block_hash = reconstructhash.deconstruct_hash(block_hash)
self.total_fail_count: int = 0
self.total_success_count: int = 0
self.peer_fails: Dict[UserID, int] = {}
self.peer_exists: Dict[UserID, bool] = {}
def fail_peer(self, peer):
try:
self.peer_fails[peer] += 1
except KeyError:
self.peer_fails[peer] = 0
def fail(self):
self.total_fail_count += 1
def success(self):
self.total_success_count += 1

127
src/communicatorutils/uploadblocks/sessionmanager.py
View File

@ -1,127 +0,0 @@
"""Onionr - Private P2P Communication.
Manager for upload 'sessions'
"""
from typing import List, Union, TYPE_CHECKING
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
from session import UploadSession
from onionrutils import bytesconverter
from etc import onionrvalues
from utils import reconstructhash
from . import session
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
class BlockUploadSessionManager:
"""Holds block UploadSession instances.
Optionally accepts iterable of sessions to added on init
Arguments: old_session: iterable of old UploadSession objects
"""
def __init__(self, old_sessions: List = None):
if old_sessions is None:
self.sessions = []
else:
self.sessions = old_sessions
def add_session(self,
session_or_block: Union[str,
bytes,
session.UploadSession
]
) -> session.UploadSession:
"""Create (or add existing) block upload session.
from a str/bytes block hex hash, existing UploadSession
"""
if isinstance(session_or_block, session.UploadSession):
if session_or_block not in self.sessions:
self.sessions.append(session_or_block)
return session_or_block
try:
return self.get_session(session_or_block)
except KeyError:
pass
# convert bytes hash to str
if isinstance(session_or_block, bytes):
session_or_block = bytesconverter.bytes_to_str(session_or_block)
# intentionally not elif
if isinstance(session_or_block, str):
new_session = session.UploadSession(session_or_block)
self.sessions.append(new_session)
return new_session
raise ValueError
def get_session(self,
block_hash: Union[str, bytes]
) -> session.UploadSession:
block_hash = reconstructhash.deconstruct_hash(
bytesconverter.bytes_to_str(block_hash))
for sess in self.sessions:
if sess.block_hash == block_hash:
return sess
raise KeyError
def clean_session(self,
specific_session: Union[str, 'UploadSession'] = None):
comm_inst: 'OnionrCommunicatorDaemon' # type: ignore
comm_inst = self._too_many.get_by_string( # pylint: disable=E1101 type: ignore
"OnionrCommunicatorDaemon")
kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string(
"DeadSimpleKV")
sessions_to_delete = []
if kv.get('startTime') < 120:
return
onlinePeerCount = len(kv.get('onlinePeers'))
# If we have no online peers right now,
if onlinePeerCount == 0:
return
for sess in self.sessions:
# if over 50% of peers that were online for a session have
# become unavailable, don't kill sessions
if sess.total_success_count > onlinePeerCount:
if onlinePeerCount / sess.total_success_count >= 0.5:
return
# Clean sessions if they have uploaded to enough online peers
if sess.total_success_count <= 0:
continue
if (sess.total_success_count / onlinePeerCount) >= \
onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
sessions_to_delete.append(sess)
for sess in sessions_to_delete:
try:
self.sessions.remove(session)
except ValueError:
pass
# TODO cleanup to one round of search
# Remove the blocks from the sessions, upload list,
# and waitforshare list
try:
kv.get('blocksToUpload').remove(
reconstructhash.reconstruct_hash(sess.block_hash))
except ValueError:
pass
try:
kv.get('blocksToUpload').remove(sess.block_hash)
except ValueError:
pass

2
src/coredb/keydb/__init__.py
View File

@ -1 +1 @@
from . import addkeys, listkeys, removekeys, userinfo, transportinfo from . import addkeys, listkeys, removekeys, userinfo

36
src/coredb/keydb/addkeys.py
View File

@ -25,9 +25,9 @@ from etc import onionrvalues
""" """
def add_peer(peerID, name=''): def add_pub_key(peerID, name=''):
"""Add a public key to the key database (misleading function name).""" """Add a public key to the key database (misleading function name)."""
if peerID in listkeys.list_peers() or peerID == onionrcrypto.pub_key: if peerID in listkeys.list_pub_keys() or peerID == onionrcrypto.pub_key:
raise ValueError("specified id is already known") raise ValueError("specified id is already known")
# This function simply adds a peer to the DB # This function simply adds a peer to the DB
@ -54,35 +54,3 @@ def add_peer(peerID, name=''):
return True return True
def add_address(address):
"""Add an address to the address database (only tor currently)"""
if type(address) is None or len(address) == 0:
return False
if stringvalidators.validate_transport(address):
if address in gettransports.get():
return False
conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
# check if address is in database
# this is safe to do because the address is validated above, but we strip some chars here too just in case
address = address.replace('\'', '').replace(';', '').replace('"', '').replace('\\', '')
for i in c.execute("SELECT * FROM adders WHERE address = ?;", (address,)):
try:
if i[0] == address:
conn.close()
return False
except ValueError:
pass
except IndexError:
pass
t = (address, 1)
c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t)
conn.commit()
conn.close()
return True
else:
return False

30
src/coredb/keydb/listkeys.py
View File

@ -22,8 +22,9 @@ import logger
from onionrutils import epoch from onionrutils import epoch
from etc import onionrvalues from etc import onionrvalues
from .. import dbfiles from .. import dbfiles
from . import userinfo, transportinfo from . import userinfo
def list_peers(randomOrder=True, getPow=False, trust=0):
def list_pub_keys(randomOrder=True, getPow=False, trust=0):
''' '''
Return a list of public keys (misleading function name) Return a list of public keys (misleading function name)
@ -59,28 +60,3 @@ def list_peers(randomOrder=True, getPow=False, trust=0):
conn.close() conn.close()
return peerList return peerList
def list_adders(randomOrder=True, i2p=True, recent=0):
'''
Return a list of transport addresses
'''
conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
if randomOrder:
addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();')
else:
addresses = c.execute('SELECT * FROM adders;')
addressList = []
for i in addresses:
if len(i[0].strip()) == 0:
continue
addressList.append(i[0])
conn.close()
testList = list(addressList) # create new list to iterate
for address in testList:
try:
if recent > 0 and (epoch.get_epoch() - transportinfo.get_address_info(address, 'lastConnect')) > recent:
raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0
except TypeError:
addressList.remove(address)
return addressList

16
src/coredb/keydb/removekeys.py
View File

@ -24,23 +24,7 @@ from onionrutils import mnemonickeys
from .. import dbfiles from .. import dbfiles
from etc import onionrvalues from etc import onionrvalues
def remove_address(address):
'''
Remove an address from the address database
'''
if stringvalidators.validate_transport(address):
conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
t = (address,)
c.execute('Delete from adders where address=?;', t)
conn.commit()
conn.close()
#events.event('address_remove', data = {'address': address}, onionr = core_inst.onionrInst)
return True
else:
return False
def remove_user(pubkey: str)->bool: def remove_user(pubkey: str)->bool:
''' '''

85
src/coredb/keydb/transportinfo.py
View File

@ -1,85 +0,0 @@
"""Onionr - Private P2P Communication.
get or set transport address meta information
"""
import sqlite3
from .. import dbfiles
from etc import onionrvalues
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
info_numbers = {
'address': 0,
'type': 1,
'knownPeer': 2,
'speed': 3,
'success': 4,
'powValue': 5,
'failure': 6,
'lastConnect': 7,
'trust': 8,
'introduced': 9}
def get_address_info(address, info):
"""Get info about an address from its database entry.
address text, 0
type int, 1
knownPeer text, 2
speed int, 3
success int, 4
powValue 5
failure int 6
lastConnect 7
trust 8
introduced 9
"""
conn = sqlite3.connect(
dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
command = (address,)
info = info_numbers[info]
iter_count = 0
retVal = ''
for row in c.execute('SELECT * FROM adders WHERE address=?;', command):
for i in row:
if iter_count == info:
retVal = i
break
else:
iter_count += 1
conn.close()
return retVal
def set_address_info(address, key, data):
"""Update an address for a key."""
conn = sqlite3.connect(
dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
command = (data, address)
if key not in info_numbers.keys():
raise ValueError(
"Got invalid database key when setting address info, must be in whitelist")
else:
c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command)
conn.commit()
conn.close()

34
src/httpapi/__init__.py
View File

@ -1,24 +1,26 @@
""" """
Onionr - Private P2P Communication Onionr - Private P2P Communication
This file registers plugin's flask blueprints for the client http server Register plugins flask blueprints for the client http server
"""
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
""" """
import onionrplugins import onionrplugins
import config import config
from .fdsafehandler import FDSafeHandler
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def load_plugin_blueprints(flaskapp, blueprint: str = 'flask_blueprint'): def load_plugin_blueprints(flaskapp, blueprint: str = 'flask_blueprint'):
"""Iterate enabled plugins and load any http endpoints they have""" """Iterate enabled plugins and load any http endpoints they have"""

31
src/httpapi/miscclientapi/addpeer.py
View File

@ -1,31 +0,0 @@
"""Onionr - Private P2P Communication.
add a transport address to the db
"""
from onionrutils.stringvalidators import validate_transport
from coredb.keydb.addkeys import add_address
from coredb.keydb.listkeys import list_adders
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def add_peer(peer):
if peer in list_adders():
return "already added"
if add_address(peer):
return "success"
else:
return "failure, invalid address"

78
src/httpapi/miscclientapi/endpoints.py
View File

@ -15,14 +15,10 @@ import unpaddedbase32
from httpapi import apiutils from httpapi import apiutils
import onionrcrypto import onionrcrypto
import config import config
from netcontroller import NetController
from onionrstatistics.serializeddata import SerializedData
from onionrutils import mnemonickeys from onionrutils import mnemonickeys
from onionrutils import bytesconverter from onionrutils import bytesconverter
from etc import onionrvalues from etc import onionrvalues
from utils import reconstructhash from utils import reconstructhash
from utils.gettransports import get as get_tor
from .addpeer import add_peer
""" """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
@ -49,16 +45,6 @@ class PrivateEndpoints:
private_endpoints_bp = Blueprint('privateendpoints', __name__) private_endpoints_bp = Blueprint('privateendpoints', __name__)
self.private_endpoints_bp = private_endpoints_bp self.private_endpoints_bp = private_endpoints_bp
@private_endpoints_bp.route('/addpeer/<name>', methods=['post'])
def add_peer_endpoint(name):
result = add_peer(name)
if result == "success":
return Response("success")
else:
if "already" in result:
return Response(result, 409)
else:
return Response(result, 400)
@private_endpoints_bp.route('/www/<path:path>', endpoint='www') @private_endpoints_bp.route('/www/<path:path>', endpoint='www')
def wwwPublic(path): def wwwPublic(path):
@ -75,33 +61,13 @@ class PrivateEndpoints:
def get_is_atty(): def get_is_atty():
return Response(str(sys_stdout.isatty()).lower()) return Response(str(sys_stdout.isatty()).lower())
@private_endpoints_bp.route('/hitcount')
def get_hit_count():
return Response(str(client_api.publicAPI.hitCount))
@private_endpoints_bp.route('/ping') @private_endpoints_bp.route('/ping')
def ping(): def ping():
# Used to check if client api is working # Used to check if client api is working
return Response("pong!") return Response("pong!")
@private_endpoints_bp.route('/lastconnect')
def last_connect():
return Response(str(client_api.publicAPI.lastRequest))
@private_endpoints_bp.route('/waitforshare/<name>', methods=['post'])
def wait_for_share(name):
"""Prevent the **public** api from sharing blocks.
Used for blocks we created usually
"""
if not name.isalnum():
raise ValueError('block hash needs to be alpha numeric')
name = reconstructhash.reconstruct_hash(name)
if name in client_api.publicAPI.hideBlocks:
return Response("will be removed")
else:
client_api.publicAPI.hideBlocks.append(name)
return Response("added")
@private_endpoints_bp.route('/shutdown') @private_endpoints_bp.route('/shutdown')
def shutdown(): def shutdown():
@ -116,17 +82,6 @@ class PrivateEndpoints:
def get_hidden_blocks(): def get_hidden_blocks():
return Response('\n'.join(client_api.publicAPI.hideBlocks)) return Response('\n'.join(client_api.publicAPI.hideBlocks))
@private_endpoints_bp.route('/getstats')
def get_stats():
"""Return serialized node statistics."""
while True:
try:
return Response(client_api._too_many.get(
SerializedData).get_stats())
except AttributeError:
pass
except FileNotFoundError:
pass
@private_endpoints_bp.route('/getuptime') @private_endpoints_bp.route('/getuptime')
def show_uptime(): def show_uptime():
@ -159,32 +114,6 @@ class PrivateEndpoints:
def get_os_system(): def get_os_system():
return Response(platform.system().lower()) return Response(platform.system().lower())
@private_endpoints_bp.route('/gettorsocks')
def get_tor_socks():
while True:
try:
return Response(
str(
g.too_many.get_by_string(
'NetController').socksPort))
except KeyError:
sleep(0.1)
@private_endpoints_bp.route('/torready')
def is_tor_ready():
"""If Tor is starting up, the web UI is not ready to be used."""
try:
return Response(
str(g.too_many.get_by_string('NetController').readyState).lower())
except KeyError:
return Response("false")
@private_endpoints_bp.route('/gettoraddress')
def get_tor_address():
"""Return public Tor v3 Onion address for this node"""
if not config.get('general.security_level', 0) == 0:
abort(404)
return Response(get_tor()[0])
@private_endpoints_bp.route('/getgeneratingblocks') @private_endpoints_bp.route('/getgeneratingblocks')
def get_generating_blocks() -> Response: def get_generating_blocks() -> Response:
@ -195,10 +124,3 @@ class PrivateEndpoints:
)) ))
) )
@private_endpoints_bp.route('/getblockstoupload')
def get_blocks_to_upload() -> Response:
return Response(
','.join(
g.too_many.get_by_string('DeadSimpleKV').get('blocksToUpload')
)
)

6
src/httpapi/miscpublicapi/__init__.py
View File

@ -1,6 +0,0 @@
from . import announce, upload, getblocks, endpoints
announce = announce.handle_announce # endpoint handler for accepting peer announcements
upload = upload.accept_upload # endpoint handler for accepting public uploads
public_block_list = getblocks.get_public_block_list # endpoint handler for getting block lists
public_get_block_data = getblocks.get_block_data # endpoint handler for responding to peers requests for block data

62
src/httpapi/miscpublicapi/announce.py
View File

@ -1,62 +0,0 @@
"""Onionr - Private P2P Communication.
Handle announcements to the public API server
"""
from flask import Response, g
import deadsimplekv
import logger
from etc import onionrvalues
from onionrutils import stringvalidators, bytesconverter
import filepaths
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def handle_announce(request):
"""accept announcement posts, validating POW
clientAPI should be an instance of the clientAPI server running,
request is a instance of a flask request
"""
resp = 'failure'
newNode = ''
try:
newNode = request.form['node'].encode()
except KeyError:
logger.warn('No node specified for upload')
else:
newNode = bytesconverter.bytes_to_str(newNode)
announce_queue = deadsimplekv.DeadSimpleKV(filepaths.announce_cache)
announce_queue_list = announce_queue.get('new_peers')
if announce_queue_list is None:
announce_queue_list = []
else:
if len(announce_queue_list) >= onionrvalues.MAX_NEW_PEER_QUEUE:
newNode = ''
if stringvalidators.validate_transport(newNode) and \
newNode not in announce_queue_list:
g.shared_state.get(
deadsimplekv.DeadSimpleKV).get('newPeers').append(newNode)
announce_queue.put('new_peers',
announce_queue_list.append(newNode))
announce_queue.flush()
resp = 'Success'
resp = Response(resp)
if resp == 'failure':
return resp, 406
return resp

91
src/httpapi/miscpublicapi/endpoints.py
View File

@ -1,91 +0,0 @@
"""Onionr - Private P2P Communication.
Misc public API endpoints too small to need their own file
and that need access to the public api inst
"""
from flask import Response, Blueprint, request, send_from_directory, abort, g
from . import getblocks, upload, announce
from coredb import keydb
import config
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
class PublicEndpoints:
def __init__(self, public_api):
public_endpoints_bp = Blueprint('publicendpoints', __name__)
self.public_endpoints_bp = public_endpoints_bp
@public_endpoints_bp.route('/')
def banner():
# Display info to people who visit a node address in their browser
try:
with open('../static-data/index.html', 'r') as html:
resp = Response(html.read(), mimetype='text/html')
except FileNotFoundError:
resp = Response("")
return resp
@public_endpoints_bp.route('/getblocklist')
def get_block_list():
"""Get a list of blocks, optionally filtered by epoch time stamp,
excluding those hidden"""
return getblocks.get_public_block_list(public_api, request)
@public_endpoints_bp.route('/getdata/<name>')
def get_block_data(name):
# Share data for a block if we have it and it isn't hidden
return getblocks.get_block_data(public_api, name)
@public_endpoints_bp.route('/www/<path:path>')
def www_public(path):
# A way to share files directly over your .onion
if not config.get("www.public.run", True):
abort(403)
return send_from_directory(
config.get('www.public.path', 'static-data/www/public/'), path)
@public_endpoints_bp.route('/plaintext')
def plaintext_enabled_endpoint():
return Response(str(config.get("general.store_plaintext_blocks", True)).lower())
@public_endpoints_bp.route('/ping')
def ping():
# Endpoint to test if nodes are up
return Response("pong!")
@public_endpoints_bp.route('/pex')
def peer_exchange():
response = ','.join(keydb.listkeys.list_adders(recent=3600))
if len(response) == 0:
response = ''
return Response(response)
@public_endpoints_bp.route('/announce', methods=['post'])
def accept_announce():
"""Accept announcements with pow token to prevent spam"""
g.shared_state = public_api._too_many
resp = announce.handle_announce(request)
return resp
@public_endpoints_bp.route('/upload', methods=['post'])
def upload_endpoint():
"""Accept file uploads.
In the future this will be done more often than on creation
to speed up block sync
"""
return upload.accept_upload(request)

73
src/httpapi/miscpublicapi/getblocks.py
View File

@ -1,73 +0,0 @@
"""Onionr - Private P2P Communication.
Public endpoints to get block data and lists
"""
from flask import Response, abort
import config
from onionrutils import bytesconverter, stringvalidators
from coredb import blockmetadb
from utils import reconstructhash
from onionrblocks import BlockList
from onionrblocks.onionrblockapi import Block
from .. import apiutils
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def get_public_block_list(public_API, request):
# Provide a list of our blocks, with a date offset
date_adjust = request.args.get('date')
type_filter = request.args.get('type')
b_list = blockmetadb.get_block_list(date_rec=date_adjust)
share_list = ''
if config.get('general.hide_created_blocks', True):
for b in public_API.hideBlocks:
if b in b_list:
# Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block
b_list.remove(b)
for b in b_list:
if type_filter:
if Block(b, decrypt=False).getType() != type_filter:
continue
share_list += '%s\n' % (reconstructhash.deconstruct_hash(b),)
return Response(share_list)
def get_block_data(public_API, b_hash):
"""return block data by hash unless we are hiding it"""
resp = ''
b_hash = reconstructhash.reconstruct_hash(b_hash)
if stringvalidators.validate_hash(b_hash):
if not config.get('general.hide_created_blocks', True) \
or b_hash not in public_API.hideBlocks:
if b_hash in public_API._too_many.get(BlockList).get():
block = apiutils.GetBlockData().get_block_data(
b_hash, raw=True, decrypt=False)
try:
# Encode in case data is binary
block = block.encode('utf-8')
except AttributeError:
# 404 if no block data
if not block:
abort(404)
if not len(block):
abort(404)
resp = block
if len(resp) == 0:
abort(404)
resp = ""
# Has to be octet stream, otherwise binary data fails hash check
return Response(resp, mimetype='application/octet-stream')

94
src/httpapi/miscpublicapi/upload.py
View File

@ -1,94 +0,0 @@
"""Onionr - Private P2P Communication.
Accept block uploads to the public API server
"""
import sys
from gevent import spawn
from flask import Response
from flask import abort
from flask import g
from onionrutils import localcommand
from onionrblocks import blockimporter
import onionrexceptions
import logger
import config
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def accept_upload(request):
"""Accept uploaded blocks to our public Onionr protocol API server"""
resp = 'failure'
data = request.get_data()
data_size = sys.getsizeof(data)
b_hash = None
if data_size < 30:
resp = 'size'
elif data_size < 100000000:
try:
b_hash = blockimporter.import_block_from_data(data)
if b_hash:
# Upload mixing is where a node will hide and reupload a block
# to act like it is also a creator
# This adds deniability but is very slow
if g.too_many.get_by_string(
"DeadSimpleKV").get('onlinePeers') and \
config.get('general.upload_mixing', False):
spawn(
localcommand.local_command,
'/daemon-event/upload_event',
post=True,
is_json=True,
post_data={'block': b_hash}
).get(timeout=10)
resp = 'success'
else:
resp = 'failure'
logger.warn(
f'Error encountered importing uploaded block {b_hash}')
except onionrexceptions.BlacklistedBlock:
logger.debug('uploaded block is blacklisted')
resp = 'failure'
except onionrexceptions.InvalidProof:
resp = 'proof'
except onionrexceptions.DataExists:
resp = 'exists'
except onionrexceptions.PlaintextNotSupported:
logger.debug(f"attempted plaintext upload to us: {b_hash}")
resp = 'failure'
except onionrexceptions.InvalidMetadata:
logger.debug(
f'uploaded block {b_hash} has invalid metadata')
resp = 'failure'
if resp == 'failure':
abort(400)
elif resp == 'size':
resp = Response(resp, 400)
logger.warn(
f'Error importing uploaded block, invalid size {b_hash}')
elif resp == 'proof':
resp = Response(resp, 400)
if b_hash:
logger.warn(
f'Error importing uploaded block, invalid proof {b_hash}')
else:
logger.warn(
'Error importing uploaded block, invalid proof')
else:
resp = Response(resp)
return resp

10
src/httpapi/sse/private/__init__.py
View File

@ -13,7 +13,6 @@ from onionrblocks.onionrblockapi import Block
from coredb.dbfiles import block_meta_db from coredb.dbfiles import block_meta_db
from coredb.blockmetadb import get_block_list from coredb.blockmetadb import get_block_list
from onionrutils.epoch import get_epoch from onionrutils.epoch import get_epoch
from onionrstatistics.transports.tor import TorStats
from .. import wrapper from .. import wrapper
""" """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
@ -44,15 +43,6 @@ def stream_hello():
return SSEWrapper.handle_sse_request(print_hello) return SSEWrapper.handle_sse_request(print_hello)
@private_sse_blueprint.route('/torcircuits')
def stream_tor_circuits():
tor_stats = g.too_many.get(TorStats)
def circuit_stat_stream():
while True:
yield "data: " + tor_stats.get_json() + "\n\n"
sleep(10)
return SSEWrapper.handle_sse_request(circuit_stat_stream)
@private_sse_blueprint.route('/recentblocks') @private_sse_blueprint.route('/recentblocks')
def stream_recent_blocks(): def stream_recent_blocks():
def _compile_json(b_list): def _compile_json(b_list):

37
src/lan/__init__.py
View File

@ -1,37 +0,0 @@
"""Onionr - Private P2P Communication.
LAN manager
"""
from typing import TYPE_CHECKING
from threading import Thread
if TYPE_CHECKING:
from toomanyobjs import TooMany
from .discover import learn_services, advertise_service
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
class LANManager:
"""Initialize and start/top LAN transport threads."""
def __init__(self, too_many: "TooMany"):
self.too_many = too_many
self.peers: "exploded IP Address string" = []
def start(self):
Thread(target=learn_services, daemon=True).start()
Thread(target=advertise_service, daemon=True).start()

76
src/lan/client/__init__.py
View File

@ -1,76 +0,0 @@
"""Onionr - Private P2P Communication.
LAN transport client thread
"""
import requests
from typing import Set
from onionrtypes import LANIP
import logger
from coredb.blockmetadb import get_block_list
from onionrblocks.blockimporter import import_block_from_data
import onionrexceptions
from ..server import ports
from onionrproofs import hashMeetsDifficulty
from threading import Thread
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
connected_lan_peers: Set[LANIP] = set([])
def _lan_work(peer: LANIP):
def _sync_peer(url):
our_blocks = get_block_list()
blocks = requests.get(url + 'blist/0').text.splitlines()
for block in blocks:
if block not in our_blocks and hashMeetsDifficulty(block):
try:
import_block_from_data(
requests.get(
url + f'get/{block}', stream=True).raw.read(6000000))
except onionrexceptions.InvalidMetadata:
logger.warn(f"Could not get {block} from lan peer")
except onionrexceptions.InvalidProof:
logger.warn(
f"Invalid proof for {block} from lan peer {peer}", terminal=True)
break
for port in ports:
try:
root = f'http://{peer}:{port}/'
if requests.get(f'{root}ping').text != 'onionr!':
connected_lan_peers.remove(peer)
else:
logger.info(f'[LAN] Connected to {peer}:{port}', terminal=True)
while True:
try:
_sync_peer(root)
except requests.exceptions.ConnectionError:
break
break
except requests.exceptions.ConnectionError:
pass
else:
connected_lan_peers.remove(peer)
def connect_peer(peer: LANIP):
if peer not in connected_lan_peers:
connected_lan_peers.add(peer)
Thread(target=_lan_work, args=[peer], daemon=True).start()

83
src/lan/discover.py
View File

@ -1,83 +0,0 @@
"""Onionr - Private P2P Communication.
Discover and publish private-network
"""
import socket
import struct
from ipaddress import ip_address
from .getip import lan_ips, best_ip
from utils.bettersleep import better_sleep
from . import client
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
MCAST_GRP = '224.0.0.112'
MCAST_PORT = 1337
IS_ALL_GROUPS = True
ANNOUNCE_LOOP_SLEEP = 30
def learn_services():
"""Take a list to infintely add lan service info to."""
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if IS_ALL_GROUPS:
# on this port, receives ALL multicast groups
sock.bind(('', MCAST_PORT))
else:
# on this port, listen ONLY to MCAST_GRP
sock.bind((MCAST_GRP, MCAST_PORT))
mreq = struct.pack("4sl", socket.inet_aton(MCAST_GRP), socket.INADDR_ANY)
sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
while True:
service_ips = sock.recv(200).decode('utf-8')
if 'onionr' not in service_ips:
continue
service_ips = service_ips.replace('onionr-', '').split('-')
for service in service_ips:
try:
ip_address(service)
if not ip_address(service).is_private:
raise ValueError
if service in lan_ips:
raise ValueError
except ValueError:
pass
else:
client.connect_peer(service)
def advertise_service(specific_ips=None):
# regarding socket.IP_MULTICAST_TTL
# ---------------------------------
# for all packets sent, after three hops on the network the packet will not
# be re-sent/broadcast
# (see https://www.tldp.org/HOWTO/Multicast-HOWTO-6.html)
MULTICAST_TTL = 3
ips = best_ip
if not ips:
return
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, MULTICAST_TTL)
while True:
sock.sendto(f"onionr-{ips}".encode('utf-8'), (MCAST_GRP, MCAST_PORT))
better_sleep(ANNOUNCE_LOOP_SLEEP)

51
src/lan/getip.py
View File

@ -1,51 +0,0 @@
"""Onionr - Private P2P Communication.
Identify LAN ip addresses and determine the best one
"""
from ipaddress import IPv4Address
from psutil import net_if_addrs
from socket import AF_INET
import logger
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
lan_ips = []
# https://psutil.readthedocs.io/en/latest/#psutil.net_if_addrs
def _get_lan_ips():
for interface in net_if_addrs().keys():
for address in net_if_addrs()[interface]:
# Don't see benefit in ipv6, so just check for v4 addresses
if address[0] == AF_INET:
# Mark the address for use in LAN if it is a private address
if IPv4Address(address[1]).is_private and not IPv4Address(address[1]).is_loopback:
lan_ips.append(address[1])
try:
_get_lan_ips()
except OSError:
logger.warn("Could not identify LAN ips due to OSError.")
# These are more likely to be actual local subnets rather than VPNs
for ip in lan_ips:
if '192.168' in ip:
best_ip = ip
break
else:
try:
best_ip = lan_ips[0]
except IndexError:
best_ip = ""

111
src/lan/server/__init__.py
View File

@ -1,111 +0,0 @@
"""Onionr - Private P2P Communication.
LAN transport server thread
"""
import ipaddress
import time
from threading import Thread
from gevent.pywsgi import WSGIServer
from flask import Flask
from flask import Response
from flask import request
from flask import abort
from onionrblocks.onionrblockapi import Block
from httpapi.fdsafehandler import FDSafeHandler
from netcontroller import get_open_port
import config
from coredb.blockmetadb import get_block_list
from lan.getip import best_ip, lan_ips
from onionrutils import stringvalidators
from httpapi.miscpublicapi.upload import accept_upload
import logger
from utils.bettersleep import better_sleep
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
ports = range(1337, 1340)
_start_time = time.time()
class LANServer:
def __init__(self, shared_state):
app = Flask(__name__)
self.app = app
self.host = config.get('lan.bind_ip', '')
self.server = None
if self.host == '':
self.host = best_ip
self.port = None
@app.before_request
def dns_rebinding_prevention():
if request.remote_addr in lan_ips or \
ipaddress.ip_address(request.remote_addr).is_loopback:
if time.time() - _start_time > 600:
abort(403)
if request.host != f'{self.host}:{self.port}':
logger.warn('Potential DNS rebinding attack on LAN server:')
logger.warn(
f'Hostname {request.host} was used instead of {self.host}:{self.port}') # noqa
abort(403)
@app.route('/blist/<time>')
def get_block_list_for_lan(time):
return Response('\n'.join(get_block_list(date_rec=time)))
@app.route('/get/<block>')
def get_block_data(block):
if not stringvalidators.validate_hash(block):
raise ValueError
return Response(
Block(block).raw, mimetype='application/octet-stream')
@app.route("/ping")
def ping():
return Response("onionr!")
@app.route('/upload', methods=['POST'])
def upload_endpoint():
return accept_upload(request)
def start_server(self):
def _show_lan_bind(port):
better_sleep(1)
if self.server.started and port == self.server.server_port:
logger.info(
f'Serving to LAN on {self.host}:{self.port}',
terminal=True)
if self.host == "":
logger.info(
"Not binding to LAN due to no private network configured.",
terminal=True)
return
for i in ports:
self.server = WSGIServer((self.host, i),
self.app, log=None,
handler_class=FDSafeHandler)
self.port = self.server.server_port
try:
Thread(target=_show_lan_bind, args=[i], daemon=True).start()
self.server.serve_forever()
except OSError:
pass
else:
break
else:
logger.warn("Could not bind to any LAN ports " +
str(min(ports)) + "-" + str(max(ports)), terminal=True)
return

18
src/netcontroller/__init__.py
View File

@ -1,7 +1,11 @@
from . import getopenport, torcontrol from . import getopenport
from . import torcontrol import os
from . import cleanephemeral from shutil import which
tor_binary = torcontrol.torbinary.tor_binary
get_open_port = getopenport.get_open_port
NetController = torcontrol.NetController def tor_binary():
clean_ephemeral_services = cleanephemeral.clean_ephemeral_services """Return tor binary path or none if not exists"""
tor_path = './tor'
if not os.path.exists(tor_path):
tor_path = which('tor')
return tor_path

40
src/netcontroller/cleanephemeral.py
View File

@ -1,40 +0,0 @@
"""Onionr - Private P2P Communication.
Remove ephemeral services
"""
from filenuke.nuke import clean
from onionrutils.stringvalidators import validate_transport
from filepaths import ephemeral_services_file
from netcontroller.torcontrol.torcontroller import get_controller
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def clean_ephemeral_services():
"""Remove transport's ephemeral services from respective controllers"""
try:
with open(ephemeral_services_file, 'r') as services:
services = services.readlines()
with get_controller() as torcontroller:
for hs in services:
hs += '.onion'
if validate_transport(hs):
torcontroller.remove_ephemeral_hidden_service(hs)
except FileNotFoundError:
pass
else:
clean(ephemeral_services_file)

171
src/netcontroller/torcontrol/__init__.py
View File

@ -1,171 +0,0 @@
"""Onionr - Private P2P Communication.
Netcontroller library, used to control/work with Tor and send requests through
"""
import os
import subprocess
import signal
import time
import multiprocessing
from onionrtypes import BooleanSuccessState
import logger
from .. import getopenport
from .. import watchdog
from . import customtorrc
from . import gentorrc
from . import addbridges
from . import torbinary
from utils import identifyhome
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
TOR_KILL_WAIT = 3
addbridges = addbridges.add_bridges
class NetController:
"""Handle Tor daemon and onion service setup on Tor."""
def __init__(self, hsPort, apiServerIP='127.0.0.1'):
# set data dir
self.dataDir = identifyhome.identify_home()
self.socksPort = getopenport.get_open_port()
self.torConfigLocation = self.dataDir + 'torrc'
self.readyState = False
self.hsPort = hsPort
self._torInstnace = ''
self.myID = ''
self.apiServerIP = apiServerIP
self.torBinary = torbinary.tor_binary()
def startTor(self, gen_torrc=True) -> BooleanSuccessState:
"""
Start Tor with onion service on port 80 & socks proxy on random port
"""
if gen_torrc:
gentorrc.generate_torrc(self, self.apiServerIP)
if os.path.exists('./tor'):
self.torBinary = './tor'
elif os.path.exists('/usr/bin/tor'):
self.torBinary = '/usr/bin/tor'
else:
self.torBinary = 'tor'
try:
tor = subprocess.Popen([self.torBinary, '-f', self.torConfigLocation], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except FileNotFoundError:
logger.fatal("Tor was not found in your path or the Onionr directory. Please install Tor and try again.", terminal=True)
return False
else:
# Test Tor Version
torVersion = subprocess.Popen([self.torBinary, '--version'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
for line in iter(torVersion.stdout.readline, b''):
if 'Tor 0.2.' in line.decode():
logger.fatal('Tor 0.3+ required', terminal=True)
return False
torVersion.kill()
# wait for tor to get to 100% bootstrap
try:
for line in iter(tor.stdout.readline, b''):
for word in ('bootstrapped', '%'):
if word not in line.decode().lower():
break
else:
if '100' not in line.decode():
logger.info(line.decode().strip(), terminal=True)
if 'bootstrapped 100' in line.decode().lower():
logger.info(line.decode(), terminal=True)
break
elif 'opening socks listener' in line.decode().lower():
logger.debug(line.decode().replace('\n', ''))
else:
if 'err' in line.decode():
logger.error(
line.decode().replace('\n', ''), terminal=True)
elif 'warn' in line.decode():
logger.warn(
line.decode().replace('\n', ''), terminal=True)
else:
logger.debug(line.decode().replace('\n', ''))
else:
logger.fatal('Failed to start Tor. Maybe a stray instance of Tor used by Onionr is still running? This can also be a result of file permissions being too open', terminal=True)
return False
except KeyboardInterrupt:
logger.fatal('Got keyboard interrupt. Onionr will exit soon.', timestamp = False, terminal=True)
return False
try:
myID = open(self.dataDir + 'hs/hostname', 'r')
self.myID = myID.read().replace('\n', '')
myID.close()
except FileNotFoundError:
self.myID = ""
with open(self.dataDir + 'torPid.txt', 'w') as tor_pid_file:
tor_pid_file.write(str(tor.pid))
#multiprocessing.Process(target=watchdog.watchdog,
# args=[os.getpid(), tor.pid], daemon=True).start()
logger.info('Finished starting Tor.', terminal=True)
self.readyState = True
return True
def killTor(self):
"""Properly kill tor based on pid saved to file."""
try:
with open(self.dataDir + 'torPid.txt', 'r') as torPid:
pidN = torPid.read()
except FileNotFoundError:
return
try:
try:
# Extra int()
os.kill(int(pidN), signal.SIGTERM)
except PermissionError:
# seems to happen on win 10
pass
except ValueError:
# Happens if int() check is not valid
logger.error("torPid.txt contained invalid integer. " +
"This indicates corruption " +
"and should not be bypassed for security reasons")
return
os.remove(self.dataDir + 'torPid.txt')
except ProcessLookupError:
pass
except FileNotFoundError:
pass
try:
time.sleep(TOR_KILL_WAIT)
except KeyboardInterrupt:
pass
try:
os.kill(int(pidN), signal.SIGKILL)
except (ProcessLookupError, PermissionError):
pass
try:
os.remove(self.dataDir + 'tordata/lock')
except FileNotFoundError:
pass

36
src/netcontroller/torcontrol/addbridges.py
View File

@ -1,36 +0,0 @@
"""Onionr - Private P2P Communication.
Add bridge info to torrc configuration string
"""
import config
import logger
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def add_bridges(torrc: str) -> str:
"""Configure tor to use a bridge using Onionr config keys."""
config.reload()
if config.get('tor.use_bridge', False) is True:
bridge = config.get('tor.bridge_ip', None)
if bridge is not None:
# allow blank fingerprint purposefully
fingerprint = config.get('tor.bridge_fingerprint', '')
torrc += '\nUseBridges 1\nBridge %s %s\n' % (bridge, fingerprint)
if not bridge:
logger.error('Bridge was enabled but not specified in config, ' +
'this probably won\'t work', terminal=True)
return torrc

44
src/netcontroller/torcontrol/customtorrc.py
View File

@ -1,44 +0,0 @@
"""Onionr - Private P2P Communication.
Load or set custom torrc
"""
from utils import identifyhome
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
CUSTOM_TORRC_FILE = identifyhome.identify_home() + '/torrc-custom'
def set_custom_torrc(torrc_data: str):
"""write torrc_data to custom torrc file stored in home dir.
if set it will be used in addition to onionr's generated settings
"""
torrc_comment = f'\n# BEGIN CUSTOM TORRC FROM {CUSTOM_TORRC_FILE}\n'
torrc_data = torrc_comment + torrc_data
with open(CUSTOM_TORRC_FILE, 'w') as torrc:
torrc.write(torrc_data)
def get_custom_torrc() -> str:
"""read torrc_data from custom torrc file stored in home dir.
if set it will be used in addition to onionr's generated settings
"""
torrc = ''
try:
with open(CUSTOM_TORRC_FILE, 'r') as torrc:
torrc = torrc.read()
except FileNotFoundError:
pass
return '\n' + torrc

93
src/netcontroller/torcontrol/gentorrc.py
View File

@ -1,93 +0,0 @@
"""Onionr - Private P2P Communication.
Generate a generate a torrc file for our Onionr instance
"""
import base64
import os
import subprocess
from typing import TYPE_CHECKING
from .. import getopenport
from . import customtorrc
from . import addbridges
from . import torbinary
from utils import identifyhome
import config
if TYPE_CHECKING:
from netcontroller import NetController
from onionrtypes import LoopBackIP
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
add_bridges = addbridges.add_bridges
def generate_torrc(net_controller: 'NetController',
api_server_ip: 'LoopBackIP'):
"""Generate a torrc file for our tor instance."""
socks_port = net_controller.socksPort
hs_port = net_controller.hsPort
home_dir = identifyhome.identify_home()
tor_config_location = home_dir + '/torrc'
hs_ver = 'HiddenServiceVersion 3'
"""
Set the Tor control password.
Meant to make it harder to manipulate our Tor instance
"""
plaintext = base64.b85encode(
os.urandom(50)).decode()
config.set('tor.controlpassword', plaintext, savefile=True)
config.set('tor.socksport', socks_port, savefile=True)
control_port = getopenport.get_open_port()
config.set('tor.controlPort', control_port, savefile=True)
hashedPassword = subprocess.Popen([torbinary.tor_binary(),
'--hash-password',
plaintext],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
for line in iter(hashedPassword.stdout.readline, b''):
password = line.decode()
if 'warn' not in password:
break
torrc_data = """SocksPort """ + str(socks_port) + """ OnionTrafficOnly
DataDirectory """ + home_dir + """tordata/
CookieAuthentication 1
KeepalivePeriod 40
CircuitsAvailableTimeout 86400
ControlPort """ + str(control_port) + """
HashedControlPassword """ + str(password) + """
"""
if config.get('general.security_level', 1) == 0:
torrc_data += """\nHiddenServiceDir """ + home_dir + """hs/
\n""" + hs_ver + """\n
HiddenServiceNumIntroductionPoints 20
HiddenServiceMaxStreams 500
HiddenServiceMaxStreamsCloseCircuit 1
HiddenServicePort 80 """ + api_server_ip + """:""" + str(hs_port)
torrc_data = add_bridges(torrc_data)
torrc_data += customtorrc.get_custom_torrc()
torrc = open(tor_config_location, 'w')
torrc.write(torrc_data)
torrc.close()

43
src/netcontroller/torcontrol/onionservicecreator.py
View File

@ -1,43 +0,0 @@
"""Onionr - Private P2P Communication.
Create an ephemeral onion service
"""
import stem
from .torcontroller import get_controller
from filepaths import ephemeral_services_file
import logger
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def create_onion_service(port=80, record_to_service_removal_file=True):
try:
controller = get_controller()
except stem.SocketError:
logger.error("Could not connect to Tor control")
raise
hs = controller.create_ephemeral_hidden_service(
{80: port},
key_type='NEW',
key_content='ED25519-V3',
await_publication=True,
detached=True)
if record_to_service_removal_file:
with open(ephemeral_services_file, 'a') as service_file:
service_file.write(hs.service_id + '\n')
return (hs.service_id, hs.private_key)

35
src/netcontroller/torcontrol/rebuildtor.py
View File

@ -1,35 +0,0 @@
"""Onionr - P2P Anonymous Storage Network.
Send Tor restart command
"""
from gevent import spawn
from onionrutils import localcommand
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def rebuild():
"""Send Tor restart command"""
spawn(
localcommand.local_command,
f'/daemon-event/restart_tor',
post=True,
is_json=True,
post_data={}
).get(10)
rebuild.onionr_help = "If Onionr is running and is managing its own Tor daemon, restart that daemon."

11
src/netcontroller/torcontrol/togglenetwork.py
View File

@ -1,11 +0,0 @@
from . import torcontroller
def enable_tor_network_connection():
with torcontroller.get_controller() as controller:
c.set_conf("DisableNetwork", 0)
def disable_tor_network_connection():
with torcontroller.get_controller() as controller:
c.set_conf("DisableNetwork", 1)

28
src/netcontroller/torcontrol/torbinary.py
View File

@ -1,28 +0,0 @@
"""Onionr - Private P2P Communication.
get the tor binary path
"""
import os
from shutil import which
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def tor_binary():
"""Return tor binary path or none if not exists"""
tor_path = './tor'
if not os.path.exists(tor_path):
tor_path = which('tor')
return tor_path

34
src/netcontroller/torcontrol/torcontroller.py
View File

@ -1,34 +0,0 @@
"""Onionr - P2P Anonymous Storage Network.
Return stem Tor controller instance
"""
from stem.control import Controller
import config
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
config.reload()
def get_controller() -> Controller:
"""Create and return a Tor controller connection."""
port = config.get('tor.controlPort', 0)
password = config.get('tor.controlpassword', '')
if config.get('tor.use_existing_tor', False):
port = config.get('tor.existing_control_port', 0)
password = config.get('tor.existing_control_password', '')
c = Controller.from_port(port=port)
c.authenticate(password)
return c

44
src/netcontroller/watchdog.py
View File

@ -1,44 +0,0 @@
"""
Onionr - Private P2P Communication
Watch 1 process, then terminate second safely
"""
import time
import os
import psutil
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def watchdog(parent_proc, child_proc):
"""watch for proc1 to die, then kill proc2"""
try:
if os.forkpty() != 0:
return
except AttributeError:
pass
parent_proc = psutil.Process(parent_proc)
child_proc = psutil.Process(child_proc)
while parent_proc.is_running():
time.sleep(10)
if child_proc.is_running():
child_proc.terminate()

124
src/onionrcommands/daemonlaunch/__init__.py
View File

@ -16,14 +16,11 @@ from deadsimplekv import DeadSimpleKV
import psutil import psutil
import config import config
import onionrstatistics
from onionrstatistics import serializeddata
import apiservers import apiservers
import logger import logger
import communicator
from onionrplugins import onionrevents as events from onionrplugins import onionrevents as events
from netcontroller import NetController
from netcontroller import clean_ephemeral_services
from onionrutils import localcommand from onionrutils import localcommand
from utils import identifyhome from utils import identifyhome
import filepaths import filepaths
@ -35,29 +32,24 @@ from httpapi import daemoneventsapi
from .. import version from .. import version
from .getapihost import get_api_host_until_available from .getapihost import get_api_host_until_available
from utils.bettersleep import better_sleep from utils.bettersleep import better_sleep
from netcontroller.torcontrol.onionservicecreator import create_onion_service
from .killdaemon import kill_daemon # noqa from .killdaemon import kill_daemon # noqa
from .showlogo import show_logo from .showlogo import show_logo
from lan import LANManager
from lan.server import LANServer
from sneakernet import sneakernet_import_thread from sneakernet import sneakernet_import_thread
from onionrstatistics.devreporting import statistics_reporter
from setupkvvars import setup_kv from setupkvvars import setup_kv
from communicatorutils.housekeeping import clean_blocks_not_meeting_pow
from .spawndaemonthreads import spawn_client_threads
""" """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or the Free Software Foundation, either version 3 of the License, or
(at your option) any later version. (at your option) any later version.
This program is distributed in the hope that it will be useful, This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details. GNU General Public License for more details.
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
""" """
@ -78,53 +70,6 @@ def _show_info_messages():
getourkeypair.get_keypair()[0][:52])) getourkeypair.get_keypair()[0][:52]))
def _setup_online_mode(
use_existing_tor: bool,
net: NetController,
security_level: int):
if config.get('transports.tor', True):
# If we are using tor, check if we are using an existing tor instance
# if we are, we need to create an onion service on it and set attrs on our NetController
# if not, we need to tell netcontroller to start one
if use_existing_tor:
try:
os.mkdir(filepaths.tor_hs_loc)
except FileExistsError:
pass
net.socksPort = config.get('tor.existing_socks_port')
try:
net.myID = create_onion_service(
port=net.apiServerIP + ':' + str(net.hsPort))[0]
except stem.SocketError:
logger.error(
"Could not connect to existing Tor service", terminal=True)
localcommand.local_command('shutdown')
cleanup.delete_run_files()
sys.exit(1)
except IncorrectPassword:
# Exit if we cannot connect to the existing Tor instance
logger.error('Invalid Tor control password', terminal=True)
localcommand.local_command('shutdown')
cleanup.delete_run_files()
sys.exit(1)
if not net.myID.endswith('.onion'):
net.myID += '.onion'
with open(filepaths.tor_hs_address_file, 'w') as tor_file:
tor_file.write(net.myID)
else:
logger.info('Tor is starting...', terminal=True)
if not net.startTor():
# Exit if we cannot start Tor.
localcommand.local_command('shutdown')
cleanup.delete_run_files()
sys.exit(1)
if len(net.myID) > 0 and security_level == 0:
logger.debug(
'Started .onion service: %s' %
(logger.colors.underline + net.myID))
else:
logger.debug('.onion service disabled')
def daemon(): def daemon():
@ -167,18 +112,13 @@ def daemon():
Thread(target=shared_state.get(apiservers.ClientAPI).start, Thread(target=shared_state.get(apiservers.ClientAPI).start,
daemon=True, name='client HTTP API').start() daemon=True, name='client HTTP API').start()
if not offline_mode:
Thread(target=shared_state.get(apiservers.PublicAPI).start,
daemon=True, name='public HTTP API').start()
# Init run time tester # Init run time tester
# (ensures Onionr is running right, for testing purposes) # (ensures Onionr is running right, for testing purposes)
# Run time tests are not normally run # Run time tests are not normally run
shared_state.get(runtests.OnionrRunTestManager) shared_state.get(runtests.OnionrRunTestManager)
# Create singleton
shared_state.get(serializeddata.SerializedData)
shared_state.share_object() # share the parent object to the threads shared_state.share_object() # share the parent object to the threads
show_logo() show_logo()
@ -189,54 +129,22 @@ def daemon():
if not offline_mode: if not offline_mode:
apiHost = get_api_host_until_available() apiHost = get_api_host_until_available()
net = NetController(config.get('client.public.port', 59497),
apiServerIP=apiHost)
shared_state.add(net)
shared_state.get(onionrstatistics.tor.TorStats)
security_level = config.get('general.security_level', 1) security_level = config.get('general.security_level', 1)
use_existing_tor = config.get('tor.use_existing_tor', False) use_existing_tor = config.get('tor.use_existing_tor', False)
if not offline_mode:
# we need to setup tor for use
_setup_online_mode(use_existing_tor, net, security_level)
_show_info_messages() _show_info_messages()
logger.info( logger.info(
"Onionr daemon is running under " + str(os.getpid()), terminal=True) "Onionr daemon is running under " + str(os.getpid()), terminal=True)
events.event('init', threaded=False) events.event('init', threaded=False)
events.event('daemon_start') events.event('daemon_start')
if config.get('transports.lan', True):
if not onionrvalues.IS_QUBES:
Thread(target=LANServer(shared_state).start_server,
daemon=True).start()
LANManager(shared_state).start()
else:
logger.warn('LAN not supported on Qubes', terminal=True)
if config.get('transports.sneakernet', True): if config.get('transports.sneakernet', True):
Thread(target=sneakernet_import_thread, daemon=True).start() Thread(target=sneakernet_import_thread, daemon=True).start()
Thread(target=statistics_reporter,
args=[shared_state], daemon=True).start()
shared_state.get(DeadSimpleKV).put(
'proxyPort', net.socksPort)
spawn_client_threads(shared_state)
clean_blocks_not_meeting_pow(shared_state)
communicator.startCommunicator(shared_state)
clean_ephemeral_services()
if not offline_mode and not use_existing_tor:
net.killTor()
else:
try:
os.remove(filepaths.tor_hs_address_file)
except FileNotFoundError:
pass
better_sleep(5) better_sleep(5)

2
src/onionrcommands/daemonlaunch/killdaemon.py
View File

@ -10,7 +10,6 @@ from gevent import spawn
from onionrplugins import events from onionrplugins import events
from onionrutils import localcommand from onionrutils import localcommand
import logger import logger
from netcontroller import NetController
import config import config
""" """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
@ -43,7 +42,6 @@ def kill_daemon():
pass pass
events.event('daemon_stop') events.event('daemon_stop')
net = NetController(config.get('client.port', 59496))
try: try:
spawn( spawn(
localcommand.local_command, localcommand.local_command,

14
src/onionrcommands/daemonlaunch/spawndaemonthreads.py
View File

@ -1,14 +0,0 @@
from typing import TYPE_CHECKING
from onionrthreads import add_onionr_thread
from communicator.onlinepeers import get_online_peers
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
from toomanyobjs import TooMany
def spawn_client_threads(shared_state: 'TooMany'):
kv: 'DeadSimpleKV' = shared_state.get_by_string('DeadSimpleKV')
add_onionr_thread(get_online_peers, (shared_state,), 3, 1)

17
src/onionrcommands/openwebinterface.py
View File

@ -27,20 +27,6 @@ from .daemonlaunch import geturl
""" """
def _tell_if_ui_not_ready():
if local_command('/torready') != 'true':
logger.warn('The UI is not ready yet, waiting on Tor to start.', terminal=True)
def _wait_for_ui_to_be_ready():
if config.get('general.offline_mode', False) or \
not config.get('transports.tor', True) or \
config.get('tor.use_existing_tor'):
return
_tell_if_ui_not_ready()
while local_command('/torready') != 'true':
sleep(0.5)
logger.info("Tor is ready, opening UI", terminal=True)
def get_url() -> str: def get_url() -> str:
@ -61,8 +47,7 @@ def open_home():
'Onionr seems to not be running (could not get api host)', 'Onionr seems to not be running (could not get api host)',
terminal=True) terminal=True)
else: else:
_wait_for_ui_to_be_ready() # wait for Tor/transports to start sleep(3) # Sleep a little to wait for web UI to init some vars it needs
sleep(3) # Sleep a little longer to wait for web UI to init some vars it needs
url = get_url() url = get_url()
logger.info( logger.info(
'If Onionr does not open automatically, use this URL: ' + url, 'If Onionr does not open automatically, use this URL: ' + url,

2
src/onionrcommands/parser/arguments.py
View File

@ -19,7 +19,6 @@ from .. import motdcreator # cmd to generate new Onionr MOTDs
from .. import sitecreator # cmd to create multi-page sites from .. import sitecreator # cmd to create multi-page sites
from .. import togglebootstrap # cmd to toggle bootstrap file usage from .. import togglebootstrap # cmd to toggle bootstrap file usage
from ..listsites import print_site_list # cmd to list list ids from ..listsites import print_site_list # cmd to list list ids
from netcontroller.torcontrol.rebuildtor import rebuild as tor_restart
import onionrexceptions import onionrexceptions
from onionrutils import importnewblocks # func to import new blocks from onionrutils import importnewblocks # func to import new blocks
@ -54,7 +53,6 @@ def get_arguments() -> dict:
('start', 'daemon'): daemonlaunch.start, ('start', 'daemon'): daemonlaunch.start,
('stop', 'kill'): daemonlaunch.kill_daemon, ('stop', 'kill'): daemonlaunch.kill_daemon,
('restart',): restartonionr.restart, ('restart',): restartonionr.restart,
('restart-tor', 'restarttor'): tor_restart,
('add-address', 'addaddress', 'addadder'): keyadders.add_address, ('add-address', 'addaddress', 'addadder'): keyadders.add_address,
('openhome', 'gui', 'openweb', ('openhome', 'gui', 'openweb',
'open-home', 'open-web'): openwebinterface.open_home, 'open-home', 'open-web'): openwebinterface.open_home,

4
src/onionrpeers/__init__.py
View File

@ -1,4 +0,0 @@
from . import scoresortedpeerlist, peercleanup, peerprofiles
get_score_sorted_peer_list = scoresortedpeerlist.get_score_sorted_peer_list
peer_cleanup = peercleanup.peer_cleanup
PeerProfiles = peerprofiles.PeerProfiles

54
src/onionrpeers/peercleanup.py
View File

@ -1,54 +0,0 @@
'''
Onionr - Private P2P Communication
Cleanup the peer database
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3
import logger
from onionrutils import epoch
from . import scoresortedpeerlist, peerprofiles
from onionrblocks import onionrblacklist
import config
from coredb import keydb
def peer_cleanup():
'''Removes peers who have been offline too long or score too low'''
logger.info('Cleaning peers...')
blacklist = onionrblacklist.OnionrBlackList()
adders = scoresortedpeerlist.get_score_sorted_peer_list()
adders.reverse()
if len(adders) > 1:
min_score = int(config.get('peers.minimum_score', -100))
max_peers = int(config.get('peers.max_stored', 5000))
for address in adders:
# Remove peers that go below the negative score
if peerprofiles.PeerProfiles(address).score < min_score:
keydb.removekeys.remove_address(address)
try:
lastConnect = int(keydb.transportinfo.get_address_info(address, 'lastConnect'))
expireTime = 86400 - int(epoch.get_epoch()) - lastConnect
blacklist.addToDB(address, dataType=1, expire=expireTime)
except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue
pass
except ValueError:
pass
logger.warn('Removed address ' + address + '.')
# Unban probably not malicious peers TODO improve
blacklist.deleteExpired(dataType=1)

81
src/onionrpeers/peerprofiles.py
View File

@ -1,81 +0,0 @@
'''
Onionr - Private P2P Communication
This file contains both the PeerProfiles class for network profiling of Onionr nodes
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
from coredb import keydb
from onionrutils import epoch
from onionrutils import stringvalidators
from onionrblocks import onionrblacklist
import onionrexceptions
UPDATE_DELAY = 300
class PeerProfiles:
'''
PeerProfiles
'''
def __init__(self, address):
if not stringvalidators.validate_transport(address): raise onionrexceptions.InvalidAddress
self.address = address # node address
self.score = None
self.friendSigCount = 0
self.success = 0
self.failure = 0
self.connectTime = None
self.loadScore()
self.getConnectTime()
self.last_updated = {'connect_time': UPDATE_DELAY, 'score': UPDATE_DELAY} # Last time a given value was updated
if not address in keydb.listkeys.list_adders() and not onionrblacklist.OnionrBlackList().inBlacklist(address):
keydb.addkeys.add_address(address)
def loadScore(self):
'''Load the node's score from the database'''
try:
self.success = int(keydb.transportinfo.get_address_info(self.address, 'success'))
except (TypeError, ValueError) as e:
self.success = 0
self.score = self.success
def getConnectTime(self):
"""set the connectTime variable for when we last connected to them, using the db value"""
try:
self.connectTime = int(keydb.transportinfo.get_address_info(self.address, 'lastConnect'))
except (KeyError, ValueError, TypeError) as e:
pass
else:
return self.connectTime
def update_connect_time(self):
if epoch.get_epoch() - self.last_updated['connect_time'] >= UPDATE_DELAY:
self.last_updated['connect_time'] = epoch.get_epoch()
keydb.transportinfo.set_address_info(self.address, 'lastConnect', epoch.get_epoch())
def saveScore(self):
'''Save the node's score to the database'''
if epoch.get_epoch() - self.last_updated['score'] >= UPDATE_DELAY:
self.last_updated['score'] = epoch.get_epoch()
keydb.transportinfo.set_address_info(self.address, 'success', self.score)
return
def addScore(self, toAdd):
'''Add to the peer's score (can add negative)'''
self.score += toAdd
self.saveScore()

39
src/onionrpeers/scoresortedpeerlist.py
View File

@ -1,39 +0,0 @@
'''
Onionr - Private P2P Communication
Return a reliability score sorted list of peers
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
from . import peerprofiles
from coredb import keydb
def get_score_sorted_peer_list():
peer_list = keydb.listkeys.list_adders()
peer_scores = {}
peer_times = {}
for address in peer_list:
# Load peer's profiles into a list
profile = peerprofiles.PeerProfiles(address)
peer_scores[address] = profile.score
if not isinstance(profile.connectTime, type(None)):
peer_times[address] = profile.connectTime
else:
peer_times[address] = 9000
# Sort peers by their score, greatest to least, and then last connected time
peer_list = sorted(peer_scores, key=peer_scores.get, reverse=True)
peer_list = sorted(peer_times, key=peer_times.get, reverse=True)
return peer_list

2
src/onionrproofs/vdf.py
View File

@ -12,7 +12,7 @@ def _wrap_vdf_verify(queue, block_data_bytes, block_hash_hex, rounds):
def rounds_for_bytes(int: byte_count): def rounds_for_bytes(byte_count: int):
return byte_count * 1000 return byte_count * 1000

4
src/onionrsetup/setupconfig.py
View File

@ -81,10 +81,10 @@ def setup_config():
if type(config.get('client.webpassword')) is type(None): if type(config.get('client.webpassword')) is type(None):
config.set('client.webpassword', base64.b16encode(os.urandom(32)).decode('utf-8'), savefile=True) config.set('client.webpassword', base64.b16encode(os.urandom(32)).decode('utf-8'), savefile=True)
if type(config.get('client.client.port')) is type(None): if type(config.get('client.client.port')) is type(None):
randomPort = netcontroller.get_open_port() randomPort = netcontroller.getopenport.get_open_port()
config.set('client.client.port', randomPort, savefile=True) config.set('client.client.port', randomPort, savefile=True)
if type(config.get('client.public.port')) is type(None): if type(config.get('client.public.port')) is type(None):
randomPort = netcontroller.get_open_port() randomPort = netcontroller.getopenport.get_open_port()
config.set('client.public.port', randomPort, savefile=True) config.set('client.public.port', randomPort, savefile=True)
if type(config.get('client.api_version')) is type(None): if type(config.get('client.api_version')) is type(None):
config.set('client.api_version', onionrvalues.API_VERSION, savefile=True) config.set('client.api_version', onionrvalues.API_VERSION, savefile=True)

1
src/onionrstatistics/__init__.py
View File

@ -1 +0,0 @@
from .transports import tor

47
src/onionrstatistics/devreporting/__init__.py
View File

@ -1,47 +0,0 @@
"""Onionr - Private P2P Communication.
Dev utility to profile an Onionr subnetwork.
"""
import ujson as json
import config
from utils.bettersleep import better_sleep
from utils.gettransports import get as get_transports
from onionrutils import basicrequests
from onionrutils import epoch
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def statistics_reporter(shared_state):
server = config.get('statistics.server', '')
if not config.get('statistics.i_dont_want_privacy', False) or \
not server:
return
def compile_data():
return {
'time': epoch.get_epoch(),
'adders': get_transports(),
'peers': shared_state.get_by_string(
'DeadSimpleKV').get('onlinePeers')
}
while True:
better_sleep(5)
data = compile_data()
basicrequests.do_post_request(
f'http://{server}/sendstats/' + get_transports()[0],
data=json.dumps(data))

70
src/onionrstatistics/serializeddata.py
View File

@ -1,70 +0,0 @@
"""Onionr - Private P2P Communication.
Serialize various node information
"""
from typing import TYPE_CHECKING
from gevent import sleep
from psutil import Process
import ujson as json
from coredb import blockmetadb
from utils.sizeutils import size, human_size
from utils.identifyhome import identify_home
from onionrutils.epoch import get_epoch
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
class SerializedData:
def __init__(self):
"""
Serialized data is in JSON format:
{
'success': bool,
'foo': 'bar',
etc
}
"""
def get_stats(self):
"""Return statistics about our node"""
stats = {}
proc = Process()
def get_open_files():
return proc.num_fds()
try:
self._too_many
except AttributeError:
sleep(1)
kv: "DeadSimpleKV" = self._too_many.get_by_string("DeadSimpleKV")
connected = []
[connected.append(x)
for x in kv.get('onlinePeers') if x not in connected]
stats['uptime'] = get_epoch() - kv.get('startTime')
stats['connectedNodes'] = '\n'.join(connected)
stats['blockCount'] = len(blockmetadb.get_block_list())
stats['blockQueueCount'] = len(kv.get('blockQueue'))
stats['threads'] = proc.num_threads()
stats['ramPercent'] = proc.memory_percent()
stats['fd'] = get_open_files()
stats['diskUsage'] = human_size(size(identify_home()))
return json.dumps(stats)

1
src/onionrstatistics/transports/__init__.py
View File

@ -1 +0,0 @@
from . import tor

65
src/onionrstatistics/transports/tor/__init__.py
View File

@ -1,65 +0,0 @@
"""Onionr - Private P2P Communication.
tor stats info
"""
import ujson as json
from stem import CircStatus
import logger
from netcontroller.torcontrol.torcontroller import get_controller
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
class TorStats:
def __init__(self):
self.circuits = {}
self.json_data = ""
self.controller = None
def get_json(self):
"""Refresh circuits then serialize them into form:
"nodes": list of tuples containing fingerprint and nickname strings"
"purpose": https://stem.torproject.org/api/control.html#stem.CircPurpose
"""
if self.controller is None:
self.controller = get_controller()
if not self.controller.is_alive():
logger.info(f'{__name__} reconnecting to tor control')
self.controller = get_controller()
self.get_circuits()
json_serialized = {}
for circuit in self.circuits.keys():
json_serialized[circuit] = {
"nodes": [],
"purpose": self.circuits[circuit][1]
}
for entry in self.circuits[circuit][0]:
json_serialized[circuit]["nodes"].append({'finger': entry[0],
'nick': entry[1]})
self.json_data = json.dumps(json_serialized)
return self.json_data
def get_circuits(self):
"""Update the circuit dictionary"""
circuits = {}
for circ in list(sorted(self.controller.get_circuits())):
if circ.status != CircStatus.BUILT:
continue
circuits[circ.id] = (circ.path, circ.purpose)
self.circuits = circuits

2
src/onionrusers/onionrusers.py
View File

@ -254,6 +254,6 @@ class OnionrUser:
@classmethod @classmethod
def list_friends(cls): def list_friends(cls):
friendList = [] friendList = []
for x in keydb.listkeys.list_peers(trust=1): for x in keydb.listkeys.list_pub_keys(trust=1):
friendList.append(cls(x)) friendList.append(cls(x))
return list(friendList) return list(friendList)

48
src/onionrutils/stringvalidators.py
View File

@ -60,54 +60,6 @@ def validate_pub_key(key):
retVal = True retVal = True
return retVal return retVal
def validate_transport(id):
try:
idLength = len(id)
retVal = True
idNoDomain = ''
peerType = ''
# i2p b32 addresses are 60 characters long (including .b32.i2p)
if idLength == 60:
peerType = 'i2p'
if not id.endswith('.b32.i2p'):
retVal = False
else:
idNoDomain = id.split('.b32.i2p')[0]
# Onion v2's are 22 (including .onion), v3's are 62 with .onion
elif idLength == 22 or idLength == 62:
peerType = 'onion'
if not id.endswith('.onion'):
retVal = False
else:
idNoDomain = id.split('.onion')[0]
else:
retVal = False
if retVal:
if peerType == 'i2p':
try:
id.split('.b32.i2p')[2]
except IndexError:
pass
else:
retVal = False
elif peerType == 'onion':
try:
id.split('.onion')[2]
except IndexError:
pass
else:
retVal = False
if not idNoDomain.isalnum():
retVal = False
# Validate address is valid base32 (when capitalized and minus extension); v2/v3 onions and .b32.i2p use base32
for x in idNoDomain.upper():
if x not in string.ascii_uppercase and x not in '234567':
retVal = False
return retVal
except Exception as e:
return False
def is_integer_string(data): def is_integer_string(data):
'''Check if a string is a valid base10 integer (also returns true if already an int)''' '''Check if a string is a valid base10 integer (also returns true if already an int)'''

2
src/runtests/__init__.py
View File

@ -14,7 +14,6 @@ from .webpasstest import webpass_test
from .osver import test_os_ver_endpoint from .osver import test_os_ver_endpoint
from .clearnettor import test_clearnet_tor_request from .clearnettor import test_clearnet_tor_request
from .housekeeping import test_inserted_housekeeping from .housekeeping import test_inserted_housekeeping
from .lanservertest import test_lan_server
from .sneakernettest import test_sneakernet_import from .sneakernettest import test_sneakernet_import
from .dnsrebindingtest import test_dns_rebinding from .dnsrebindingtest import test_dns_rebinding
""" """
@ -41,7 +40,6 @@ RUN_TESTS = [uicheck.check_ui,
test_os_ver_endpoint, test_os_ver_endpoint,
test_clearnet_tor_request, test_clearnet_tor_request,
test_inserted_housekeeping, test_inserted_housekeeping,
test_lan_server,
sneakernettest.test_sneakernet_import, sneakernettest.test_sneakernet_import,
test_dns_rebinding test_dns_rebinding
] ]

1
src/runtests/inserttest.py
View File

@ -4,7 +4,6 @@ import time
import onionrblocks import onionrblocks
import logger import logger
import coredb import coredb
from communicator import peeraction
def _check_remote_node(testmanager): def _check_remote_node(testmanager):
return return

50
src/runtests/lanservertest.py
View File

@ -1,50 +0,0 @@
import requests
from lan.getip import best_ip
from onionrblocks import insert, onionrblockapi
from gevent import sleep
from coredb import blockmetadb
from onionrutils.epoch import get_epoch
import logger
from etc import onionrvalues
def test_lan_server(testmanager):
if onionrvalues.IS_QUBES:
logger.warn("Cannot test LAN on QubesOS", terminal=True)
return
start_time = get_epoch()
for i in range(1337, 1340):
try:
if not best_ip or not best_ip.startswith(('192.168')):
logger.warn(
"lanservertest not running, not in standard 192.168 lan " +
"run this test on a lan before release",
terminal=True)
return
if requests.get(f"http://{best_ip}:{i}/ping").text == 'onionr!':
bl = insert('test data')
sleep(10)
bl2 = insert('test data2')
sleep(30)
bl3 = insert('test data3')
l = requests.get(f"http://{best_ip}:{i}/blist/0").text.split('\n')
if bl not in l or bl2 not in l or bl3 not in l:
logger.error('blocks not in blist ' + '-'.join(l))
raise ValueError
time = blockmetadb.get_block_date(bl3) - 1
l = requests.get(f"http://{best_ip}:{i}/blist/{time}").text.split('\n')
if (bl in l and bl2 in l and bl3 in l) or len(l) == 0:
logger.error('Failed to get appopriate time' + '-'.join(l))
raise ValueError
if onionrblockapi.Block(bl).raw != requests.get(f"http://{best_ip}:{i}/get/{bl}", stream=True).raw.read(6000000):
logger.error('Block doesn\'t match')
raise ValueError
break
except requests.exceptions.ConnectionError:
pass
else:
raise ValueError

4
tests/test_onionrusers.py
View File

@ -30,12 +30,12 @@ class OnionrUserTests(unittest.TestCase):
def test_contact_init_no_save(self): def test_contact_init_no_save(self):
contact = crypto.generate()[0] contact = crypto.generate()[0]
contact = contactmanager.ContactManager(contact) contact = contactmanager.ContactManager(contact)
self.assertFalse(contact.publicKey in keydb.listkeys.list_peers()) self.assertFalse(contact.publicKey in keydb.listkeys.list_pub_keys())
def test_contact_create(self): def test_contact_create(self):
contact = crypto.generate()[0] contact = crypto.generate()[0]
contact = contactmanager.ContactManager(contact, saveUser=True) contact = contactmanager.ContactManager(contact, saveUser=True)
self.assertTrue(contact.publicKey in keydb.listkeys.list_peers()) self.assertTrue(contact.publicKey in keydb.listkeys.list_pub_keys())
def test_contact_set_info(self): def test_contact_set_info(self):
contact = crypto.generate()[0] contact = crypto.generate()[0]