diff --git a/docs/dev/http-api.md b/docs/dev/http-api.md index 6c189fd8..f1635cd8 100755 --- a/docs/dev/http-api.md +++ b/docs/dev/http-api.md @@ -64,18 +64,6 @@ Please note: endpoints that simply provide static web app files are not document * /getuptime - Methods: GET - Returns uptime in seconds -* /getActivePubkey - - Methods: GET - - Returns the current active public key in base32 format -* /getHumanReadable/pubkey - - Methods: GET - - Echos the specified public key in mnemonic format -* /insertblock - - Methods: POST - - Accepts JSON data for creating a new block. 'message' contains the block data, 'to' specifies the peer's public key to encrypt the data to, 'sign' is a boolean for signing the message. -* /torready - - Methods: POST - - Returns boolean if Tor is started or not # Public API diff --git a/scripts/disable-dev-config.py b/scripts/disable-dev-config.py index 734b8694..527019f3 100755 --- a/scripts/disable-dev-config.py +++ b/scripts/disable-dev-config.py @@ -6,26 +6,12 @@ import json conf = json.load(open('static-data/default_config.json', 'r')) -conf['tor']['use_existing_tor'] = False -conf['tor']['existing_control_port'] = 0 -conf['tor']['existing_control_password'] = "" -conf['tor']['existing_socks_port'] = 0 conf['general']['dev_mode'] = False -conf['general']['insert_deniable_blocks'] = True -conf['general']['random_bind_ip'] = True conf['general']['display_header'] = True conf['general']['security_level'] = 0 -conf['general']['use_bootstrap_list'] = True conf['onboarding']['done'] = False -conf['general']['minimum_block_pow'] = 5 -conf['general']['minimum_send_pow'] = 5 conf['log']['file']['remove_on_exit'] = True -conf['transports']['lan'] = True -conf['transports']['tor'] = True -conf['transports']['sneakernet'] = True -conf['statistics']['i_dont_want_privacy'] = False -conf['statistics']['server'] = '' conf['ui']['animated_background'] = True conf['runtests']['skip_slow'] = False diff --git a/scripts/enable-dev-config.py b/scripts/enable-dev-config.py index 6ca06cfb..e9a5af28 100755 --- a/scripts/enable-dev-config.py +++ b/scripts/enable-dev-config.py @@ -8,33 +8,16 @@ input("enter to continue") # hack to avoid vscode term input conf = json.load(open('static-data/default_config.json', 'r')) -block_pow = int(input("Block POW level:")) - conf['general']['security_level'] = int(input("Security level:")) -conf['transports']['tor'] = False -if input('Use Tor? y/n').lower() == 'y': - conf['transports']['tor'] = True - if input("Reuse Tor? y/n:").lower() == 'y': - conf['tor']['use_existing_tor'] = True - conf['tor']['existing_control_port'] = int(input("Enter existing control port:")) - conf['tor']['existing_control_password'] = input("Tor pass:") - conf['tor']['existing_socks_port'] = int(input("Existing socks port:")) conf['general']['dev_mode'] = True -conf['general']['insert_deniable_blocks'] = False conf['general']['random_bind_ip'] = False conf['onboarding']['done'] = True -conf['general']['minimum_block_pow'] = block_pow -conf['general']['minimum_send_pow'] = block_pow -conf['general']['use_bootstrap_list'] = False -if input("Use bootstrap list? y/n").lower() == 'y': - conf['general']['use_bootstrap_list'] = True + conf['log']['file']['remove_on_exit'] = False conf['ui']['animated_background'] = False conf['runtests']['skip_slow'] = True -if input('Stat reporting? y/n') == 'y': - conf['statistics']['i_dont_want_privacy'] = True - conf['statistics']['server'] = input('Statistics server') + json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4) diff --git a/src/__init__.py b/src/__init__.py index c78a7478..2120af0b 100755 --- a/src/__init__.py +++ b/src/__init__.py @@ -70,7 +70,6 @@ createdirs.create_dirs() import bigbrother # noqa from onionrcommands import parser # noqa from onionrplugins import onionrevents as events # noqa -from oldblocks.deleteplaintext import delete_plaintext_no_blacklist # noqa setup.setup_config() @@ -84,8 +83,6 @@ if config.get('advanced.security_auditing', True): except onionrexceptions.PythonVersion: pass -if not config.get('general.store_plaintext_blocks', True): - delete_plaintext_no_blacklist() setup.setup_default_plugins() diff --git a/src/apiservers/README.md b/src/apiservers/README.md index 3a439ac3..e9671385 100644 --- a/src/apiservers/README.md +++ b/src/apiservers/README.md @@ -6,4 +6,3 @@ Contains the WSGI servers Onionr uses for remote peer communication and local da * \_\_init\_\_.py: Exposes the server classes * private: Contains the client API (the server used to interact with the local Onionr daemon, and view the web UI) -* public: Contains the public API (the server used by remote peers to talk to our daemon) \ No newline at end of file diff --git a/src/apiservers/__init__.py b/src/apiservers/__init__.py index 82599cc7..f9926150 100755 --- a/src/apiservers/__init__.py +++ b/src/apiservers/__init__.py @@ -4,7 +4,6 @@ Public is net-facing server meant for other nodes Private is meant for controlling and accessing this node """ -from . import public, private +from . import private -PublicAPI = public.PublicAPI ClientAPI = private.PrivateAPI diff --git a/src/apiservers/private/__init__.py b/src/apiservers/private/__init__.py index 1c4bed9f..00930bb5 100644 --- a/src/apiservers/private/__init__.py +++ b/src/apiservers/private/__init__.py @@ -17,7 +17,6 @@ import logger from etc import waitforsetvar from . import register_private_blueprints import config -from .. import public """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -69,7 +68,7 @@ class PrivateAPI: self.httpServer = '' self.queueResponse = {} - self.get_block_data = httpapi.apiutils.GetBlockData(self) + register_private_blueprints.register_private_blueprints(self, app) httpapi.load_plugin_blueprints(app) self.app = app @@ -79,17 +78,11 @@ class PrivateAPI: waitforsetvar.wait_for_set_var(self, "_too_many") fd_handler = httpapi.fdsafehandler.FDSafeHandler self._too_many.add(httpapi.wrappedfunctions.SubProcVDFGenerator(self._too_many)) - self.publicAPI = self._too_many.get( # pylint: disable=E1101 - public.PublicAPI) self.httpServer = WSGIServer((self.host, self.bindPort), self.app, log=None, handler_class=fd_handler) self.httpServer.serve_forever() - def setPublicAPIInstance(self, inst): - """Dynamically set public API instance.""" - self.publicAPI = inst - def validateToken(self, token): """Validate that the client token matches the given token. @@ -112,10 +105,3 @@ class PrivateAPI: # Don't error on race condition with startup pass - def getBlockData(self, bHash, decrypt=False, raw=False, - headerOnly=False) -> bytes: - """Returns block data bytes.""" - return self.get_block_data.get_block_data(bHash, - decrypt=decrypt, - raw=raw, - headerOnly=headerOnly) diff --git a/src/apiservers/private/register_private_blueprints.py b/src/apiservers/private/register_private_blueprints.py index 40039ddf..4474a248 100644 --- a/src/apiservers/private/register_private_blueprints.py +++ b/src/apiservers/private/register_private_blueprints.py @@ -5,8 +5,8 @@ This file registers blueprints for the private api server from threading import Thread from gevent import sleep -from httpapi import security, friendsapi, configapi, insertblock -from httpapi import miscclientapi, onionrsitesapi, apiutils +from httpapi import security, configapi +from httpapi import miscclientapi, apiutils from httpapi import themeapi from httpapi import fileoffsetreader from httpapi.sse.private import private_sse_blueprint @@ -31,14 +31,9 @@ def register_private_blueprints(private_api, app): """Register private API plask blueprints.""" app.register_blueprint(security.client.ClientAPISecurity( private_api).client_api_security_bp) - app.register_blueprint(friendsapi.friends) app.register_blueprint(configapi.config_BP) - app.register_blueprint(insertblock.ib) - app.register_blueprint(miscclientapi.getblocks.client_get_blocks) app.register_blueprint(miscclientapi.endpoints.PrivateEndpoints( private_api).private_endpoints_bp) - app.register_blueprint(miscclientapi.motd.bp) - app.register_blueprint(onionrsitesapi.site_api) app.register_blueprint(apiutils.shutdown.shutdown_bp) app.register_blueprint(miscclientapi.staticfiles.static_files_bp) app.register_blueprint(themeapi.theme_blueprint) diff --git a/src/apiservers/public/__init__.py b/src/apiservers/public/__init__.py deleted file mode 100644 index 2a2c610a..00000000 --- a/src/apiservers/public/__init__.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Onionr - Private P2P Communication. - -This file handles all incoming http requests -to the public api server, using Flask -""" -import time -import threading -import flask -from gevent.pywsgi import WSGIServer -from httpapi import apiutils, security, fdsafehandler, miscpublicapi -import logger -import config -import filepaths -from utils import gettransports -from etc import onionrvalues, waitforsetvar -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def _get_tor_adder(pub_api): - transports = [] - while len(transports) == 0: - transports = gettransports.get() - time.sleep(0.3) - pub_api.torAdder = transports[0] - - -class PublicAPI: - """The new client api server, isolated from the public api.""" - - def __init__(self): - """Setup the public api app.""" - app = flask.Flask('PublicAPI') - app.config['MAX_CONTENT_LENGTH'] = 5 * 1024 * 1024 - self.i2pEnabled = config.get('i2p.host', False) - self.hideBlocks = [] # Blocks to be denied sharing - self.host = apiutils.setbindip.set_bind_IP( - filepaths.public_API_host_file) - - threading.Thread(target=_get_tor_adder, - args=[self], daemon=True).start() - - self.torAdder = "" - self.bindPort = config.get('client.public.port') - self.lastRequest = 0 - # total rec requests to public api since server started - self.hitCount = 0 - self.config = config - self.API_VERSION = onionrvalues.API_VERSION - logger.info('Running public api on %s:%s' % (self.host, self.bindPort)) - - app.register_blueprint( - security.public.PublicAPISecurity(self).public_api_security_bp) - app.register_blueprint( - miscpublicapi.endpoints.PublicEndpoints(self).public_endpoints_bp) - self.app = app - - def start(self): - """Start the Public API server.""" - waitforsetvar.wait_for_set_var(self, "_too_many") - self.httpServer = WSGIServer((self.host, self.bindPort), - self.app, log=None, - handler_class=fdsafehandler.FDSafeHandler) - self.httpServer.serve_forever() diff --git a/src/bigbrother/ministry/ofexec.py b/src/bigbrother/ministry/ofexec.py index 705d9072..8baf9188 100644 --- a/src/bigbrother/ministry/ofexec.py +++ b/src/bigbrother/ministry/ofexec.py @@ -37,6 +37,7 @@ def block_exec(event, info): # because libraries have stupid amounts of compile/exec/eval, # We have to use a whitelist where it can be tolerated # Generally better than nothing, not a silver bullet + return whitelisted_code = [ 'netrc.py', 'shlex.py', diff --git a/src/communicator/__init__.py b/src/communicator/__init__.py index 43122111..face00c6 100755 --- a/src/communicator/__init__.py +++ b/src/communicator/__init__.py @@ -9,26 +9,21 @@ import time import config import logger import onionrplugins as plugins -from communicatorutils import uploadblocks -from . import uploadqueue -from onionrthreads import add_onionr_thread from onionrcommands.openwebinterface import get_url -from netcontroller import NetController -from . import bootstrappeers from . import daemoneventhooks """ - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. - You should have received a copy of the GNU General Public License - along with this program. If not, see . +You should have received a copy of the GNU General Public License +along with this program. If not, see . """ config.reload() @@ -51,11 +46,7 @@ class OnionrCommunicatorDaemon: if config.get('general.offline_mode', False): self.kv.put('isOnline', False) - # initialize core with Tor socks port being 3rd argument - self.proxyPort = shared_state.get(NetController).socksPort - self.upload_session_manager = self.shared_state.get( - uploadblocks.sessionmanager.BlockUploadSessionManager) self.shared_state.share_object() # loop time.sleep delay in seconds @@ -67,12 +58,6 @@ class OnionrCommunicatorDaemon: # Loads in and starts the enabled plugins plugins.reload() - # extends our upload list and saves our list when Onionr exits - uploadqueue.UploadQueue(self) - - if config.get('general.use_bootstrap_list', True): - bootstrappeers.add_bootstrap_list_to_peer_list( - self.kv, [], db_only=True) daemoneventhooks.daemon_event_handlers(shared_state) @@ -104,20 +89,6 @@ class OnionrCommunicatorDaemon: logger.info( 'Goodbye. (Onionr is cleaning up, and will exit)', terminal=True) - def getPeerProfileInstance(self, peer): - """Gets a peer profile instance from the list of profiles""" - for i in self.kv.get('peerProfiles'): - # if the peer's profile is already loaded, return that - if i.address == peer: - retData = i - break - else: - # if the peer's profile is not loaded, return a new one. - # connectNewPeer also adds it to the list on connect - retData = onionrpeers.PeerProfiles(peer) - self.kv.get('peerProfiles').append(retData) - return retData - def startCommunicator(shared_state): OnionrCommunicatorDaemon(shared_state) diff --git a/src/communicator/bootstrappeers.py b/src/communicator/bootstrappeers.py deleted file mode 100644 index 191c3165..00000000 --- a/src/communicator/bootstrappeers.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Onionr - Private P2P Communication. - -add bootstrap peers to the communicator peer list -""" -from typing import TYPE_CHECKING - -from utils import readstatic, gettransports -from coredb import keydb -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - -bootstrap_peers = readstatic.read_static('bootstrap-nodes.txt').split(',') - - -def add_bootstrap_list_to_peer_list(kv, peerList, db_only=False): - """Add the bootstrap list to the peer list (no duplicates).""" - for i in bootstrap_peers: - # Add bootstrap peers to peerList (does not save them) - # Don't add them if they're already added or in the offline list - if i not in peerList and i not in kv.get('offlinePeers') \ - and i not in gettransports.get() and len(str(i).strip()) > 0: - if not db_only: - peerList.append(i) - keydb.addkeys.add_address(i) diff --git a/src/communicator/daemoneventhooks/__init__.py b/src/communicator/daemoneventhooks/__init__.py index f43ac54f..4eb12e34 100644 --- a/src/communicator/daemoneventhooks/__init__.py +++ b/src/communicator/daemoneventhooks/__init__.py @@ -4,14 +4,10 @@ Hooks to handle daemon events """ from threading import Thread -from .removefrominsertqueue import remove_from_insert_queue - from typing import TYPE_CHECKING from gevent import sleep -from communicatorutils.uploadblocks import mixmate -from communicatorutils import restarttor if TYPE_CHECKING: from toomanyobjs import TooMany @@ -19,7 +15,6 @@ if TYPE_CHECKING: from communicator import OnionrCommunicatorDaemon from httpapi.daemoneventsapi import DaemonEventsBP from onionrtypes import BlockHash - from apiservers import PublicAPI """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -44,39 +39,16 @@ def daemon_event_handlers(shared_state: 'TooMany'): except KeyError: sleep(0.2) comm_inst = _get_inst('OnionrCommunicatorDaemon') - public_api: 'PublicAPI' = _get_inst('PublicAPI') events_api: 'DaemonEventsBP' = _get_inst('DaemonEventsBP') kv: 'DeadSimpleKV' = _get_inst('DeadSimpleKV') - def remove_from_insert_queue_wrapper(block_hash: 'BlockHash'): - remove_from_insert_queue(comm_inst, block_hash) - return "removed" - def print_test(text=''): print("It works!", text) return f"It works! {text}" - def upload_event(block: 'BlockHash' = ''): - if not block: - raise ValueError - public_api.hideBlocks.append(block) - try: - mixmate.block_mixer(kv.get('blocksToUpload'), block) - except ValueError: - pass - return "removed" - - def restart_tor(): - restarttor.restart(shared_state) - kv.put('offlinePeers', []) - kv.put('onlinePeers', []) - def test_runtime(): Thread(target=comm_inst.shared_state.get_by_string( "OnionrRunTestManager").run_tests).start() - events_api.register_listener(remove_from_insert_queue_wrapper) - events_api.register_listener(restart_tor) events_api.register_listener(print_test) - events_api.register_listener(upload_event) events_api.register_listener(test_runtime) diff --git a/src/communicator/daemoneventhooks/removefrominsertqueue.py b/src/communicator/daemoneventhooks/removefrominsertqueue.py deleted file mode 100644 index d135ccd9..00000000 --- a/src/communicator/daemoneventhooks/removefrominsertqueue.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Onionr - P2P Anonymous Storage Network. - -Remove block hash from daemon's upload list. -""" -from typing import TYPE_CHECKING -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV - from communicator import OnionrCommunicatorDaemon - from onionrtypes import BlockHash -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def remove_from_insert_queue(comm_inst: "OnionrCommunicatorDaemon", - b_hash: "BlockHash"): - """Remove block hash from daemon's upload list.""" - kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV") - try: - kv.get('generating_blocks').remove(b_hash) - except ValueError: - pass diff --git a/src/communicator/onlinepeers/README.md b/src/communicator/onlinepeers/README.md deleted file mode 100644 index fe56995f..00000000 --- a/src/communicator/onlinepeers/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Online Peers - -Manages a pool of peers to perform actions with. Since Onionr does not maintain socket connections, it holds a list of peers. - - -## Files - -* \_\_init\_\_.py: exposes some functions to interact with the pool -* clearofflinepeer.py: Pop the oldest peer in the offline list -* onlinepeers.py: communicator timer to add new peers to the pool randomly -* pickonlinepeers.py: returns a random peer from the online pool -* removeonlinepeer.py: removes a specified peer from the online pool \ No newline at end of file diff --git a/src/communicator/onlinepeers/__init__.py b/src/communicator/onlinepeers/__init__.py deleted file mode 100644 index a3b107ee..00000000 --- a/src/communicator/onlinepeers/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from . import clearofflinepeer, onlinepeers, pickonlinepeers, removeonlinepeer - -clear_offline_peer = clearofflinepeer.clear_offline_peer -get_online_peers = onlinepeers.get_online_peers -pick_online_peer = pickonlinepeers.pick_online_peer -remove_online_peer = removeonlinepeer.remove_online_peer \ No newline at end of file diff --git a/src/communicator/onlinepeers/clearofflinepeer.py b/src/communicator/onlinepeers/clearofflinepeer.py deleted file mode 100644 index 47128a73..00000000 --- a/src/communicator/onlinepeers/clearofflinepeer.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Onionr - Private P2P Communication. - -clear offline peer in a communicator instance -""" -from typing import TYPE_CHECKING - -import logger -if TYPE_CHECKING: - from communicator import OnionrCommunicatorDaemon -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def clear_offline_peer(kv: 'DeadSimpleKV'): - """Remove the longest offline peer to retry later.""" - try: - removed = kv.get('offlinePeers').pop(0) - except IndexError: - pass - else: - logger.debug('Removed ' + removed + - ' from offline list, will try them again.') - diff --git a/src/communicator/onlinepeers/onlinepeers.py b/src/communicator/onlinepeers/onlinepeers.py deleted file mode 100644 index e9966308..00000000 --- a/src/communicator/onlinepeers/onlinepeers.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Onionr - Private P2P Communication. - -get online peers in a communicator instance -""" -import time -from typing import TYPE_CHECKING - -import config -from etc.humanreadabletime import human_readable_time -from communicatorutils.connectnewpeers import connect_new_peer_to_communicator -import logger -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def get_online_peers(shared_state): - """Manage the kv.get('onlinePeers') attribute list. - - Connect to more peers if we have none connected - """ - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - if config.get('general.offline_mode', False): - return - logger.info('Refreshing peer pool...') - max_peers = int(config.get('peers.max_connect', 10)) - needed = max_peers - len(kv.get('onlinePeers')) - - last_seen = 'never' - if not isinstance(kv.get('lastNodeSeen'), type(None)): - last_seen = human_readable_time(kv.get('lastNodeSeen')) - - for _ in range(needed): - if len(kv.get('onlinePeers')) == 0: - connect_new_peer_to_communicator(shared_state, useBootstrap=True) - else: - connect_new_peer_to_communicator(shared_state) - - if kv.get('shutdown'): - break - else: - if len(kv.get('onlinePeers')) == 0: - logger.debug('Couldn\'t connect to any peers.' + - f' Last node seen {last_seen} ago.') - try: - get_online_peers(shared_state) - except RecursionError: - pass - else: - kv.put('lastNodeSeen', time.time()) diff --git a/src/communicator/onlinepeers/pickonlinepeers.py b/src/communicator/onlinepeers/pickonlinepeers.py deleted file mode 100644 index dc11bceb..00000000 --- a/src/communicator/onlinepeers/pickonlinepeers.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Onionr - Private P2P Communication. - -pick online peers in a communicator instance -""" -import secrets -from typing import TYPE_CHECKING - -import onionrexceptions - -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def pick_online_peer(kv: 'DeadSimpleKV'): - """Randomly picks peer from pool without bias (using secrets module).""" - ret_data = '' - peer_length = len(kv.get('onlinePeers')) - if peer_length <= 0: - raise onionrexceptions.OnlinePeerNeeded - - while True: - peer_length = len(kv.get('onlinePeers')) - - try: - # Get a random online peer, securely. - # May get stuck in loop if network is lost - ret_data = kv.get('onlinePeers')[secrets.randbelow(peer_length)] - except IndexError: - pass - else: - break - return ret_data diff --git a/src/communicator/onlinepeers/removeonlinepeer.py b/src/communicator/onlinepeers/removeonlinepeer.py deleted file mode 100644 index 0eb0b338..00000000 --- a/src/communicator/onlinepeers/removeonlinepeer.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Onionr - Private P2P Communication. - -remove an online peer from the pool in a communicator instance -""" -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def remove_online_peer(kv, peer): - """Remove an online peer.""" - try: - del kv.get('connectTimes')[peer] - except KeyError: - pass - try: - del kv.get('dbTimestamps')[peer] - except KeyError: - pass - try: - kv.get('onlinePeers').remove(peer) - except ValueError: - pass diff --git a/src/communicator/peeraction.py b/src/communicator/peeraction.py deleted file mode 100644 index f09814f6..00000000 --- a/src/communicator/peeraction.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Onionr - Private P2P Communication. - -This file implements logic for performing requests to Onionr peers -""" -from typing import TYPE_CHECKING - -import streamedrequests -import logger -from onionrutils import epoch, basicrequests -from coredb import keydb -from . import onlinepeers -from onionrtypes import OnionAddressString -from onionrpeers.peerprofiles import PeerProfiles -from etc.waitforsetvar import wait_for_set_var -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def get_peer_profile(kv, address: OnionAddressString) -> 'PeerProfiles': - profile_inst_list = kv.get('peerProfiles') - for profile in profile_inst_list: - if profile.address == address: - return profile - p = PeerProfiles(address) - profile_inst_list.append(p) - return p - - -def peer_action(shared_state, peer, action, - returnHeaders=False, max_resp_size=5242880): - """Perform a get request to a peer.""" - penalty_score = -10 - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - if len(peer) == 0: - return False - url = 'http://%s/%s' % (peer, action) - - try: - ret_data = basicrequests.do_get_request(url, port=kv.get('proxyPort'), - max_size=max_resp_size) - except streamedrequests.exceptions.ResponseLimitReached: - logger.warn( - 'Request failed due to max response size being overflowed', - terminal=True) - ret_data = False - penalty_score = -100 - # if request failed, (error), mark peer offline - if ret_data is False: - try: - get_peer_profile(kv, peer).addScore(penalty_score) - onlinepeers.remove_online_peer(kv, peer) - keydb.transportinfo.set_address_info( - peer, 'lastConnectAttempt', epoch.get_epoch()) - if action != 'ping' and not kv.get('shutdown'): - logger.warn(f'Lost connection to {peer}', terminal=True) - # Will only add a new peer to pool if needed - onlinepeers.get_online_peers(kv) - except ValueError: - pass - else: - peer_profile = get_peer_profile(kv, peer) - peer_profile.update_connect_time() - peer_profile.addScore(1) - # If returnHeaders, returns tuple of data, headers. - # If not, just data string - return ret_data diff --git a/src/communicator/uploadqueue/__init__.py b/src/communicator/uploadqueue/__init__.py deleted file mode 100644 index 6b1331cc..00000000 --- a/src/communicator/uploadqueue/__init__.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Onionr - Private P2P Communication. - -Class to remember blocks that need to be uploaded -and not shared on startup/shutdown -""" -import atexit -import os -from typing import TYPE_CHECKING - -import deadsimplekv - -import filepaths -from onionrutils import localcommand -if TYPE_CHECKING: - from communicator import OnionrCommunicatorDaemon - from deadsimplekv import DeadSimpleKV -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - -UPLOAD_MEMORY_FILE = filepaths.upload_list - - -def _add_to_hidden_blocks(cache): - for bl in cache: - localcommand.local_command('waitforshare/' + bl, post=True) - - -class UploadQueue: - """Saves and loads block upload info from json file.""" - - def __init__(self, communicator: 'OnionrCommunicatorDaemon'): - """Start the UploadQueue object, loading left over uploads into queue. - - register save shutdown function - """ - self.communicator = communicator - cache: deadsimplekv.DeadSimpleKV = deadsimplekv.DeadSimpleKV( - UPLOAD_MEMORY_FILE) - self.kv: "DeadSimpleKV" = \ - communicator.shared_state.get_by_string("DeadSimpleKV") - self.store_obj = cache - cache = cache.get('uploads') - if cache is None: - cache = [] - - _add_to_hidden_blocks(cache) - self.kv.get('blocksToUpload').extend(cache) - - atexit.register(self.save) - - def save(self): - """Save to disk on shutdown or if called manually.""" - bl: deadsimplekv.DeadSimpleKV = self.kv.get('blocksToUpload') - if len(bl) == 0: - try: - os.remove(UPLOAD_MEMORY_FILE) - except FileNotFoundError: - pass - else: - self.store_obj.put('uploads', bl) - self.store_obj.flush() diff --git a/src/communicatorutils/README.md b/src/communicatorutils/README.md deleted file mode 100755 index 393328c3..00000000 --- a/src/communicatorutils/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# communicatorutils - -The files in this submodule handle various subtasks and utilities for the onionr communicator. - -## Files: - -announcenode.py: Uses a communicator instance to announce our transport address to connected nodes - -connectnewpeers.py: takes a communicator instance and has it connect to as many peers as needed, and/or to a new specified peer. - -cooldownpeer.py: randomly selects a connected peer in a communicator and disconnects them for the purpose of security and network balancing. - -daemonqueuehandler.py: checks for new commands in the daemon queue and processes them accordingly. - -deniableinserts.py: insert fake blocks with the communicator for plausible deniability - -downloadblocks.py: iterates a communicator instance's block download queue and attempts to download the blocks from online peers - -housekeeping.py: cleans old blocks and forward secrecy keys - -lookupadders.py: ask connected peers to share their list of peer transport addresses - -lookupblocks.py: lookup new blocks from connected peers from the communicator - -netcheck.py: check if the node is online based on communicator status and onion server ping results - -onionrcommunicataortimers.py: create a timer for a function to be launched on an interval. Control how many possible instances of a timer may be running a function at once and control if the timer should be ran in a thread or not. - -proxypicker.py: returns a string name for the appropriate proxy to be used with a particular peer transport address. - -servicecreator.py: iterate connection blocks and create new direct connection servers for them. - -uploadblocks.py: iterate a communicator's upload queue and upload the blocks to connected peers \ No newline at end of file diff --git a/src/communicatorutils/__init__.py b/src/communicatorutils/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/src/communicatorutils/announcenode.py b/src/communicatorutils/announcenode.py deleted file mode 100755 index 91caa987..00000000 --- a/src/communicatorutils/announcenode.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Onionr - Private P2P Communication. - -Use a communicator instance to announce -our transport address to connected nodes -""" -from typing import TYPE_CHECKING - -import logger -from onionrutils import basicrequests -from utils import gettransports -from netcontroller import NetController -from communicator import onlinepeers -from coredb import keydb -import onionrexceptions - -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def announce_node(shared_state): - """Announce our node to our peers.""" - ret_data = False - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - config = shared_state.get_by_string("OnionrCommunicatorDaemon").config - # Do not let announceCache get too large - if len(kv.get('announceCache')) >= 10000: - kv.get('announceCache').popitem() - - if config.get('general.security_level', 0) == 0: - # Announce to random online peers - for i in kv.get('onlinePeers'): - if i not in kv.get('announceCache'): - peer = i - break - else: - try: - peer = onlinepeers.pick_online_peer(kv) - except onionrexceptions.OnlinePeerNeeded: - peer = "" - - try: - ourID = gettransports.get()[0] - if not peer: - raise onionrexceptions.OnlinePeerNeeded - except (IndexError, onionrexceptions.OnlinePeerNeeded): - pass - else: - url = 'http://' + peer + '/announce' - data = {'node': ourID} - - logger.info('Announcing node to ' + url) - if basicrequests.do_post_request( - url, - data, - port=shared_state.get(NetController).socksPort)\ - == 'Success': - logger.info('Successfully introduced node to ' + peer, - terminal=True) - ret_data = True - keydb.transportinfo.set_address_info(peer, 'introduced', 1) - - return ret_data diff --git a/src/communicatorutils/connectnewpeers.py b/src/communicatorutils/connectnewpeers.py deleted file mode 100755 index 3278c569..00000000 --- a/src/communicatorutils/connectnewpeers.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Onionr - Private P2P Communication. - -Connect a new peer to our communicator instance. -Does so randomly if no peer is specified -""" -import time -import secrets - -import onionrexceptions -import logger -import onionrpeers -from utils import networkmerger, gettransports -from onionrutils import stringvalidators, epoch -from communicator import peeraction, bootstrappeers -from coredb import keydb -import config -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def connect_new_peer_to_communicator(shared_state, peer='', useBootstrap=False): - retData = False - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - tried = kv.get('offlinePeers') - transports = gettransports.get() - if peer != '': - if stringvalidators.validate_transport(peer): - peerList = [peer] - else: - raise onionrexceptions.InvalidAddress( - 'Will not attempt connection test to invalid address') - else: - peerList = keydb.listkeys.list_adders() - - mainPeerList = keydb.listkeys.list_adders() - if not peerList: - peerList = onionrpeers.get_score_sorted_peer_list() - - """ - If we don't have enough peers connected or random chance, - select new peers to try - """ - if len(peerList) < 8 or secrets.randbelow(4) == 3: - tryingNew = [] - for x in kv.get('newPeers'): - if x not in peerList: - peerList.append(x) - tryingNew.append(x) - for i in tryingNew: - kv.get('newPeers').remove(i) - - if len(peerList) == 0 or useBootstrap: - # Avoid duplicating bootstrap addresses in peerList - if config.get('general.use_bootstrap_list', True): - bootstrappeers.add_bootstrap_list_to_peer_list(kv, peerList) - - for address in peerList: - address = address.strip() - - # Don't connect to our own address - if address in transports: - continue - """Don't connect to invalid address or - if its already been tried/connected, or if its cooled down - """ - if len(address) == 0 or address in tried \ - or address in kv.get('onlinePeers') \ - or address in kv.get('cooldownPeer'): - continue - if kv.get('shutdown'): - return - # Ping a peer, - ret = peeraction.peer_action(shared_state, address, 'ping') - if ret == 'pong!': - time.sleep(0.1) - if address not in mainPeerList: - # Add a peer to our list if it isn't already since it connected - networkmerger.mergeAdders(address) - if address not in kv.get('onlinePeers'): - logger.info('Connected to ' + address, terminal=True) - kv.get('onlinePeers').append(address) - kv.get('connectTimes')[address] = epoch.get_epoch() - retData = address - - # add peer to profile list if they're not in it - for profile in kv.get('peerProfiles'): - if profile.address == address: - break - else: - kv.get('peerProfiles').append( - onionrpeers.PeerProfiles(address)) - try: - del kv.get('plaintextDisabledPeers')[address] - except KeyError: - pass - if peeraction.peer_action( - shared_state, address, 'plaintext') == 'false': - kv.get('plaintextDisabledPeers')[address] = True - break - - else: - # Mark a peer as tried if they failed to respond to ping - tried.append(address) - logger.debug('Failed to connect to %s: %s ' % (address, ret)) - return retData diff --git a/src/communicatorutils/cooldownpeer.py b/src/communicatorutils/cooldownpeer.py deleted file mode 100755 index 0e57db8b..00000000 --- a/src/communicatorutils/cooldownpeer.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Onionr - Private P2P Communication. - -Select random online peer in a communicator instance and have them "cool down" -""" -from typing import TYPE_CHECKING - -from onionrutils import epoch -from communicator import onlinepeers - -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def cooldown_peer(shared_state): - """Randomly add an online peer to cooldown, so we can connect a new one.""" - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - config = shared_state.get_by_string("OnionrCommunicatorDaemon").config - online_peer_amount = len(kv.get('onlinePeers')) - minTime = 300 - cooldown_time = 600 - to_cool = '' - tempConnectTimes = dict(kv.get('connectTimes')) - - # Remove peers from cooldown that have been there long enough - tempCooldown = dict(kv.get('cooldownPeer')) - for peer in tempCooldown: - if (epoch.get_epoch() - tempCooldown[peer]) >= cooldown_time: - del kv.get('cooldownPeer')[peer] - - # Cool down a peer, if we have max connections alive for long enough - if online_peer_amount >= config.get('peers.max_connect', 10, save=True): - finding = True - - while finding: - try: - to_cool = min(tempConnectTimes, key=tempConnectTimes.get) - if (epoch.get_epoch() - tempConnectTimes[to_cool]) < minTime: - del tempConnectTimes[to_cool] - else: - finding = False - except ValueError: - break - else: - onlinepeers.remove_online_peer(kv, to_cool) - kv.get('cooldownPeer')[to_cool] = epoch.get_epoch() - diff --git a/src/communicatorutils/deniableinserts.py b/src/communicatorutils/deniableinserts.py deleted file mode 100755 index aff6e2b2..00000000 --- a/src/communicatorutils/deniableinserts.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Onionr - Private P2P Communication. - -Use the communicator to insert fake mail messages -""" -import secrets - -from etc import onionrvalues -import oldblocks -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def insert_deniable_block(): - """Insert a fake block to make it more difficult to track real blocks.""" - fakePeer = '' - chance = 10 - if secrets.randbelow(chance) == (chance - 1): - # This assumes on the libsodium primitives to have key-privacy - fakePeer = onionrvalues.DENIABLE_PEER_ADDRESS - data = secrets.token_hex(secrets.randbelow(5120) + 1) - oldblocks.insert(data, header='pm', encryptType='asym', - asymPeer=fakePeer, disableForward=True, - meta={'subject': 'foo'}) diff --git a/src/communicatorutils/downloadblocks/__init__.py b/src/communicatorutils/downloadblocks/__init__.py deleted file mode 100755 index 9be16770..00000000 --- a/src/communicatorutils/downloadblocks/__init__.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Onionr - Private P2P Communication. - -Download blocks using the communicator instance. -""" -from typing import TYPE_CHECKING -from secrets import SystemRandom - -if TYPE_CHECKING: - from communicator import OnionrCommunicatorDaemon - from deadsimplekv import DeadSimpleKV - -from gevent import spawn - -import onionrexceptions -import logger -import onionrpeers - -from communicator import peeraction -from communicator import onlinepeers -from oldblocks import blockmetadata -from onionrutils import validatemetadata -from coredb import blockmetadb -from onionrutils.localcommand import local_command -import onionrcrypto -import onionrstorage -from oldblocks import onionrblacklist -from oldblocks import storagecounter -from . import shoulddownload -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - -storage_counter = storagecounter.StorageCounter() - - -def download_blocks_from_communicator(shared_state: "TooMany"): - """Use communicator instance to download blocks in the comms's queue""" - blacklist = onionrblacklist.OnionrBlackList() - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter - count: int = 0 - metadata_validation_result: bool = False - # Iterate the block queue in the communicator - for blockHash in list(kv.get('blockQueue')): - count += 1 - - try: - blockPeers = list(kv.get('blockQueue')[blockHash]) - except KeyError: - blockPeers = [] - removeFromQueue = True - - if not shoulddownload.should_download(shared_state, blockHash): - continue - - if kv.get('shutdown') or not kv.get('isOnline') or \ - storage_counter.is_full(): - # Exit loop if shutting down or offline, or disk allocation reached - break - # Do not download blocks being downloaded - if blockHash in kv.get('currentDownloading'): - continue - - if len(kv.get('onlinePeers')) == 0: - break - - # So we can avoid concurrent downloading in other threads of same block - kv.get('currentDownloading').append(blockHash) - if len(blockPeers) == 0: - try: - peerUsed = onlinepeers.pick_online_peer(kv) - except onionrexceptions.OnlinePeerNeeded: - continue - else: - SystemRandom().shuffle(blockPeers) - peerUsed = blockPeers.pop(0) - - if not kv.get('shutdown') and peerUsed.strip() != '': - logger.info( - f"Attempting to download %s from {peerUsed}..." % (blockHash[:12],)) - content = peeraction.peer_action( - shared_state, peerUsed, - 'getdata/' + blockHash, - max_resp_size=3000000) # block content from random peer - - if content is not False and len(content) > 0: - try: - content = content.encode() - except AttributeError: - pass - - realHash = onionrcrypto.hashers.sha3_hash(content) - try: - realHash = realHash.decode() # bytes on some versions for some reason - except AttributeError: - pass - if realHash == blockHash: - #content = content.decode() # decode here because sha3Hash needs bytes above - metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata - metadata = metas[0] - try: - metadata_validation_result = \ - validatemetadata.validate_metadata(metadata, metas[2]) - except onionrexceptions.PlaintextNotSupported: - logger.debug(f"Not saving {blockHash} due to plaintext not enabled") - removeFromQueue = True - except onionrexceptions.DataExists: - metadata_validation_result = False - if metadata_validation_result: # check if metadata is valid, and verify nonce - if onionrcrypto.cryptoutils.verify_POW(content): # check if POW is enough/correct - logger.info('Attempting to save block %s...' % blockHash[:12]) - try: - onionrstorage.set_data(content) - except onionrexceptions.DataExists: - logger.warn('Data is already set for %s ' % (blockHash,)) - except onionrexceptions.DiskAllocationReached: - logger.error('Reached disk allocation allowance, cannot save block %s.' % (blockHash,)) - removeFromQueue = False - else: - blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db - blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database - spawn( - local_command, - f'/daemon-event/upload_event', - post=True, - is_json=True, - post_data={'block': blockHash} - ) - else: - logger.warn('POW failed for block %s.' % (blockHash,)) - else: - if blacklist.inBlacklist(realHash): - logger.warn('Block %s is blacklisted.' % (realHash,)) - else: - logger.warn('Metadata for block %s is invalid.' % (blockHash,)) - blacklist.addToDB(blockHash) - else: - # if block didn't meet expected hash - tempHash = onionrcrypto.hashers.sha3_hash(content) # lazy hack, TODO use var - try: - tempHash = tempHash.decode() - except AttributeError: - pass - # Punish peer for sharing invalid block (not always malicious, but is bad regardless) - onionrpeers.PeerProfiles(peerUsed).addScore(-50) - if tempHash != 'ed55e34cb828232d6c14da0479709bfa10a0923dca2b380496e6b2ed4f7a0253': - # Dumb hack for 404 response from peer. Don't log it if 404 since its likely not malicious or a critical error. - logger.warn( - 'Block hash validation failed for ' + - blockHash + ' got ' + tempHash) - else: - removeFromQueue = False # Don't remove from queue if 404 - if removeFromQueue: - try: - del kv.get('blockQueue')[blockHash] # remove from block queue both if success or false - if count == LOG_SKIP_COUNT: - logger.info('%s blocks remaining in queue' % - [len(kv.get('blockQueue'))], terminal=True) - count = 0 - except KeyError: - pass - kv.get('currentDownloading').remove(blockHash) diff --git a/src/communicatorutils/downloadblocks/shoulddownload.py b/src/communicatorutils/downloadblocks/shoulddownload.py deleted file mode 100644 index cd1282de..00000000 --- a/src/communicatorutils/downloadblocks/shoulddownload.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Onionr - Private P2P Communication. - -Check if a block should be downloaded -(if we already have it or its blacklisted or not) -""" -from coredb import blockmetadb -from oldblocks import onionrblacklist -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def should_download(shared_state, block_hash) -> bool: - """Return bool for if a (assumed to exist) block should be downloaded.""" - blacklist = onionrblacklist.OnionrBlackList() - should = True - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - if block_hash in blockmetadb.get_block_list(): - # Don't download block we have - should = False - else: - if blacklist.inBlacklist(block_hash): - # Don't download blacklisted block - should = False - if should is False: - # Remove block from communicator queue if it shouldn't be downloaded - try: - del kv.get('blockQueue')[block_hash] - except KeyError: - pass - return should diff --git a/src/communicatorutils/housekeeping.py b/src/communicatorutils/housekeeping.py deleted file mode 100755 index 212ca259..00000000 --- a/src/communicatorutils/housekeeping.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Onionr - Private P2P Communication. - -Cleanup old Onionr blocks and forward secrecy keys using the communicator. -Ran from a communicator timer usually -""" -import sqlite3 -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV - -import logger -from onionrusers import onionrusers -from onionrutils import epoch -from coredb import blockmetadb, dbfiles -import onionrstorage -from onionrstorage import removeblock -from oldblocks import onionrblacklist -from oldblocks.storagecounter import StorageCounter -from etc.onionrvalues import DATABASE_LOCK_TIMEOUT -from onionrproofs import hashMeetsDifficulty -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - -storage_counter = StorageCounter() - - -def __remove_from_upload(shared_state, block_hash: str): - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - try: - kv.get('blocksToUpload').remove(block_hash) - except ValueError: - pass - - -def __purge_block(shared_state, block_hash, add_to_blacklist = True): - blacklist = None - - removeblock.remove_block(block_hash) - onionrstorage.deleteBlock(block_hash) - __remove_from_upload(shared_state, block_hash) - - if add_to_blacklist: - blacklist = onionrblacklist.OnionrBlackList() - blacklist.addToDB(block_hash) - - -def clean_old_blocks(shared_state): - """Delete expired blocks + old blocks if disk allocation is near full""" - blacklist = onionrblacklist.OnionrBlackList() - # Delete expired blocks - for bHash in blockmetadb.expiredblocks.get_expired_blocks(): - __purge_block(shared_state, bHash, add_to_blacklist=True) - logger.info('Deleted expired block: %s' % (bHash,)) - - while storage_counter.is_full(): - try: - oldest = blockmetadb.get_block_list()[0] - except IndexError: - break - else: - __purge_block(shared_state, bHash, add_to_blacklist=True) - logger.info('Deleted block because of full storage: %s' % (oldest,)) - - -def clean_keys(): - """Delete expired forward secrecy keys""" - conn = sqlite3.connect(dbfiles.user_id_info_db, - timeout=DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - time = epoch.get_epoch() - deleteKeys = [] - - for entry in c.execute( - "SELECT * FROM forwardKeys WHERE expire <= ?", (time,)): - logger.debug('Forward key: %s' % entry[1]) - deleteKeys.append(entry[1]) - - for key in deleteKeys: - logger.debug('Deleting forward key %s' % key) - c.execute("DELETE from forwardKeys where forwardKey = ?", (key,)) - conn.commit() - conn.close() - - onionrusers.deleteExpiredKeys() - - -def clean_blocks_not_meeting_pow(shared_state): - """Clean blocks not meeting min send/rec pow. Used if config.json POW changes""" - block_list = blockmetadb.get_block_list() - for block in block_list: - if not hashMeetsDifficulty(block): - logger.warn( - f"Deleting block {block} because it was stored" + - "with a POW level smaller than current.", terminal=True) - __purge_block(shared_state, block) diff --git a/src/communicatorutils/lookupadders.py b/src/communicatorutils/lookupadders.py deleted file mode 100755 index 788ffa12..00000000 --- a/src/communicatorutils/lookupadders.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Onionr - Private P2P Communication. - -Lookup new peer transport addresses using the communicator -""" -from typing import TYPE_CHECKING -import logger -from onionrutils import stringvalidators -from communicator import peeraction, onlinepeers -from utils import gettransports -import onionrexceptions - - -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def lookup_new_peer_transports_with_communicator(shared_state): - logger.info('Looking up new addresses...') - tryAmount = 1 - newPeers = [] - transports = gettransports.get() - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - - for i in range(tryAmount): - # Download new peer address list from random online peers - if len(newPeers) > 10000: - # Don't get new peers if we have too many queued up - break - try: - peer = onlinepeers.pick_online_peer(kv) - newAdders = peeraction.peer_action(shared_state, peer, action='pex') - except onionrexceptions.OnlinePeerNeeded: - continue - try: - newPeers = newAdders.split(',') - except AttributeError: - pass - else: - # Validate new peers are good format and not already in queue - invalid = [] - for x in newPeers: - x = x.strip() - if not stringvalidators.validate_transport(x) \ - or x in kv.get('newPeers') or x in transports: - # avoid adding if its our address - invalid.append(x) - for x in invalid: - try: - newPeers.remove(x) - except ValueError: - pass - kv.get('newPeers').extend(newPeers) \ No newline at end of file diff --git a/src/communicatorutils/lookupblocks.py b/src/communicatorutils/lookupblocks.py deleted file mode 100755 index d8f73635..00000000 --- a/src/communicatorutils/lookupblocks.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Onionr - Private P2P Communication. - -Lookup new blocks with the communicator using a random connected peer -""" -from typing import TYPE_CHECKING - -from gevent import time - -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV - -import logger -import onionrproofs -from onionrutils import stringvalidators, epoch -from communicator import peeraction, onlinepeers -from coredb.blockmetadb import get_block_list -from utils import reconstructhash -from oldblocks import onionrblacklist -import onionrexceptions -import config -from etc import onionrvalues -from oldblocks.storagecounter import StorageCounter -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - -blacklist = onionrblacklist.OnionrBlackList() -storage_counter = StorageCounter() - - -def lookup_blocks_from_communicator(shared_state: 'TooMany'): - logger.info('Looking up new blocks') - tryAmount = 2 - newBlocks = '' - # List of existing saved blocks - existingBlocks = get_block_list() - triedPeers = [] # list of peers we've tried this time around - # Max amount of *new* block hashes to have in queue - maxBacklog = 1560 - lastLookupTime = 0 # Last time we looked up a particular peer's list - new_block_count = 0 - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - for i in range(tryAmount): - # Defined here to reset it each time, time offset is added later - listLookupCommand = 'getblocklist' - if len(kv.get('blockQueue')) >= maxBacklog: - break - if not kv.get('isOnline'): - break - # check if disk allocation is used - if storage_counter.is_full(): - logger.debug( - 'Not looking up new blocks due to maximum amount of disk used') - break - try: - # select random online peer - peer = onlinepeers.pick_online_peer(kv) - except onionrexceptions.OnlinePeerNeeded: - time.sleep(1) - continue - # if we've already tried all the online peers this time around, stop - if peer in triedPeers: - if len(kv.get('onlinePeers')) == len(triedPeers): - break - else: - continue - triedPeers.append(peer) - - # Get the last time we looked up a peer's stamp, - # to only fetch blocks since then. - # Saved in memory only for privacy reasons - try: - lastLookupTime = kv.get('dbTimestamps')[peer] - except KeyError: - lastLookupTime = epoch.get_epoch() - onionrvalues.DEFAULT_EXPIRE - listLookupCommand += '?date=%s' % (lastLookupTime,) - try: - newBlocks = peeraction.peer_action( - shared_state, - peer, listLookupCommand) # get list of new block hashes - except Exception as error: - logger.warn( - f'Could not get new blocks from {peer}.', - error=error) - newBlocks = False - - if newBlocks != False: # noqa - # if request was a success - for i in newBlocks.split('\n'): - if stringvalidators.validate_hash(i): - i = reconstructhash.reconstruct_hash(i) - # if newline seperated string is valid hash - - # if block does not exist on disk + is not already in queue - if i not in existingBlocks: - if i not in kv.get('blockQueue'): - if onionrproofs.hashMeetsDifficulty(i) and \ - not blacklist.inBlacklist(i): - if len(kv.get('blockQueue')) <= 1000000: - # add blocks to download queue - kv.get('blockQueue')[i] = [peer] - new_block_count += 1 - kv.get('dbTimestamps')[peer] = \ - epoch.get_rounded_epoch(roundS=60) - else: - if peer not in kv.get('blockQueue')[i]: - if len(kv.get('blockQueue')[i]) < 10: - kv.get('blockQueue')[i].append(peer) - if new_block_count > 0: - block_string = "" - if new_block_count > 1: - block_string = "s" - logger.info( - f'Discovered {new_block_count} new block{block_string}', - terminal=True) diff --git a/src/communicatorutils/netcheck.py b/src/communicatorutils/netcheck.py deleted file mode 100755 index b45a4361..00000000 --- a/src/communicatorutils/netcheck.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -Onionr - Private P2P Communication. - -Determine if our node is able to use Tor based -on the status of a communicator instance -and the result of pinging onion http servers -""" -import logger -from utils import netutils -from onionrutils import localcommand, epoch -from . import restarttor - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def net_check(shared_state): - """Check if we are connected to the internet. - - or not when we can't connect to any peers - """ - # for detecting if we have received incoming connections recently - rec = False - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - proxy_port = shared_state.get_by_string("NetController").socksPort - - if len(kv.get('onlinePeers')) == 0: - try: - if (epoch.get_epoch() - int(localcommand.local_command - ('/lastconnect'))) <= 60: - kv.put('isOnline', True) - rec = True - except ValueError: - pass - if not rec and not netutils.check_network(torPort=proxy_port): - if not kv.get('shutdown'): - if not shared_state.get_by_string( - "OnionrCommunicatorDaemon").config.get( - 'general.offline_mode', False): - logger.warn('Network check failed, are you connected to ' + - 'the Internet, and is Tor working? ' + - 'This is usually temporary, but bugs and censorship can cause this to persist, in which case you should report it to beardog [at] mailbox.org', # noqa - terminal=True) - restarttor.restart(shared_state) - kv.put('offlinePeers', []) - kv.put('isOnline', False) - else: - kv.put('isOnline', True) diff --git a/src/communicatorutils/proxypicker.py b/src/communicatorutils/proxypicker.py deleted file mode 100755 index 85dc4d5b..00000000 --- a/src/communicatorutils/proxypicker.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -Onionr - Private P2P Communication. - -Pick a proxy to use based on a peer's address -""" -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def pick_proxy(peer_address): - if peer_address.endswith('.onion'): - return 'tor' - elif peer_address.endswith('.i2p'): - return 'i2p' - raise ValueError( - f"Peer address not ending with acceptable domain: {peer_address}") diff --git a/src/communicatorutils/restarttor.py b/src/communicatorutils/restarttor.py deleted file mode 100644 index 35388d05..00000000 --- a/src/communicatorutils/restarttor.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -Onionr - Private P2P Communication. - -Restart Onionr managed Tor -""" -import netcontroller -import config -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def restart(shared_state): - if not config.get('tor.use_existing_tor', False): - net = shared_state.get(netcontroller.NetController) - net.killTor() - net.startTor() diff --git a/src/communicatorutils/uploadblocks/__init__.py b/src/communicatorutils/uploadblocks/__init__.py deleted file mode 100755 index 31db2955..00000000 --- a/src/communicatorutils/uploadblocks/__init__.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Onionr - Private P2P Communication. - -Upload blocks in the upload queue to peers from the communicator -""" -from typing import TYPE_CHECKING -from time import sleep -from threading import Thread -from secrets import SystemRandom - -from . import sessionmanager - -from onionrtypes import UserID -import logger -from communicatorutils import proxypicker -import onionrexceptions -from oldblocks import onionrblockapi as block -from oldblocks.blockmetadata.fromdata import get_block_metadata_from_data -from onionrutils import stringvalidators, basicrequests -from onionrutils.validatemetadata import validate_metadata -from communicator import onlinepeers -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV - from communicator import OnionrCommunicatorDaemon -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def upload_blocks_from_communicator(shared_state: 'OnionrCommunicatorDaemon'): - """Accept a communicator instance + upload blocks from its upload queue.""" - """when inserting a block, we try to upload - it to a few peers to add some deniability & increase functionality""" - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - - session_manager: sessionmanager.BlockUploadSessionManager - session_manager = shared_state.get( - sessionmanager.BlockUploadSessionManager) - tried_peers: UserID = [] - finishedUploads = [] - - SystemRandom().shuffle(kv.get('blocksToUpload')) - - def remove_from_hidden(bl): - sleep(60) - try: - shared_state.get_by_string( - 'PublicAPI').hideBlocks.remove(bl) - except ValueError: - pass - - if len(kv.get('blocksToUpload')) != 0: - for bl in kv.get('blocksToUpload'): - if not stringvalidators.validate_hash(bl): - logger.warn('Requested to upload invalid block', terminal=True) - return - session = session_manager.add_session(bl) - for _ in range(min(len(kv.get('onlinePeers')), 6)): - try: - peer = onlinepeers.pick_online_peer(kv) - if not block.Block(bl).isEncrypted: - if peer in kv.get('plaintextDisabledPeers'): - logger.info(f"Cannot upload plaintext block to peer that denies it {peer}") # noqa - continue - except onionrexceptions.OnlinePeerNeeded: - continue - try: - session.peer_exists[peer] - continue - except KeyError: - pass - try: - if session.peer_fails[peer] > 3: - continue - except KeyError: - pass - if peer in tried_peers: - continue - tried_peers.append(peer) - url = f'http://{peer}/upload' - try: - data = block.Block(bl).getRaw() - if not data: - logger.warn( - f"Couldn't get data for block in upload list {bl}", - terminal=True) - raise onionrexceptions.NoDataAvailable - try: - def __check_metadata(): - metadata = get_block_metadata_from_data(data)[0] - if not validate_metadata(metadata, data): - logger.warn( - f"Metadata for uploading block not valid {bl}") - raise onionrexceptions.InvalidMetadata - __check_metadata() - except onionrexceptions.DataExists: - pass - except( # noqa - onionrexceptions.NoDataAvailable, - onionrexceptions.InvalidMetadata) as _: - finishedUploads.append(bl) - break - proxy_type = proxypicker.pick_proxy(peer) - logger.info( - f"Uploading block {bl[:8]} to {peer}", terminal=True) - resp = basicrequests.do_post_request( - url, data=data, proxyType=proxy_type, - content_type='application/octet-stream') - if resp is not False: - if resp == 'success': - Thread(target=remove_from_hidden, - args=[bl], daemon=True).start() - session.success() - session.peer_exists[peer] = True - elif resp == 'exists': - session.success() - session.peer_exists[peer] = True - else: - session.fail() - session.fail_peer(peer) - shared_state.get_by_string( - 'OnionrCommunicatorDaemon').getPeerProfileInstance( - peer).addScore(-5) - logger.warn( - f'Failed to upload {bl[:8]}, reason: {resp}', - terminal=True) - else: - session.fail() - session_manager.clean_session() - for x in finishedUploads: - try: - kv.get('blocksToUpload').remove(x) - - shared_state.get_by_string( - 'PublicAPI').hideBlocks.remove(x) - - except ValueError: - pass \ No newline at end of file diff --git a/src/communicatorutils/uploadblocks/mixmate/__init__.py b/src/communicatorutils/uploadblocks/mixmate/__init__.py deleted file mode 100644 index 9b235455..00000000 --- a/src/communicatorutils/uploadblocks/mixmate/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Onionr - Private P2P Communication. - -Perform block mixing -""" -import time -from typing import List - -import onionrtypes -from oldblocks import onionrblockapi - -from .pool import UploadPool -from .pool import PoolFullException - -from etc import onionrvalues - -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" -upload_pool = UploadPool(4) - - -def block_mixer(upload_list: List[onionrtypes.BlockHash], - block_to_mix: onionrtypes.BlockHash): - """Delay and mix block inserts. - - Take a block list and a received/created block and add it - to the said block list - """ - bl = onionrblockapi.Block(block_to_mix) - - try: - if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE: - raise ValueError - except TypeError: - pass - if block_to_mix: - upload_list.append(block_to_mix) diff --git a/src/communicatorutils/uploadblocks/mixmate/pool.py b/src/communicatorutils/uploadblocks/mixmate/pool.py deleted file mode 100644 index f5600082..00000000 --- a/src/communicatorutils/uploadblocks/mixmate/pool.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Onionr - Private P2P Communication. - -Upload pool -""" -from typing import List -from secrets import SystemRandom - -import onionrutils -import onionrtypes -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -class PoolFullException(Exception): - """For when the UploadPool is full. - - Raise when a new hash is attempted to be added - """ - - -class PoolNotReady(Exception): - """Raise when UploadPool pool access is attempted without it being full.""" - - -class AlreadyInPool(Exception): - """Raise when a hash already in pool is attempted to be added again.""" - - -class UploadPool: - """Upload pool for mixing blocks together and delaying uploads.""" - - def __init__(self, pool_size: int): - """Create a new pool with a specified max size. - - Uses private var and getter to avoid direct adding - """ - self._pool: List[onionrtypes.BlockHash] = [] - self._pool_size = pool_size - self.birthday = onionrutils.epoch.get_epoch() - - def add_to_pool(self, item: List[onionrtypes.BlockHash]): - """Add a new hash to the pool. Raise PoolFullException if full.""" - if len(self._pool) >= self._pool_size: - raise PoolFullException - if not onionrutils.stringvalidators.validate_hash(item): - raise ValueError - self._pool.append(item) - - def get_pool(self) -> List[onionrtypes.BlockHash]: - """Get the hash pool in secure random order.""" - if len(self._pool) != self._pool_size: - raise PoolNotReady - - final_pool: List[onionrtypes.BlockHash] = list(self._pool) - SystemRandom().shuffle(final_pool) - - self._pool.clear() - self.birthday = onionrutils.epoch.get_epoch() - return final_pool diff --git a/src/communicatorutils/uploadblocks/session.py b/src/communicatorutils/uploadblocks/session.py deleted file mode 100644 index d9b715bf..00000000 --- a/src/communicatorutils/uploadblocks/session.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Onionr - Private P2P Communication. - -Virtual upload "sessions" for blocks -""" -from typing import Union, Dict - -from onionrtypes import UserID -from onionrutils import stringvalidators -from onionrutils import bytesconverter -from onionrutils import epoch -from utils import reconstructhash -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -class UploadSession: - """Manage statistics for an Onionr block upload session. - - accept a block hash (incl. unpadded) as an argument - """ - - def __init__(self, block_hash: Union[str, bytes]): - block_hash = bytesconverter.bytes_to_str(block_hash) - block_hash = reconstructhash.reconstruct_hash(block_hash) - if not stringvalidators.validate_hash(block_hash): - raise ValueError - - self.start_time = epoch.get_epoch() - self.block_hash = reconstructhash.deconstruct_hash(block_hash) - self.total_fail_count: int = 0 - self.total_success_count: int = 0 - self.peer_fails: Dict[UserID, int] = {} - self.peer_exists: Dict[UserID, bool] = {} - - def fail_peer(self, peer): - try: - self.peer_fails[peer] += 1 - except KeyError: - self.peer_fails[peer] = 0 - - def fail(self): - self.total_fail_count += 1 - - def success(self): - self.total_success_count += 1 diff --git a/src/communicatorutils/uploadblocks/sessionmanager.py b/src/communicatorutils/uploadblocks/sessionmanager.py deleted file mode 100644 index 486823a3..00000000 --- a/src/communicatorutils/uploadblocks/sessionmanager.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Onionr - Private P2P Communication. - -Manager for upload 'sessions' -""" -from typing import List, Union, TYPE_CHECKING -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV - from session import UploadSession - -from onionrutils import bytesconverter -from etc import onionrvalues -from utils import reconstructhash - -from . import session -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -class BlockUploadSessionManager: - """Holds block UploadSession instances. - - Optionally accepts iterable of sessions to added on init - Arguments: old_session: iterable of old UploadSession objects - """ - - def __init__(self, old_sessions: List = None): - if old_sessions is None: - self.sessions = [] - else: - self.sessions = old_sessions - - def add_session(self, - session_or_block: Union[str, - bytes, - session.UploadSession - ] - ) -> session.UploadSession: - """Create (or add existing) block upload session. - - from a str/bytes block hex hash, existing UploadSession - """ - if isinstance(session_or_block, session.UploadSession): - if session_or_block not in self.sessions: - self.sessions.append(session_or_block) - return session_or_block - try: - return self.get_session(session_or_block) - except KeyError: - pass - # convert bytes hash to str - if isinstance(session_or_block, bytes): - session_or_block = bytesconverter.bytes_to_str(session_or_block) - # intentionally not elif - if isinstance(session_or_block, str): - new_session = session.UploadSession(session_or_block) - self.sessions.append(new_session) - return new_session - raise ValueError - - def get_session(self, - block_hash: Union[str, bytes] - ) -> session.UploadSession: - block_hash = reconstructhash.deconstruct_hash( - bytesconverter.bytes_to_str(block_hash)) - for sess in self.sessions: - if sess.block_hash == block_hash: - return sess - raise KeyError - - def clean_session(self, - specific_session: Union[str, 'UploadSession'] = None): - - comm_inst: 'OnionrCommunicatorDaemon' # type: ignore - comm_inst = self._too_many.get_by_string( # pylint: disable=E1101 type: ignore - "OnionrCommunicatorDaemon") - kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string( - "DeadSimpleKV") - sessions_to_delete = [] - if kv.get('startTime') < 120: - return - onlinePeerCount = len(kv.get('onlinePeers')) - - # If we have no online peers right now, - if onlinePeerCount == 0: - return - - for sess in self.sessions: - # if over 50% of peers that were online for a session have - # become unavailable, don't kill sessions - if sess.total_success_count > onlinePeerCount: - if onlinePeerCount / sess.total_success_count >= 0.5: - return - # Clean sessions if they have uploaded to enough online peers - if sess.total_success_count <= 0: - continue - if (sess.total_success_count / onlinePeerCount) >= \ - onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT: - sessions_to_delete.append(sess) - for sess in sessions_to_delete: - try: - self.sessions.remove(session) - except ValueError: - pass - # TODO cleanup to one round of search - # Remove the blocks from the sessions, upload list, - # and waitforshare list - try: - kv.get('blocksToUpload').remove( - reconstructhash.reconstruct_hash(sess.block_hash)) - except ValueError: - pass - try: - kv.get('blocksToUpload').remove(sess.block_hash) - except ValueError: - pass diff --git a/src/coredb/__init__.py b/src/coredb/__init__.py deleted file mode 100644 index d60f2150..00000000 --- a/src/coredb/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import keydb, blockmetadb \ No newline at end of file diff --git a/src/coredb/blockmetadb/__init__.py b/src/coredb/blockmetadb/__init__.py deleted file mode 100644 index 10e36f0a..00000000 --- a/src/coredb/blockmetadb/__init__.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Onionr - Private P2P Communication. - -Work with information relating to blocks stored on the node -""" -import sqlite3 - -from etc import onionrvalues -from . import expiredblocks, updateblockinfo, add -from .. import dbfiles -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - -update_block_info = updateblockinfo.update_block_info -add_to_block_DB = add.add_to_block_DB - - -def get_block_list(date_rec=None, unsaved=False): - """Get list of our blocks.""" - if date_rec is None: - date_rec = 0 - - conn = sqlite3.connect( - dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - - execute = 'SELECT hash FROM hashes WHERE dateReceived' + \ - ' >= ? ORDER BY dateReceived ASC;' - args = (date_rec,) - rows = list() - for row in c.execute(execute, args): - for i in row: - rows.append(i) - conn.close() - return rows - - -def get_block_date(blockHash): - """Return the date a block was received.""" - conn = sqlite3.connect( - dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - - execute = 'SELECT dateReceived FROM hashes WHERE hash=?;' - args = (blockHash,) - for row in c.execute(execute, args): - for i in row: - return int(i) - conn.close() - return None - - -def get_blocks_by_type(blockType, orderDate=True): - """Return a list of blocks by the type.""" - - conn = sqlite3.connect( - dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - - if orderDate: - execute = 'SELECT hash FROM hashes WHERE dataType=? ORDER BY dateReceived;' - else: - execute = 'SELECT hash FROM hashes WHERE dataType=?;' - - args = (blockType,) - rows = list() - - for row in c.execute(execute, args): - for i in row: - rows.append(i) - conn.close() - return rows - diff --git a/src/coredb/blockmetadb/add.py b/src/coredb/blockmetadb/add.py deleted file mode 100644 index 71194b4b..00000000 --- a/src/coredb/blockmetadb/add.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Onionr - Private P2P Communication. - -Add an entry to the block metadata database -""" -import sqlite3 -import secrets -from onionrutils import epoch -from oldblocks import blockmetadata -from etc import onionrvalues -from .. import dbfiles -from onionrexceptions import BlockMetaEntryExists -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def add_to_block_DB(newHash, selfInsert=False, dataSaved=False): - """ - Add a hash value to the block db - - Should be in hex format! - """ - - if blockmetadata.has_block(newHash): - raise BlockMetaEntryExists - conn = sqlite3.connect( - dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - currentTime = epoch.get_epoch() + secrets.randbelow(61) - if selfInsert or dataSaved: - selfInsert = 1 - else: - selfInsert = 0 - data = (newHash, currentTime, '', selfInsert) - c.execute( - 'INSERT INTO hashes (hash, dateReceived, dataType, dataSaved) VALUES(?, ?, ?, ?);', data) - conn.commit() - conn.close() diff --git a/src/coredb/blockmetadb/expiredblocks.py b/src/coredb/blockmetadb/expiredblocks.py deleted file mode 100644 index efaa75a4..00000000 --- a/src/coredb/blockmetadb/expiredblocks.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Onionr - Private P2P Communication. - -Get a list of expired blocks still stored -""" -import sqlite3 -from onionrutils import epoch -from .. import dbfiles -from etc import onionrvalues -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def get_expired_blocks(): - """Return a list of expired blocks.""" - conn = sqlite3.connect( - dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - date = int(epoch.get_epoch()) - - compiled = (date,) - execute = 'SELECT hash FROM hashes WHERE ' + \ - 'expire <= ? ORDER BY dateReceived;' - - rows = list() - for row in c.execute(execute, compiled): - for i in row: - rows.append(i) - conn.close() - return rows diff --git a/src/coredb/blockmetadb/updateblockinfo.py b/src/coredb/blockmetadb/updateblockinfo.py deleted file mode 100644 index 41a67cca..00000000 --- a/src/coredb/blockmetadb/updateblockinfo.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Onionr - Private P2P Communication. - -Update block information in the metadata database by a field name -""" -import sqlite3 - -from .. import dbfiles -from etc import onionrvalues -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def update_block_info(hash, key, data): - """set info associated with a block - - hash - the hash of a block - dateReceived - the date the block was recieved, not necessarily when it was created - decrypted - if we can successfully decrypt the block - dataType - data type of the block - dataFound - if the data has been found for the block - dataSaved - if the data has been saved for the block - sig - defunct - author - defunct - dateClaimed - timestamp claimed inside the block, only as trustworthy as the block author is - expire - expire date for a block - """ - if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', - 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'): - raise ValueError('Key must be in the allowed list') - - conn = sqlite3.connect(dbfiles.block_meta_db, - timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - args = (data, hash) - # Unfortunately, not really possible to prepare this statement - c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args) - conn.commit() - conn.close() - - return True diff --git a/src/coredb/dbfiles.py b/src/coredb/dbfiles.py deleted file mode 100644 index 3d1e8383..00000000 --- a/src/coredb/dbfiles.py +++ /dev/null @@ -1,11 +0,0 @@ -from utils import identifyhome -import filepaths -home = identifyhome.identify_home() -if not home.endswith('/'): home += '/' - -block_meta_db = '%sblock-metadata.db' % (home) -block_data_db = '%s/block-data.db' % (filepaths.block_data_location,) -address_info_db = '%saddress.db' % (home,) -user_id_info_db = '%susers.db' % (home,) -forward_keys_db = '%sforward-keys.db' % (home,) -blacklist_db = '%sblacklist.db' % (home,) \ No newline at end of file diff --git a/src/coredb/keydb/__init__.py b/src/coredb/keydb/__init__.py deleted file mode 100644 index 16d602d5..00000000 --- a/src/coredb/keydb/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import addkeys, listkeys, removekeys, userinfo, transportinfo \ No newline at end of file diff --git a/src/coredb/keydb/addkeys.py b/src/coredb/keydb/addkeys.py deleted file mode 100644 index 9c21c78c..00000000 --- a/src/coredb/keydb/addkeys.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Onionr - Private P2P Communication. - -add user keys or transport addresses -""" -import sqlite3 -from onionrutils import stringvalidators -from . import listkeys -from utils import gettransports -from .. import dbfiles -import onionrcrypto -from etc import onionrvalues -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def add_peer(peerID, name=''): - """Add a public key to the key database (misleading function name).""" - if peerID in listkeys.list_peers() or peerID == onionrcrypto.pub_key: - raise ValueError("specified id is already known") - - # This function simply adds a peer to the DB - if not stringvalidators.validate_pub_key(peerID): - return False - - conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - hashID = "" - c = conn.cursor() - t = (peerID, name, 'unknown', hashID, 0) - - for i in c.execute("SELECT * FROM peers WHERE id = ?;", (peerID,)): - try: - if i[0] == peerID: - conn.close() - return False - except ValueError: - pass - except IndexError: - pass - c.execute('INSERT INTO peers (id, name, dateSeen, hashID, trust) VALUES(?, ?, ?, ?, ?);', t) - conn.commit() - conn.close() - - return True - - -def add_address(address): - """Add an address to the address database (only tor currently)""" - - if type(address) is None or len(address) == 0: - return False - if stringvalidators.validate_transport(address): - if address in gettransports.get(): - return False - conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - # check if address is in database - # this is safe to do because the address is validated above, but we strip some chars here too just in case - address = address.replace('\'', '').replace(';', '').replace('"', '').replace('\\', '') - for i in c.execute("SELECT * FROM adders WHERE address = ?;", (address,)): - try: - if i[0] == address: - conn.close() - return False - except ValueError: - pass - except IndexError: - pass - - t = (address, 1) - c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t) - conn.commit() - conn.close() - - return True - else: - return False diff --git a/src/coredb/keydb/listkeys.py b/src/coredb/keydb/listkeys.py deleted file mode 100644 index ff56b15e..00000000 --- a/src/coredb/keydb/listkeys.py +++ /dev/null @@ -1,86 +0,0 @@ -''' - Onionr - Private P2P Communication - - get lists for user keys or transport addresses -''' -''' - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -''' -import sqlite3 -import logger -from onionrutils import epoch -from etc import onionrvalues -from .. import dbfiles -from . import userinfo, transportinfo -def list_peers(randomOrder=True, getPow=False, trust=0): - ''' - Return a list of public keys (misleading function name) - - randomOrder determines if the list should be in a random order - trust sets the minimum trust to list - ''' - conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - - payload = '' - - if trust not in (0, 1, 2): - logger.error('Tried to select invalid trust.') - return - - if randomOrder: - payload = 'SELECT * FROM peers WHERE trust >= ? ORDER BY RANDOM();' - else: - payload = 'SELECT * FROM peers WHERE trust >= ?;' - - peerList = [] - - for i in c.execute(payload, (trust,)): - try: - if len(i[0]) != 0: - if getPow: - peerList.append(i[0] + '-' + i[1]) - else: - peerList.append(i[0]) - except TypeError: - pass - - conn.close() - - return peerList - -def list_adders(randomOrder=True, i2p=True, recent=0): - ''' - Return a list of transport addresses - ''' - conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - if randomOrder: - addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();') - else: - addresses = c.execute('SELECT * FROM adders;') - addressList = [] - for i in addresses: - if len(i[0].strip()) == 0: - continue - addressList.append(i[0]) - conn.close() - testList = list(addressList) # create new list to iterate - for address in testList: - try: - if recent > 0 and (epoch.get_epoch() - transportinfo.get_address_info(address, 'lastConnect')) > recent: - raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0 - except TypeError: - addressList.remove(address) - return addressList \ No newline at end of file diff --git a/src/coredb/keydb/removekeys.py b/src/coredb/keydb/removekeys.py deleted file mode 100644 index 46d2d641..00000000 --- a/src/coredb/keydb/removekeys.py +++ /dev/null @@ -1,60 +0,0 @@ -''' - Onionr - Private P2P Communication - - Remove a transport address but don't ban them -''' -''' - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -''' -import sqlite3 -from onionrplugins import onionrevents as events -from onionrutils import stringvalidators -from onionrutils import mnemonickeys -from .. import dbfiles -from etc import onionrvalues - -def remove_address(address): - ''' - Remove an address from the address database - ''' - - if stringvalidators.validate_transport(address): - conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - t = (address,) - c.execute('Delete from adders where address=?;', t) - conn.commit() - conn.close() - - #events.event('address_remove', data = {'address': address}, onionr = core_inst.onionrInst) - return True - else: - return False - -def remove_user(pubkey: str)->bool: - ''' - Remove a user from the user database - ''' - pubkey = mnemonickeys.get_base32(pubkey) - if stringvalidators.validate_pub_key(pubkey): - conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - t = (pubkey,) - c.execute('Delete from peers where id=?;', t) - conn.commit() - conn.close() - - return True - else: - return False diff --git a/src/coredb/keydb/transportinfo.py b/src/coredb/keydb/transportinfo.py deleted file mode 100644 index 9f03b323..00000000 --- a/src/coredb/keydb/transportinfo.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Onionr - Private P2P Communication. - -get or set transport address meta information -""" -import sqlite3 -from .. import dbfiles -from etc import onionrvalues -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" -info_numbers = { - 'address': 0, - 'type': 1, - 'knownPeer': 2, - 'speed': 3, - 'success': 4, - 'powValue': 5, - 'failure': 6, - 'lastConnect': 7, - 'trust': 8, - 'introduced': 9} - - -def get_address_info(address, info): - """Get info about an address from its database entry. - - address text, 0 - type int, 1 - knownPeer text, 2 - speed int, 3 - success int, 4 - powValue 5 - failure int 6 - lastConnect 7 - trust 8 - introduced 9 - """ - conn = sqlite3.connect( - dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - - command = (address,) - - info = info_numbers[info] - iter_count = 0 - retVal = '' - - for row in c.execute('SELECT * FROM adders WHERE address=?;', command): - for i in row: - if iter_count == info: - retVal = i - break - else: - iter_count += 1 - conn.close() - - return retVal - - -def set_address_info(address, key, data): - """Update an address for a key.""" - conn = sqlite3.connect( - dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - - command = (data, address) - - if key not in info_numbers.keys(): - raise ValueError( - "Got invalid database key when setting address info, must be in whitelist") - else: - c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command) - conn.commit() - conn.close() \ No newline at end of file diff --git a/src/coredb/keydb/userinfo.py b/src/coredb/keydb/userinfo.py deleted file mode 100644 index 3f055b49..00000000 --- a/src/coredb/keydb/userinfo.py +++ /dev/null @@ -1,73 +0,0 @@ -''' - Onionr - Private P2P Communication - - get or set information about a user id -''' -''' - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -''' -import sqlite3 -from .. import dbfiles -from etc import onionrvalues - -def get_user_info(peer, info): - ''' - Get info about a peer from their database entry - - id text 0 - name text, 1 - adders text, 2 - dateSeen not null, 3 - trust int 4 - hashID text 5 - ''' - conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - - command = (peer,) - infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'trust': 4, 'hashID': 5} - info = infoNumbers[info] - iterCount = 0 - retVal = '' - - for row in c.execute('SELECT * FROM peers WHERE id=?;', command): - for i in row: - if iterCount == info: - retVal = i - break - else: - iterCount += 1 - - conn.close() - - return retVal - -def set_peer_info(peer, key, data): - ''' - Update a peer for a key - ''' - - conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - - command = (data, peer) - - if key not in ('id', 'name', 'pubkey', 'forwardKey', 'dateSeen', 'trust'): - raise ValueError("Got invalid database key when setting peer info") - - c.execute('UPDATE peers SET ' + key + ' = ? WHERE id=?', command) - conn.commit() - conn.close() - -set_user_info = set_peer_info \ No newline at end of file diff --git a/src/etc/cleanup/__init__.py b/src/etc/cleanup/__init__.py index ac6bab3f..ba913d9b 100644 --- a/src/etc/cleanup/__init__.py +++ b/src/etc/cleanup/__init__.py @@ -33,7 +33,6 @@ def delete_run_files(): Test: test_cleanup.py """ - _safe_remove(filepaths.public_API_host_file) _safe_remove(filepaths.private_API_host_file) _safe_remove(filepaths.daemon_mark_file) _safe_remove(filepaths.lock_file) diff --git a/src/etc/humanreadabletime.py b/src/etc/humanreadabletime.py index 7e3ff3bc..bd4fed0f 100755 --- a/src/etc/humanreadabletime.py +++ b/src/etc/humanreadabletime.py @@ -1,22 +1,24 @@ -''' - Onionr - Private P2P Communication +""" +Onionr - Private P2P Communication. - human_readable_time takes integer seconds and returns a human readable string -''' -''' - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. +human_readable_time takes integer seconds and returns a human readable string +""" +""" +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - You should have received a copy of the GNU General Public License - along with this program. If not, see . -''' def human_readable_time(seconds): build = '' diff --git a/src/etc/onionrvalues.py b/src/etc/onionrvalues.py index 43c7dead..676009bd 100755 --- a/src/etc/onionrvalues.py +++ b/src/etc/onionrvalues.py @@ -20,23 +20,16 @@ import filepaths You should have received a copy of the GNU General Public License along with this program. If not, see . """ -DENIABLE_PEER_ADDRESS = "OVPCZLOXD6DC5JHX4EQ3PSOGAZ3T24F75HQLIUZSDSMYPEOXCPFA" PASSWORD_LENGTH = 25 ONIONR_TAGLINE = 'Private P2P Communication - GPLv3 - https://Onionr.net' -ONIONR_VERSION = '8.0.2' -ONIONR_VERSION_CODENAME = 'Genesis' +ONIONR_VERSION = '9.0.0' +ONIONR_VERSION_CODENAME = 'Nexus' ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION) API_VERSION = '2' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you. MIN_PY_VERSION = 7 # min version of 7 so we can take advantage of non-cyclic type hints DEVELOPMENT_MODE = False IS_QUBES = False -"""limit type length for a block (soft enforced, ignored if invalid but block still stored).""" -MAX_BLOCK_TYPE_LENGTH = 15 -"""limit clock timestamp for new blocks to be skewed in the future in seconds, -2 minutes to allow plenty of time for slow block insertion and slight clock inaccuracies""" -MAX_BLOCK_CLOCK_SKEW = 120 -"""Onionr user IDs are ed25519 keys, which are always 32 bytes in length""" -MAIN_PUBLIC_KEY_SIZE = 32 + ORIG_RUN_DIR_ENV_VAR = 'ORIG_ONIONR_RUN_DIR' DATABASE_LOCK_TIMEOUT = 60 @@ -46,27 +39,9 @@ MIN_BLOCK_UPLOAD_PEER_PERCENT = 0.1 WSGI_SERVER_REQUEST_TIMEOUT_SECS = 120 -MAX_NEW_PEER_QUEUE = 1000 BLOCK_EXPORT_FILE_EXT = '.onionr' -# Begin OnionrValues migrated values - -"""30 days is plenty of time for someone to decide to renew a block""" -DEFAULT_EXPIRE = 2678400 -# Metadata header section length limits, in bytes -BLOCK_METADATA_LENGTHS = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, - 'n': 1000, 'c': 1000, 'encryptType': 4, 'expire': 14} - -# Pool Eligibility Max Age -BLOCK_POOL_MAX_AGE = 300 - -"""Public key that signs MOTD messages shown in the web UI""" -MOTD_SIGN_KEY = "TRH763JURNY47QPBTTQ4LLPYCYQK6Q5YA33R6GANKZK5C5DKCIGQ" - -"""Public key that signs update notifications.""" -UPDATE_SIGN_KEY = "TRH763JURNY47QPBTTQ4LLPYCYQK6Q5YA33R6GANKZK5C5DKCIGQ" - if os.path.exists(filepaths.daemon_mark_file): SCRIPT_NAME = 'start-daemon.sh' diff --git a/src/filepaths/__init__.py b/src/filepaths/__init__.py index 013c5e93..ab95bbb8 100644 --- a/src/filepaths/__init__.py +++ b/src/filepaths/__init__.py @@ -6,15 +6,11 @@ if not home.endswith('/'): home += '/' app_root = os.path.dirname(os.path.realpath(__file__)) + '/../../' usage_file = home + 'disk-usage.txt' block_data_location = home + 'blocks/' -contacts_location = home + 'contacts/' -public_API_host_file = home + 'public-host.txt' + private_API_host_file = home + 'private-host.txt' -bootstrap_file_location = 'static-data/bootstrap-nodes.txt' -data_nonce_file = home + 'block-nonces.dat' -forward_keys_file = home + 'forward-keys.db' + cached_storage = home + 'cachedstorage.dat' announce_cache = home + 'announcecache.dat' -export_location = home + 'block-export/' upload_list = home + 'upload-list.json' config_file = home + 'config.json' daemon_mark_file = home + '/daemon-true.txt' @@ -22,21 +18,12 @@ lock_file = home + 'onionr.lock' main_safedb = home + "main.safe.db" -site_cache = home + 'onionr-sites.txt' - -tor_hs_loc = home + 'hs/' -tor_hs_address_file = home + 'hs/hostname' - data_nonce_file = home + 'block-nonces.dat' -keys_file = home + 'keys.txt' - onboarding_mark_file = home + 'onboarding-completed' log_file = home + 'onionr.log' -ephemeral_services_file = home + 'ephemeral-services.list' - restarting_indicator = home + "is-restarting" secure_erase_key_file = home + "erase-key" diff --git a/src/httpapi/README.md b/src/httpapi/README.md index a4cf7eeb..4905c955 100755 --- a/src/httpapi/README.md +++ b/src/httpapi/README.md @@ -8,6 +8,4 @@ configapi: manage onionr configuration from the client http api friendsapi: add, remove and list friends from the client http api -miscpublicapi: misculanious onionr network interaction from the **public** httpapi, such as announcements, block fetching and uploading. - profilesapi: work in progress in returning a profile page for an Onionr user \ No newline at end of file diff --git a/src/httpapi/__init__.py b/src/httpapi/__init__.py index ee69a8aa..a63dfa81 100755 --- a/src/httpapi/__init__.py +++ b/src/httpapi/__init__.py @@ -7,6 +7,7 @@ import onionrplugins import config from . import wrappedfunctions +from . import fdsafehandler """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/httpapi/apiutils/__init__.py b/src/httpapi/apiutils/__init__.py index cab13c98..d8c4f227 100644 --- a/src/httpapi/apiutils/__init__.py +++ b/src/httpapi/apiutils/__init__.py @@ -1,3 +1 @@ -from . import shutdown, setbindip, getblockdata - -GetBlockData = getblockdata.GetBlockData \ No newline at end of file +from . import shutdown, setbindip \ No newline at end of file diff --git a/src/httpapi/apiutils/getblockdata.py b/src/httpapi/apiutils/getblockdata.py deleted file mode 100644 index 74988138..00000000 --- a/src/httpapi/apiutils/getblockdata.py +++ /dev/null @@ -1,38 +0,0 @@ -import ujson as json - -from oldblocks import onionrblockapi -from onionrutils import bytesconverter, stringvalidators -import onionrexceptions -class GetBlockData: - def __init__(self, client_api_inst=None): - return - - def get_block_data(self, bHash, decrypt=False, raw=False, headerOnly=False): - if not stringvalidators.validate_hash(bHash): - raise onionrexceptions.InvalidHexHash( - "block hash not valid hash format") - bl = onionrblockapi.Block(bHash) - if decrypt: - bl.decrypt() - if bl.isEncrypted and not bl.decrypted: - raise ValueError - - if not raw: - if not headerOnly: - retData = {'meta':bl.bheader, 'metadata': bl.bmetadata, 'content': bl.bcontent} - for x in list(retData.keys()): - try: - retData[x] = retData[x].decode() - except AttributeError: - pass - else: - validSig = False - signer = bytesconverter.bytes_to_str(bl.signer) - if bl.isSigned() and stringvalidators.validate_pub_key(signer) and bl.isSigner(signer): - validSig = True - bl.bheader['validSig'] = validSig - bl.bheader['meta'] = '' - retData = {'meta': bl.bheader, 'metadata': bl.bmetadata} - return json.dumps(retData) - else: - return bl.raw \ No newline at end of file diff --git a/src/httpapi/apiutils/shutdown.py b/src/httpapi/apiutils/shutdown.py index 63bdb440..07e53ff8 100644 --- a/src/httpapi/apiutils/shutdown.py +++ b/src/httpapi/apiutils/shutdown.py @@ -24,7 +24,6 @@ shutdown_bp = Blueprint('shutdown', __name__) def shutdown(client_api_inst): try: - client_api_inst.publicAPI.httpServer.stop() client_api_inst.httpServer.stop() except AttributeError: pass diff --git a/src/httpapi/friendsapi/__init__.py b/src/httpapi/friendsapi/__init__.py deleted file mode 100755 index 329ea7f4..00000000 --- a/src/httpapi/friendsapi/__init__.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Onionr - Private P2P Communication. - -This file creates http endpoints for friend management -""" -import ujson as json - -from onionrusers import contactmanager -from flask import Blueprint, Response, request, abort, redirect -from coredb import keydb -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" -friends = Blueprint('friends', __name__) - - -@friends.route('/friends/list') -def list_friends(): - pubkey_list = {} - friend_list = contactmanager.ContactManager.list_friends() - for friend in friend_list: - pubkey_list[friend.publicKey] = {'name': friend.get_info('name')} - return json.dumps(pubkey_list) - - -@friends.route('/friends/add/', methods=['POST']) -def add_friend(pubkey): - contactmanager.ContactManager(pubkey, saveUser=True).setTrust(1) - try: - return redirect(request.referrer + '#' + request.form['token']) - except TypeError: - return Response( - "Added, but referrer not set, cannot return to friends page") - - -@friends.route('/friends/remove/', methods=['POST']) -def remove_friend(pubkey): - contactmanager.ContactManager(pubkey).setTrust(0) - contactmanager.ContactManager(pubkey).delete_contact() - keydb.removekeys.remove_user(pubkey) - try: - return redirect(request.referrer + '#' + request.form['token']) - except TypeError: - return Response( - "Friend removed, but referrer not set, cannot return to page") - - -@friends.route('/friends/setinfo//', methods=['POST']) -def set_info(pubkey, key): - data = request.form['data'] - contactmanager.ContactManager(pubkey).set_info(key, data) - try: - return redirect(request.referrer + '#' + request.form['token']) - except TypeError: - return Response( - "Info set, but referrer not set, cannot return to friends page") - - -@friends.route('/friends/getinfo//') -def get_info(pubkey, key): - ret_data = contactmanager.ContactManager(pubkey).get_info(key) - if ret_data is None: - abort(404) - else: - return ret_data \ No newline at end of file diff --git a/src/httpapi/insertblock.py b/src/httpapi/insertblock.py deleted file mode 100644 index db12d6ee..00000000 --- a/src/httpapi/insertblock.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Onionr - Private P2P Communication. - -Create blocks with the client api server -""" -import ujson as json -import threading -from typing import TYPE_CHECKING - -from flask import Blueprint, Response, request, g - -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV - -import oldblocks -from onionrcrypto import hashers -from onionrutils import bytesconverter -from onionrutils import mnemonickeys -from onionrtypes import JSONSerializable - -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" -ib = Blueprint('insertblock', __name__) - - -@ib.route('/insertblock', methods=['POST']) -def client_api_insert_block(): - insert_data: JSONSerializable = request.get_json(force=True) - message = insert_data['message'] - message_hash = bytesconverter.bytes_to_str(hashers.sha3_hash(message)) - kv: 'DeadSimpleKV' = g.too_many.get_by_string('DeadSimpleKV') - - # Detect if message (block body) is not specified - if type(message) is None: - return 'failure due to unspecified message', 400 - - # Detect if block with same message is already being inserted - if message_hash in kv.get('generating_blocks'): - return 'failure due to duplicate insert', 400 - else: - kv.get('generating_blocks').append(message_hash) - - encrypt_type = '' - sign = True - meta = {} - to = '' - try: - if insert_data['encrypt']: - to = insert_data['to'].strip() - if "-" in to: - to = mnemonickeys.get_base32(to) - encrypt_type = 'asym' - except KeyError: - pass - try: - if not insert_data['sign']: - sign = False - except KeyError: - pass - try: - bType = insert_data['type'] - except KeyError: - bType = 'bin' - try: - meta = json.loads(insert_data['meta']) - except KeyError: - pass - - try: - # Setting in the mail UI is for if forward secrecy is *enabled* - disable_forward_secrecy = not insert_data['forward'] - except KeyError: - disable_forward_secrecy = False - - threading.Thread( - target=oldblocks.insert, args=(message,), - kwargs={'header': bType, 'encryptType': encrypt_type, - 'sign': sign, 'asymPeer': to, 'meta': meta, - 'disableForward': disable_forward_secrecy}).start() - return Response('success') diff --git a/src/httpapi/miscclientapi/__init__.py b/src/httpapi/miscclientapi/__init__.py index dadcb97c..1d917684 100644 --- a/src/httpapi/miscclientapi/__init__.py +++ b/src/httpapi/miscclientapi/__init__.py @@ -1 +1 @@ -from . import getblocks, staticfiles, endpoints, motd \ No newline at end of file +from . import staticfiles, endpoints \ No newline at end of file diff --git a/src/httpapi/miscclientapi/addpeer.py b/src/httpapi/miscclientapi/addpeer.py deleted file mode 100644 index c01a8098..00000000 --- a/src/httpapi/miscclientapi/addpeer.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Onionr - Private P2P Communication. - -add a transport address to the db -""" -from onionrutils.stringvalidators import validate_transport -from coredb.keydb.addkeys import add_address -from coredb.keydb.listkeys import list_adders -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def add_peer(peer): - - if peer in list_adders(): - return "already added" - if add_address(peer): - return "success" - else: - return "failure, invalid address" diff --git a/src/httpapi/miscclientapi/endpoints.py b/src/httpapi/miscclientapi/endpoints.py index 6fbfcade..0f3effd5 100644 --- a/src/httpapi/miscclientapi/endpoints.py +++ b/src/httpapi/miscclientapi/endpoints.py @@ -10,35 +10,28 @@ from sys import stdout as sys_stdout from flask import Response, Blueprint, request, send_from_directory, abort from flask import g from gevent import sleep -import unpaddedbase32 from httpapi import apiutils -import onionrcrypto import config -from netcontroller import NetController from onionrstatistics.serializeddata import SerializedData -from onionrutils import mnemonickeys from onionrutils import bytesconverter from etc import onionrvalues from utils import reconstructhash -from utils.gettransports import get as get_tor -from .addpeer import add_peer """ - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. - You should have received a copy of the GNU General Public License - along with this program. If not, see . +You should have received a copy of the GNU General Public License +along with this program. If not, see . """ -pub_key = onionrcrypto.pub_key.replace('=', '') SCRIPT_NAME = os.path.dirname(os.path.realpath(__file__)) + \ f'/../../../{onionrvalues.SCRIPT_NAME}' @@ -49,17 +42,6 @@ class PrivateEndpoints: private_endpoints_bp = Blueprint('privateendpoints', __name__) self.private_endpoints_bp = private_endpoints_bp - @private_endpoints_bp.route('/addpeer/', methods=['post']) - def add_peer_endpoint(name): - result = add_peer(name) - if result == "success": - return Response("success") - else: - if "already" in result: - return Response(result, 409) - else: - return Response(result, 400) - @private_endpoints_bp.route('/www/', endpoint='www') def wwwPublic(path): if not config.get("www.private.run", True): @@ -75,34 +57,11 @@ class PrivateEndpoints: def get_is_atty(): return Response(str(sys_stdout.isatty()).lower()) - @private_endpoints_bp.route('/hitcount') - def get_hit_count(): - return Response(str(client_api.publicAPI.hitCount)) - @private_endpoints_bp.route('/ping') def ping(): # Used to check if client api is working return Response("pong!") - @private_endpoints_bp.route('/lastconnect') - def last_connect(): - return Response(str(client_api.publicAPI.lastRequest)) - - @private_endpoints_bp.route('/waitforshare/', methods=['post']) - def wait_for_share(name): - """Prevent the **public** api from sharing blocks. - - Used for blocks we created usually - """ - if not name.isalnum(): - raise ValueError('block hash needs to be alpha numeric') - name = reconstructhash.reconstruct_hash(name) - if name in client_api.publicAPI.hideBlocks: - return Response("will be removed") - else: - client_api.publicAPI.hideBlocks.append(name) - return Response("added") - @private_endpoints_bp.route('/shutdown') def shutdown(): return apiutils.shutdown.shutdown(client_api) @@ -112,10 +71,6 @@ class PrivateEndpoints: subprocess.Popen([SCRIPT_NAME, 'restart']) return Response("bye") - @private_endpoints_bp.route('/gethidden') - def get_hidden_blocks(): - return Response('\n'.join(client_api.publicAPI.hideBlocks)) - @private_endpoints_bp.route('/getstats') def get_stats(): """Return serialized node statistics.""" @@ -132,24 +87,6 @@ class PrivateEndpoints: def show_uptime(): return Response(str(client_api.getUptime())) - @private_endpoints_bp.route('/getActivePubkey') - def get_active_pubkey(): - return Response(pub_key) - - @private_endpoints_bp.route('/getHumanReadable') - def get_human_readable_default(): - return Response(mnemonickeys.get_human_readable_ID()) - - @private_endpoints_bp.route('/getHumanReadable/') - def get_human_readable(name): - name = unpaddedbase32.repad(bytesconverter.str_to_bytes(name)) - return Response(mnemonickeys.get_human_readable_ID(name)) - - @private_endpoints_bp.route('/getBase32FromHumanReadable/') - def get_base32_from_human_readable(words): - return Response( - bytesconverter.bytes_to_str(mnemonickeys.get_base32(words))) - @private_endpoints_bp.route('/setonboarding', methods=['POST']) def set_onboarding(): return Response( @@ -159,42 +96,6 @@ class PrivateEndpoints: def get_os_system(): return Response(platform.system().lower()) - @private_endpoints_bp.route('/gettorsocks') - def get_tor_socks(): - while True: - try: - return Response( - str( - g.too_many.get_by_string( - 'NetController').socksPort)) - except KeyError: - sleep(0.1) - - @private_endpoints_bp.route('/torready') - def is_tor_ready(): - """If Tor is starting up, the web UI is not ready to be used.""" - try: - return Response( - str(g.too_many.get_by_string('NetController').readyState).lower()) - except KeyError: - return Response("false") - - @private_endpoints_bp.route('/gettoraddress') - def get_tor_address(): - """Return public Tor v3 Onion address for this node""" - if not config.get('general.security_level', 0) == 0: - abort(404) - return Response(get_tor()[0]) - - @private_endpoints_bp.route('/getgeneratingblocks') - def get_generating_blocks() -> Response: - return Response( - ','.join( - g.too_many.get_by_string('DeadSimpleKV').get( - 'generating_blocks' - )) - ) - @private_endpoints_bp.route('/getblockstoupload') def get_blocks_to_upload() -> Response: return Response( diff --git a/src/httpapi/miscclientapi/getblocks.py b/src/httpapi/miscclientapi/getblocks.py deleted file mode 100644 index 18f4fc9f..00000000 --- a/src/httpapi/miscclientapi/getblocks.py +++ /dev/null @@ -1,65 +0,0 @@ -''' - Onionr - Private P2P Communication - - Create blocks with the client api server -''' -''' - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -''' -from flask import Blueprint, Response, abort -from oldblocks import onionrblockapi -from .. import apiutils -from onionrutils import stringvalidators -from coredb import blockmetadb - -client_get_block = apiutils.GetBlockData() - -client_get_blocks = Blueprint('miscclient', __name__) - -@client_get_blocks.route('/getblocksbytype/') -def get_blocks_by_type_endpoint(name): - blocks = blockmetadb.get_blocks_by_type(name) - return Response(','.join(blocks)) - -@client_get_blocks.route('/getblockbody/') -def getBlockBodyData(name): - resp = '' - if stringvalidators.validate_hash(name): - try: - resp = onionrblockapi.Block(name, decrypt=True).bcontent - except TypeError: - pass - else: - abort(404) - return Response(resp) - -@client_get_blocks.route('/getblockdata/') -def getData(name): - resp = "" - if stringvalidators.validate_hash(name): - if name in blockmetadb.get_block_list(): - try: - resp = client_get_block.get_block_data(name, decrypt=True) - except ValueError: - pass - else: - abort(404) - else: - abort(404) - return Response(resp) - -@client_get_blocks.route('/getblockheader/') -def getBlockHeader(name): - resp = client_get_block.get_block_data(name, decrypt=True, headerOnly=True) - return Response(resp) \ No newline at end of file diff --git a/src/httpapi/miscclientapi/motd/__init__.py b/src/httpapi/miscclientapi/motd/__init__.py deleted file mode 100644 index c467d097..00000000 --- a/src/httpapi/miscclientapi/motd/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -from flask import Blueprint -from flask import Response -import unpaddedbase32 - -from coredb import blockmetadb -import oldblocks -from etc import onionrvalues -import config -from onionrutils import bytesconverter - -bp = Blueprint('motd', __name__) - -signer = config.get("motd.motd_key", onionrvalues.MOTD_SIGN_KEY) - -@bp.route('/getmotd') -def get_motd()->Response: - motds = blockmetadb.get_blocks_by_type("motd") - newest_time = 0 - message = "No MOTD currently present." - for x in motds: - bl = oldblocks.onionrblockapi.Block(x) - if not bl.verifySig() or bl.signer != bytesconverter.bytes_to_str(unpaddedbase32.repad(bytesconverter.str_to_bytes(signer))): continue - if not bl.isSigner(signer): continue - if bl.claimedTime > newest_time: - newest_time = bl.claimedTime - message = bl.bcontent - return Response(message, headers={"Content-Type": "text/plain"}) diff --git a/src/httpapi/miscpublicapi/__init__.py b/src/httpapi/miscpublicapi/__init__.py deleted file mode 100755 index 2f5f9c56..00000000 --- a/src/httpapi/miscpublicapi/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from . import announce, upload, getblocks, endpoints - -announce = announce.handle_announce # endpoint handler for accepting peer announcements -upload = upload.accept_upload # endpoint handler for accepting public uploads -public_block_list = getblocks.get_public_block_list # endpoint handler for getting block lists -public_get_block_data = getblocks.get_block_data # endpoint handler for responding to peers requests for block data \ No newline at end of file diff --git a/src/httpapi/miscpublicapi/announce.py b/src/httpapi/miscpublicapi/announce.py deleted file mode 100755 index 29a78a43..00000000 --- a/src/httpapi/miscpublicapi/announce.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Onionr - Private P2P Communication. - -Handle announcements to the public API server -""" -from flask import Response, g -import deadsimplekv - -import logger -from etc import onionrvalues -from onionrutils import stringvalidators, bytesconverter -import filepaths -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def handle_announce(request): - """accept announcement posts, validating POW - clientAPI should be an instance of the clientAPI server running, - request is a instance of a flask request - """ - resp = 'failure' - newNode = '' - - try: - newNode = request.form['node'].encode() - except KeyError: - logger.warn('No node specified for upload') - else: - newNode = bytesconverter.bytes_to_str(newNode) - announce_queue = deadsimplekv.DeadSimpleKV(filepaths.announce_cache) - announce_queue_list = announce_queue.get('new_peers') - if announce_queue_list is None: - announce_queue_list = [] - else: - if len(announce_queue_list) >= onionrvalues.MAX_NEW_PEER_QUEUE: - newNode = '' - - if stringvalidators.validate_transport(newNode) and \ - newNode not in announce_queue_list: - g.shared_state.get( - deadsimplekv.DeadSimpleKV).get('newPeers').append(newNode) - announce_queue.put('new_peers', - announce_queue_list.append(newNode)) - announce_queue.flush() - resp = 'Success' - - resp = Response(resp) - if resp == 'failure': - return resp, 406 - return resp \ No newline at end of file diff --git a/src/httpapi/miscpublicapi/endpoints.py b/src/httpapi/miscpublicapi/endpoints.py deleted file mode 100644 index 9c575c07..00000000 --- a/src/httpapi/miscpublicapi/endpoints.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Onionr - Private P2P Communication. - -Misc public API endpoints too small to need their own file -and that need access to the public api inst -""" -from flask import Response, Blueprint, request, send_from_directory, abort, g -from . import getblocks, upload, announce -from coredb import keydb -import config -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -class PublicEndpoints: - def __init__(self, public_api): - - public_endpoints_bp = Blueprint('publicendpoints', __name__) - self.public_endpoints_bp = public_endpoints_bp - - @public_endpoints_bp.route('/') - def banner(): - # Display info to people who visit a node address in their browser - try: - with open('../static-data/index.html', 'r') as html: - resp = Response(html.read(), mimetype='text/html') - except FileNotFoundError: - resp = Response("") - return resp - - @public_endpoints_bp.route('/getblocklist') - def get_block_list(): - """Get a list of blocks, optionally filtered by epoch time stamp, - excluding those hidden""" - return getblocks.get_public_block_list(public_api, request) - - @public_endpoints_bp.route('/getdata/') - def get_block_data(name): - # Share data for a block if we have it and it isn't hidden - return getblocks.get_block_data(public_api, name) - - @public_endpoints_bp.route('/www/') - def www_public(path): - # A way to share files directly over your .onion - if not config.get("www.public.run", True): - abort(403) - return send_from_directory( - config.get('www.public.path', 'static-data/www/public/'), path) - - - @public_endpoints_bp.route('/plaintext') - def plaintext_enabled_endpoint(): - return Response(str(config.get("general.store_plaintext_blocks", True)).lower()) - - @public_endpoints_bp.route('/ping') - def ping(): - # Endpoint to test if nodes are up - return Response("pong!") - - @public_endpoints_bp.route('/pex') - def peer_exchange(): - response = ','.join(keydb.listkeys.list_adders(recent=3600)) - if len(response) == 0: - response = '' - return Response(response) - - @public_endpoints_bp.route('/announce', methods=['post']) - def accept_announce(): - """Accept announcements with pow token to prevent spam""" - g.shared_state = public_api._too_many - resp = announce.handle_announce(request) - return resp - - @public_endpoints_bp.route('/upload', methods=['post']) - def upload_endpoint(): - """Accept file uploads. - In the future this will be done more often than on creation - to speed up block sync - """ - return upload.accept_upload(request) diff --git a/src/httpapi/miscpublicapi/getblocks.py b/src/httpapi/miscpublicapi/getblocks.py deleted file mode 100755 index 95efc34f..00000000 --- a/src/httpapi/miscpublicapi/getblocks.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Onionr - Private P2P Communication. - -Public endpoints to get block data and lists -""" -from flask import Response, abort - -import config -from onionrutils import bytesconverter, stringvalidators -from coredb import blockmetadb -from utils import reconstructhash -from oldblocks import BlockList -from oldblocks.onionrblockapi import Block -from .. import apiutils -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def get_public_block_list(public_API, request): - # Provide a list of our blocks, with a date offset - date_adjust = request.args.get('date') - type_filter = request.args.get('type') - b_list = blockmetadb.get_block_list(date_rec=date_adjust) - share_list = '' - if config.get('general.hide_created_blocks', True): - for b in public_API.hideBlocks: - if b in b_list: - # Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block - b_list.remove(b) - for b in b_list: - if type_filter: - if Block(b, decrypt=False).getType() != type_filter: - continue - share_list += '%s\n' % (reconstructhash.deconstruct_hash(b),) - return Response(share_list) - - -def get_block_data(public_API, b_hash): - """return block data by hash unless we are hiding it""" - resp = '' - b_hash = reconstructhash.reconstruct_hash(b_hash) - if stringvalidators.validate_hash(b_hash): - if not config.get('general.hide_created_blocks', True) \ - or b_hash not in public_API.hideBlocks: - if b_hash in public_API._too_many.get(BlockList).get(): - block = apiutils.GetBlockData().get_block_data( - b_hash, raw=True, decrypt=False) - try: - # Encode in case data is binary - block = block.encode('utf-8') - except AttributeError: - # 404 if no block data - if not block: - abort(404) - if not len(block): - abort(404) - resp = block - if len(resp) == 0: - abort(404) - resp = "" - # Has to be octet stream, otherwise binary data fails hash check - return Response(resp, mimetype='application/octet-stream') diff --git a/src/httpapi/miscpublicapi/upload.py b/src/httpapi/miscpublicapi/upload.py deleted file mode 100755 index 6535d624..00000000 --- a/src/httpapi/miscpublicapi/upload.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Onionr - Private P2P Communication. - -Accept block uploads to the public API server -""" -import sys - -from gevent import spawn -from flask import Response -from flask import abort -from flask import g - -from onionrutils import localcommand -from oldblocks import blockimporter -import onionrexceptions -import logger -import config - -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def accept_upload(request): - """Accept uploaded blocks to our public Onionr protocol API server""" - resp = 'failure' - data = request.get_data() - data_size = sys.getsizeof(data) - b_hash = None - if data_size < 30: - resp = 'size' - elif data_size < 100000000: - try: - b_hash = blockimporter.import_block_from_data(data) - if b_hash: - # Upload mixing is where a node will hide and reupload a block - # to act like it is also a creator - # This adds deniability but is very slow - if g.too_many.get_by_string( - "DeadSimpleKV").get('onlinePeers') and \ - config.get('general.upload_mixing', False): - spawn( - localcommand.local_command, - '/daemon-event/upload_event', - post=True, - is_json=True, - post_data={'block': b_hash} - ).get(timeout=10) - resp = 'success' - else: - resp = 'failure' - logger.warn( - f'Error encountered importing uploaded block {b_hash}') - except onionrexceptions.BlacklistedBlock: - logger.debug('uploaded block is blacklisted') - resp = 'failure' - except onionrexceptions.InvalidProof: - resp = 'proof' - except onionrexceptions.DataExists: - resp = 'exists' - except onionrexceptions.PlaintextNotSupported: - logger.debug(f"attempted plaintext upload to us: {b_hash}") - resp = 'failure' - except onionrexceptions.InvalidMetadata: - logger.debug( - f'uploaded block {b_hash} has invalid metadata') - resp = 'failure' - if resp == 'failure': - abort(400) - elif resp == 'size': - resp = Response(resp, 400) - logger.warn( - f'Error importing uploaded block, invalid size {b_hash}') - elif resp == 'proof': - resp = Response(resp, 400) - if b_hash: - logger.warn( - f'Error importing uploaded block, invalid proof {b_hash}') - else: - logger.warn( - 'Error importing uploaded block, invalid proof') - else: - resp = Response(resp) - return resp diff --git a/src/httpapi/onionrsitesapi/__init__.py b/src/httpapi/onionrsitesapi/__init__.py deleted file mode 100644 index 2b431ce6..00000000 --- a/src/httpapi/onionrsitesapi/__init__.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Onionr - Private P2P Communication. - -view and interact with onionr sites -""" -import base64 -import binascii -import mimetypes - -import unpaddedbase32 - -from flask import Blueprint, Response, request, abort - -from oldblocks import onionrblockapi -import onionrexceptions -from onionrutils import stringvalidators -from onionrutils import mnemonickeys -from . import sitefiles -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -site_api = Blueprint('siteapi', __name__) - -@site_api.route('/site//', endpoint='site') -def site(name: str)->Response: - """Accept a site 'name', if pubkey then show multi-page site, if hash show single page site""" - resp: str = 'Not Found' - mime_type = 'text/html' - - # If necessary convert the name to base32 from mnemonic - if mnemonickeys.DELIMITER in name: - name = mnemonickeys.get_base32(name) - - # Now make sure the key is regardless a valid base32 format ed25519 key (readding padding if necessary) - if stringvalidators.validate_pub_key(name): - name = unpaddedbase32.repad(name) - resp = sitefiles.get_file(name, 'index.html') - - elif stringvalidators.validate_hash(name): - try: - resp = onionrblockapi.Block(name).bcontent - except onionrexceptions.NoDataAvailable: - abort(404) - except TypeError: - pass - try: - resp = base64.b64decode(resp) - except binascii.Error: - pass - if resp == 'Not Found' or not resp: - abort(404) - return Response(resp) - -@site_api.route('/site//', endpoint='siteFile') -def site_file(name: str, file: str)->Response: - """Accept a site 'name', if pubkey then show multi-page site, if hash show single page site""" - resp: str = 'Not Found' - mime_type = mimetypes.MimeTypes().guess_type(file)[0] - - # If necessary convert the name to base32 from mnemonic - if mnemonickeys.DELIMITER in name: - name = mnemonickeys.get_base32(name) - - # Now make sure the key is regardless a valid base32 format ed25519 key (readding padding if necessary) - if stringvalidators.validate_pub_key(name): - name = unpaddedbase32.repad(name) - resp = sitefiles.get_file(name, file) - - elif stringvalidators.validate_hash(name): - try: - resp = onionrblockapi.Block(name).bcontent - except onionrexceptions.NoDataAvailable: - abort(404) - except TypeError: - pass - try: - resp = base64.b64decode(resp) - except binascii.Error: - pass - if resp == 'Not Found' or not resp: - abort(404) - return Response(resp, mimetype=mime_type) diff --git a/src/httpapi/onionrsitesapi/findsite.py b/src/httpapi/onionrsitesapi/findsite.py deleted file mode 100644 index 454d0131..00000000 --- a/src/httpapi/onionrsitesapi/findsite.py +++ /dev/null @@ -1,49 +0,0 @@ -""" - Onionr - Private P2P Communication - - view and interact with onionr sites -""" - -from typing import Union - -import onionrexceptions -from onionrutils import mnemonickeys -from onionrutils import stringvalidators -from coredb import blockmetadb -from oldblocks.onionrblockapi import Block -from onionrtypes import BlockHash - -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def find_site(user_id: str) -> Union[BlockHash, None]: - """Returns block hash str for latest block for a site by a given user id""" - # If mnemonic delim in key, convert to base32 version - if mnemonickeys.DELIMITER in user_id: - user_id = mnemonickeys.get_base32(user_id) - - if not stringvalidators.validate_pub_key(user_id): - raise onionrexceptions.InvalidPubkey - - found_site = None - sites = blockmetadb.get_blocks_by_type('osite') - - # Find site by searching all site blocks. eww O(N) ☹️, TODO: event based - for site in sites: - site = Block(site) - if site.isSigner(user_id) and site.verifySig(): - found_site = site.hash - return found_site diff --git a/src/httpapi/onionrsitesapi/sitefiles.py b/src/httpapi/onionrsitesapi/sitefiles.py deleted file mode 100644 index c43fd897..00000000 --- a/src/httpapi/onionrsitesapi/sitefiles.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Onionr - Private P2P Communication. - -Read onionr site files -""" -from typing import Union, Tuple -import tarfile -import io -import os - -import unpaddedbase32 - -from coredb import blockmetadb -from oldblocks import onionrblockapi -from oldblocks import insert - -# Import types. Just for type hiting -from onionrtypes import UserID, DeterministicKeyPassphrase, BlockHash - -from onionrcrypto import generate_deterministic -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def find_site_gzip(user_id: str)->tarfile.TarFile: - """Return verified site tar object""" - sites = blockmetadb.get_blocks_by_type('osite') - user_site = None - unpadded_user = user_id - user_id = unpaddedbase32.repad(user_id) - for site in sites: - block = onionrblockapi.Block(site) - if block.isSigner(user_id) or block.isSigner(unpadded_user): - user_site = block - if not user_site is None: - return tarfile.open(fileobj=io.BytesIO(user_site.bcontent), mode='r') - return None - - -def get_file(user_id, file)->Union[bytes, None]: - """Get a site file content""" - ret_data = "" - site = find_site_gzip(user_id) - - if file.endswith('/'): - file += 'index.html' - if site is None: return None - for t_file in site.getmembers(): - - if t_file.name.replace('./', '') == file: - return site.extractfile(t_file) - return None - - -def create_site(admin_pass: DeterministicKeyPassphrase, directory:str='.')->Tuple[UserID, BlockHash]: - public_key, private_key = generate_deterministic(admin_pass) - - raw_tar = io.BytesIO() - - tar = tarfile.open(mode='x:gz', fileobj=raw_tar) - tar.add(directory) - tar.close() - - raw_tar.seek(0) - - block_hash = insert(raw_tar.read(), header='osite', signing_key=private_key, sign=True) - - return (public_key, block_hash) diff --git a/src/httpapi/security/__init__.py b/src/httpapi/security/__init__.py index 653f6af4..ba0a5076 100644 --- a/src/httpapi/security/__init__.py +++ b/src/httpapi/security/__init__.py @@ -1 +1 @@ -from . import client, public \ No newline at end of file +from . import client \ No newline at end of file diff --git a/src/httpapi/security/public.py b/src/httpapi/security/public.py deleted file mode 100644 index 239160fc..00000000 --- a/src/httpapi/security/public.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Onionr - Private P2P Communication. - -Process incoming requests to the public api server for certain attacks -""" -from flask import Blueprint, request, abort, g -from httpapi import httpheaders -from onionrutils import epoch -from utils import gettransports -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -class PublicAPISecurity: - def __init__(self, public_api): - public_api_security_bp = Blueprint('publicapisecurity', __name__) - self.public_api_security_bp = public_api_security_bp - - @public_api_security_bp.before_app_request - def validate_request(): - """Validate request has the correct hostname""" - # If high security level, deny requests to public - # (HS should be disabled anyway for Tor, but might not be for I2P) - - g.is_onionr_client = False - transports = gettransports.get() - if public_api.config.get('general.security_level', default=1) > 0: - abort(403) - - if request.host not in transports: - # Abort conn if wrong HTTP hostname, to prevent DNS rebinding - if not public_api.config.get( - 'general.allow_public_api_dns_rebinding', False): - abort(403) - public_api.hitCount += 1 # raise hit count for valid requests - try: - if 'onionr' in request.headers['User-Agent'].lower(): - g.is_onionr_client = True - else: - g.is_onionr_client = False - except KeyError: - g.is_onionr_client = False - # Add shared objects - try: - g.too_many = public_api._too_many - except KeyError: - g.too_many = None - - @public_api_security_bp.after_app_request - def send_headers(resp): - """Send api, access control headers""" - resp = httpheaders.set_default_onionr_http_headers(resp) - # Network API version - resp.headers['X-API'] = public_api.API_VERSION - resp.headers['Access-Control-Allow-Origin'] = "*" - resp.headers['Access-Control-Allow-Headers'] = "*" - resp.headers['Access-Control-Allow-Methods'] = "POST, GET, OPTIONS" - # Delete some HTTP headers for Onionr user agents - NON_NETWORK_HEADERS = ( - 'Content-Security-Policy', 'X-Frame-Options', - 'X-Content-Type-Options', 'Feature-Policy', - 'Clear-Site-Data', 'Referrer-Policy', - 'Access-Control-Allow-Origin', 'Access-Control-Allow-Headers', - 'Access-Control-Allow-Methods') - - # For other nodes, we don't need to waste bits on the above headers - try: - if g.is_onionr_client: - for header in NON_NETWORK_HEADERS: - del resp.headers[header] - else: - del resp.headers['X-API'] - except AttributeError: - abort(403) - - public_api.lastRequest = epoch.get_rounded_epoch(roundS=5) - return resp diff --git a/src/httpapi/sse/__init__.py b/src/httpapi/sse/__init__.py index 2e46df3e..c452cfd0 100644 --- a/src/httpapi/sse/__init__.py +++ b/src/httpapi/sse/__init__.py @@ -3,16 +3,16 @@ server sent event modules, incl a wrapper and endpoints for client + public api """ """ - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. - You should have received a copy of the GNU General Public License - along with this program. If not, see . +You should have received a copy of the GNU General Public License +along with this program. If not, see . """ diff --git a/src/httpapi/sse/private/__init__.py b/src/httpapi/sse/private/__init__.py index 625ec89f..6c7c14cb 100644 --- a/src/httpapi/sse/private/__init__.py +++ b/src/httpapi/sse/private/__init__.py @@ -9,11 +9,7 @@ from gevent import sleep import gevent import ujson -from oldblocks.onionrblockapi import Block -from coredb.dbfiles import block_meta_db -from coredb.blockmetadb import get_block_list from onionrutils.epoch import get_epoch -from onionrstatistics.transports.tor import TorStats from .. import wrapper """ This program is free software: you can redistribute it and/or modify @@ -42,37 +38,3 @@ def stream_hello(): yield "hello\n\n" sleep(1) return SSEWrapper.handle_sse_request(print_hello) - - -@private_sse_blueprint.route('/torcircuits') -def stream_tor_circuits(): - tor_stats = g.too_many.get(TorStats) - def circuit_stat_stream(): - while True: - yield "data: " + tor_stats.get_json() + "\n\n" - sleep(10) - return SSEWrapper.handle_sse_request(circuit_stat_stream) - -@private_sse_blueprint.route('/recentblocks') -def stream_recent_blocks(): - def _compile_json(b_list): - js = {} - block_obj = None - for block in b_list: - block_obj = Block(block) - if block_obj.isEncrypted: - js[block] = 'encrypted' - else: - js[block] = Block(block).btype - return ujson.dumps({"blocks": js}, reject_bytes=True) - - def _stream_recent(): - last_time = Path(block_meta_db).stat().st_ctime - while True: - if Path(block_meta_db).stat().st_ctime != last_time: - last_time = Path(block_meta_db).stat().st_ctime - yield "data: " + _compile_json(get_block_list(get_epoch() - 5)) + "\n\n" - else: - yield "data: none" + "\n\n" - sleep(5) - return SSEWrapper.handle_sse_request(_stream_recent) diff --git a/src/keymanager.py b/src/keymanager.py deleted file mode 100755 index b2ecad97..00000000 --- a/src/keymanager.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Onionr - Private P2P Communication. - -Load, save, and delete the user's public key pairs (does not handle peer keys) -""" -from onionrutils import bytesconverter -from onionrcrypto import generate -import filepaths -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -class KeyManager: - def __init__(self): - self.keyFile = filepaths.keys_file - - def addKey(self, pubKey=None, privKey=None): - """Add a new key pair. - - either specified or None to generate a new pair automatically - """ - if type(pubKey) is type(None) and type(privKey) is type(None): - pubKey, privKey = generate.generate_pub_key() - pubKey = bytesconverter.bytes_to_str(pubKey) - privKey = bytesconverter.bytes_to_str(privKey) - try: - if pubKey in self.getPubkeyList(): - raise ValueError('Pubkey already in list: %s' % (pubKey,)) - except FileNotFoundError: - pass - - with open(self.keyFile, "a") as keyFile: - keyFile.write(pubKey + ',' + privKey + '\n') - return (pubKey, privKey) - - def removeKey(self, pubKey): - """Remove a key pair by pubkey""" - keyList = self.getPubkeyList() - keyData = '' - try: - keyList.remove(pubKey) - except ValueError: - return False - else: - keyData = ','.join(keyList) - with open(self.keyFile, "w") as keyFile: - keyFile.write(keyData) - - def getPubkeyList(self): - """Return a list of the user's keys""" - keyList = [] - try: - with open(self.keyFile, "r") as keyFile: - keyData = keyFile.read() - except FileNotFoundError: - keyData = '' - keyData = keyData.split('\n') - for pair in keyData: - if len(pair) > 0: - keyList.append(pair.split(',')[0]) - return keyList - - def getPrivkey(self, pubKey): - privKey = None - with open(self.keyFile, "r") as keyFile: - keyData = keyFile.read() - for pair in keyData.split('\n'): - if pubKey in pair or pubKey.replace('=', '') in pair: - privKey = pair.split(',')[1] - return privKey diff --git a/src/lan/__init__.py b/src/lan/__init__.py deleted file mode 100644 index ea30efd5..00000000 --- a/src/lan/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Onionr - Private P2P Communication. - -LAN manager -""" -from typing import TYPE_CHECKING -from threading import Thread -if TYPE_CHECKING: - from toomanyobjs import TooMany - -from .discover import learn_services, advertise_service -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -class LANManager: - """Initialize and start/top LAN transport threads.""" - - def __init__(self, too_many: "TooMany"): - self.too_many = too_many - self.peers: "exploded IP Address string" = [] - - def start(self): - Thread(target=learn_services, daemon=True).start() - Thread(target=advertise_service, daemon=True).start() - diff --git a/src/lan/client/__init__.py b/src/lan/client/__init__.py deleted file mode 100644 index 48bc8609..00000000 --- a/src/lan/client/__init__.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Onionr - Private P2P Communication. - -LAN transport client thread -""" -import requests - -from typing import Set - -from onionrtypes import LANIP -import logger -from coredb.blockmetadb import get_block_list -from oldblocks.blockimporter import import_block_from_data -import onionrexceptions -from ..server import ports -from onionrproofs import hashMeetsDifficulty - -from threading import Thread -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - -connected_lan_peers: Set[LANIP] = set([]) - - -def _lan_work(peer: LANIP): - def _sync_peer(url): - our_blocks = get_block_list() - blocks = requests.get(url + 'blist/0').text.splitlines() - for block in blocks: - if block not in our_blocks and hashMeetsDifficulty(block): - try: - import_block_from_data( - requests.get( - url + f'get/{block}', stream=True).raw.read(6000000)) - except onionrexceptions.InvalidMetadata: - logger.warn(f"Could not get {block} from lan peer") - except onionrexceptions.InvalidProof: - logger.warn( - f"Invalid proof for {block} from lan peer {peer}", terminal=True) - break - - for port in ports: - try: - root = f'http://{peer}:{port}/' - if requests.get(f'{root}ping').text != 'onionr!': - connected_lan_peers.remove(peer) - else: - logger.info(f'[LAN] Connected to {peer}:{port}', terminal=True) - while True: - try: - _sync_peer(root) - except requests.exceptions.ConnectionError: - break - break - except requests.exceptions.ConnectionError: - pass - else: - connected_lan_peers.remove(peer) - - -def connect_peer(peer: LANIP): - if peer not in connected_lan_peers: - connected_lan_peers.add(peer) - Thread(target=_lan_work, args=[peer], daemon=True).start() - diff --git a/src/lan/discover.py b/src/lan/discover.py deleted file mode 100644 index 42827c4f..00000000 --- a/src/lan/discover.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Onionr - Private P2P Communication. - -Discover and publish private-network -""" -import socket -import struct -from ipaddress import ip_address - -from .getip import lan_ips, best_ip -from utils.bettersleep import better_sleep -from . import client -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" -MCAST_GRP = '224.0.0.112' -MCAST_PORT = 1337 -IS_ALL_GROUPS = True -ANNOUNCE_LOOP_SLEEP = 30 - - -def learn_services(): - """Take a list to infintely add lan service info to.""" - - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if IS_ALL_GROUPS: - # on this port, receives ALL multicast groups - sock.bind(('', MCAST_PORT)) - else: - # on this port, listen ONLY to MCAST_GRP - sock.bind((MCAST_GRP, MCAST_PORT)) - mreq = struct.pack("4sl", socket.inet_aton(MCAST_GRP), socket.INADDR_ANY) - - sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) - - while True: - service_ips = sock.recv(200).decode('utf-8') - if 'onionr' not in service_ips: - continue - service_ips = service_ips.replace('onionr-', '').split('-') - - for service in service_ips: - try: - ip_address(service) - if not ip_address(service).is_private: - raise ValueError - if service in lan_ips: - raise ValueError - except ValueError: - pass - else: - client.connect_peer(service) - - -def advertise_service(specific_ips=None): - # regarding socket.IP_MULTICAST_TTL - # --------------------------------- - # for all packets sent, after three hops on the network the packet will not - # be re-sent/broadcast - # (see https://www.tldp.org/HOWTO/Multicast-HOWTO-6.html) - MULTICAST_TTL = 3 - - ips = best_ip - if not ips: - return - - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) - sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, MULTICAST_TTL) - while True: - sock.sendto(f"onionr-{ips}".encode('utf-8'), (MCAST_GRP, MCAST_PORT)) - better_sleep(ANNOUNCE_LOOP_SLEEP) - diff --git a/src/lan/getip.py b/src/lan/getip.py deleted file mode 100644 index 0031734b..00000000 --- a/src/lan/getip.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Onionr - Private P2P Communication. - -Identify LAN ip addresses and determine the best one -""" -from ipaddress import IPv4Address - -from psutil import net_if_addrs -from socket import AF_INET - -import logger -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - -lan_ips = [] - -# https://psutil.readthedocs.io/en/latest/#psutil.net_if_addrs -def _get_lan_ips(): - for interface in net_if_addrs().keys(): - for address in net_if_addrs()[interface]: - # Don't see benefit in ipv6, so just check for v4 addresses - if address[0] == AF_INET: - # Mark the address for use in LAN if it is a private address - if IPv4Address(address[1]).is_private and not IPv4Address(address[1]).is_loopback: - lan_ips.append(address[1]) -try: - _get_lan_ips() -except OSError: - logger.warn("Could not identify LAN ips due to OSError.") - -# These are more likely to be actual local subnets rather than VPNs -for ip in lan_ips: - if '192.168' in ip: - best_ip = ip - break -else: - try: - best_ip = lan_ips[0] - except IndexError: - best_ip = "" diff --git a/src/lan/server/__init__.py b/src/lan/server/__init__.py deleted file mode 100644 index 0c667c15..00000000 --- a/src/lan/server/__init__.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Onionr - Private P2P Communication. - -LAN transport server thread -""" -import ipaddress -import time -from threading import Thread - -from gevent.pywsgi import WSGIServer -from flask import Flask -from flask import Response -from flask import request -from flask import abort - -from oldblocks.onionrblockapi import Block -from httpapi.fdsafehandler import FDSafeHandler -from netcontroller import get_open_port -import config -from coredb.blockmetadb import get_block_list -from lan.getip import best_ip, lan_ips -from onionrutils import stringvalidators -from httpapi.miscpublicapi.upload import accept_upload -import logger -from utils.bettersleep import better_sleep -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" -ports = range(1337, 1340) -_start_time = time.time() - - -class LANServer: - def __init__(self, shared_state): - app = Flask(__name__) - self.app = app - self.host = config.get('lan.bind_ip', '') - self.server = None - if self.host == '': - self.host = best_ip - self.port = None - - @app.before_request - def dns_rebinding_prevention(): - if not ipaddress.ip_address(request.remote_addr).is_private: - abort(403) - if request.remote_addr in lan_ips or \ - ipaddress.ip_address(request.remote_addr).is_loopback: - if time.time() - _start_time > 600: - abort(403) - if request.host != f'{self.host}:{self.port}': - logger.warn('Potential DNS rebinding attack on LAN server:') - logger.warn( - f'Hostname {request.host} was used instead of {self.host}:{self.port}') # noqa - abort(403) - - @app.route('/blist/