diff --git a/src/apiservers/__init__.py b/src/apiservers/__init__.py index 82599cc7..f9926150 100755 --- a/src/apiservers/__init__.py +++ b/src/apiservers/__init__.py @@ -4,7 +4,6 @@ Public is net-facing server meant for other nodes Private is meant for controlling and accessing this node """ -from . import public, private +from . import private -PublicAPI = public.PublicAPI ClientAPI = private.PrivateAPI diff --git a/src/apiservers/private/__init__.py b/src/apiservers/private/__init__.py index 657c30b7..a480ce96 100644 --- a/src/apiservers/private/__init__.py +++ b/src/apiservers/private/__init__.py @@ -16,20 +16,20 @@ import logger from etc import waitforsetvar from . import register_private_blueprints import config -from .. import public + """ - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. - You should have received a copy of the GNU General Public License - along with this program. If not, see . +You should have received a copy of the GNU General Public License +along with this program. If not, see . """ @@ -50,7 +50,7 @@ class PrivateAPI: self.startTime = epoch.get_epoch() app = flask.Flask(__name__) - + bind_port = int(config.get('client.client.port', 59496)) self.bindPort = bind_port @@ -77,8 +77,7 @@ class PrivateAPI: """Start client gevent API web server with flask client app.""" waitforsetvar.wait_for_set_var(self, "_too_many") fd_handler = httpapi.fdsafehandler.FDSafeHandler - self.publicAPI = self._too_many.get( # pylint: disable=E1101 - public.PublicAPI) + self.httpServer = WSGIServer((self.host, self.bindPort), self.app, log=None, handler_class=fd_handler) diff --git a/src/apiservers/public/__init__.py b/src/apiservers/public/__init__.py deleted file mode 100644 index 2a2c610a..00000000 --- a/src/apiservers/public/__init__.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Onionr - Private P2P Communication. - -This file handles all incoming http requests -to the public api server, using Flask -""" -import time -import threading -import flask -from gevent.pywsgi import WSGIServer -from httpapi import apiutils, security, fdsafehandler, miscpublicapi -import logger -import config -import filepaths -from utils import gettransports -from etc import onionrvalues, waitforsetvar -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def _get_tor_adder(pub_api): - transports = [] - while len(transports) == 0: - transports = gettransports.get() - time.sleep(0.3) - pub_api.torAdder = transports[0] - - -class PublicAPI: - """The new client api server, isolated from the public api.""" - - def __init__(self): - """Setup the public api app.""" - app = flask.Flask('PublicAPI') - app.config['MAX_CONTENT_LENGTH'] = 5 * 1024 * 1024 - self.i2pEnabled = config.get('i2p.host', False) - self.hideBlocks = [] # Blocks to be denied sharing - self.host = apiutils.setbindip.set_bind_IP( - filepaths.public_API_host_file) - - threading.Thread(target=_get_tor_adder, - args=[self], daemon=True).start() - - self.torAdder = "" - self.bindPort = config.get('client.public.port') - self.lastRequest = 0 - # total rec requests to public api since server started - self.hitCount = 0 - self.config = config - self.API_VERSION = onionrvalues.API_VERSION - logger.info('Running public api on %s:%s' % (self.host, self.bindPort)) - - app.register_blueprint( - security.public.PublicAPISecurity(self).public_api_security_bp) - app.register_blueprint( - miscpublicapi.endpoints.PublicEndpoints(self).public_endpoints_bp) - self.app = app - - def start(self): - """Start the Public API server.""" - waitforsetvar.wait_for_set_var(self, "_too_many") - self.httpServer = WSGIServer((self.host, self.bindPort), - self.app, log=None, - handler_class=fdsafehandler.FDSafeHandler) - self.httpServer.serve_forever() diff --git a/src/communicator/README.md b/src/communicator/README.md deleted file mode 100644 index 9925f247..00000000 --- a/src/communicator/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Onionr Communicator - -Onionr communicator is the Onionr client. It "connects" to remote Onionr peers and does things such as: - -* Finding new peers -* Uploading blocks -* Downloading blocks -* Daemon maintenance/housekeeping - -## Files - -* \_\_init\_\_.py: Contains the main communicator code. Inits and launches the communicator and sets up the timers -* peeraction.py: contains a function to send commands to remote peers -* bootstrappers.py: adds peers from the bootstrap list to the communicator to try to connect to them -* onlinepeers: management of the online peer pool for the communicator \ No newline at end of file diff --git a/src/communicator/__init__.py b/src/communicator/__init__.py deleted file mode 100755 index 9a84cc8b..00000000 --- a/src/communicator/__init__.py +++ /dev/null @@ -1,206 +0,0 @@ -"""Onionr - Private P2P Communication. - -This file contains both the OnionrCommunicate class for -communcating with peers and code to operate as a daemon, -getting commands from the command queue database -""" -import time - -import config -import logger -import onionrpeers -import onionrplugins as plugins -from . import onlinepeers -from . import uploadqueue -from communicatorutils import downloadblocks -from communicatorutils import lookupblocks -from communicatorutils import lookupadders -from communicatorutils import connectnewpeers -from communicatorutils import uploadblocks -from communicatorutils import announcenode, deniableinserts -from communicatorutils import cooldownpeer -from communicatorutils import housekeeping -from communicatorutils import netcheck -from onionrthreads import add_onionr_thread -from onionrcommands.openwebinterface import get_url -from netcontroller import NetController -from . import bootstrappeers -from . import daemoneventhooks -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - -config.reload() - - -class OnionrCommunicatorDaemon: - def __init__(self, shared_state, developmentMode=None): - if developmentMode is None: - developmentMode = config.get( - 'general.dev_mode', False) - - # configure logger and stuff - self.config = config - self.shared_state = shared_state # TooManyObjects module - shared_state.add(self) - - # populate kv values - self.kv = self.shared_state.get_by_string('DeadSimpleKV') - - if config.get('general.offline_mode', False): - self.kv.put('isOnline', False) - - # initialize core with Tor socks port being 3rd argument - self.proxyPort = shared_state.get(NetController).socksPort - - self.upload_session_manager = self.shared_state.get( - uploadblocks.sessionmanager.BlockUploadSessionManager) - self.shared_state.share_object() - - # loop time.sleep delay in seconds - self.delay = 5 - - # amount of threads running by name, used to prevent too many - self.threadCounts = {} - - # Loads in and starts the enabled plugins - plugins.reload() - - # extends our upload list and saves our list when Onionr exits - uploadqueue.UploadQueue(self) - - add_onionr_thread( - lookupblocks.lookup_blocks_from_communicator, - [self.shared_state], 25, 3) - - add_onionr_thread( - downloadblocks.download_blocks_from_communicator, - [self.shared_state], - config.get('timers.getBlocks', 10), 1) - - add_onionr_thread(onlinepeers.clear_offline_peer, [self.kv], 58) - - add_onionr_thread( - housekeeping.clean_old_blocks, [self.shared_state], 10, 1) - - # Discover new peers - add_onionr_thread( - lookupadders.lookup_new_peer_transports_with_communicator, - [shared_state], 60, 3) - - # Timer for adjusting which peers - # we actively communicate to at any given time, - # to avoid over-using peers - add_onionr_thread( - cooldownpeer.cooldown_peer, [self.shared_state], 30, 60) - - # Timer to read the upload queue and upload the entries to peers - add_onionr_thread( - uploadblocks.upload_blocks_from_communicator, - [self.shared_state], 5, 1) - - # This timer creates deniable blocks, - # in an attempt to further obfuscate block insertion metadata - if config.get('general.insert_deniable_blocks', True): - add_onionr_thread( - deniableinserts.insert_deniable_block, [], 180, 10) - - if config.get('transports.tor', True): - # Timer to check for connectivity, - # through Tor to various high-profile onion services - add_onionr_thread(netcheck.net_check, [shared_state], 500, 60) - - # Announce the public API server transport address - # to other nodes if security level allows - if config.get('general.security_level', 1) == 0 \ - and config.get('general.announce_node', True): - # Default to high security level incase config breaks - add_onionr_thread( - announcenode.announce_node, [self.shared_state], 600, 60) - else: - logger.debug('Will not announce node.') - - add_onionr_thread(onionrpeers.peer_cleanup, [], 300, 300) - - add_onionr_thread(housekeeping.clean_keys, [], 15, 1) - - if config.get('general.use_bootstrap_list', True): - bootstrappeers.add_bootstrap_list_to_peer_list( - self.kv, [], db_only=True) - - daemoneventhooks.daemon_event_handlers(shared_state) - - get_url() - if not config.get('onboarding.done', True): - logger.info( - 'First run detected. Run openhome to get setup.', - terminal=True) - - while not config.get('onboarding.done', True) and \ - not self.shared_state.get_by_string( - 'DeadSimpleKV').get('shutdown'): - try: - time.sleep(2) - except KeyboardInterrupt: - self.shared_state.get_by_string( - 'DeadSimpleKV').put('shutdown', True) - - # Main daemon loop, mainly for calling timers, - # don't do any complex operations here to avoid locking - try: - while not self.shared_state.get_by_string( - 'DeadSimpleKV').get('shutdown'): - time.sleep(self.delay) - except KeyboardInterrupt: - self.shared_state.get_by_string( - 'DeadSimpleKV').put('shutdown', True) - - logger.info( - 'Goodbye. (Onionr is cleaning up, and will exit)', terminal=True) - - def decrementThreadCount(self, threadName): - """Decrement amount of a thread name if more than zero. - - called when a function meant to be run in a thread ends - """ - try: - if self.threadCounts[threadName] > 0: - self.threadCounts[threadName] -= 1 - except KeyError: - pass - - def peerCleanup(self): - """This just calls onionrpeers.cleanupPeers. - - Remove dead or bad peers (offline too long, too slow)""" - onionrpeers.peer_cleanup() - self.decrementThreadCount('peerCleanup') - - def getPeerProfileInstance(self, peer): - """Gets a peer profile instance from the list of profiles""" - for i in self.kv.get('peerProfiles'): - # if the peer's profile is already loaded, return that - if i.address == peer: - retData = i - break - else: - # if the peer's profile is not loaded, return a new one. - # connectNewPeer also adds it to the list on connect - retData = onionrpeers.PeerProfiles(peer) - self.kv.get('peerProfiles').append(retData) - return retData - - -def startCommunicator(shared_state): - OnionrCommunicatorDaemon(shared_state) diff --git a/src/communicator/bootstrappeers.py b/src/communicator/bootstrappeers.py deleted file mode 100644 index 191c3165..00000000 --- a/src/communicator/bootstrappeers.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Onionr - Private P2P Communication. - -add bootstrap peers to the communicator peer list -""" -from typing import TYPE_CHECKING - -from utils import readstatic, gettransports -from coredb import keydb -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - -bootstrap_peers = readstatic.read_static('bootstrap-nodes.txt').split(',') - - -def add_bootstrap_list_to_peer_list(kv, peerList, db_only=False): - """Add the bootstrap list to the peer list (no duplicates).""" - for i in bootstrap_peers: - # Add bootstrap peers to peerList (does not save them) - # Don't add them if they're already added or in the offline list - if i not in peerList and i not in kv.get('offlinePeers') \ - and i not in gettransports.get() and len(str(i).strip()) > 0: - if not db_only: - peerList.append(i) - keydb.addkeys.add_address(i) diff --git a/src/communicator/daemoneventhooks/__init__.py b/src/communicator/daemoneventhooks/__init__.py deleted file mode 100644 index f43ac54f..00000000 --- a/src/communicator/daemoneventhooks/__init__.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Onionr - Private P2P Communication. - -Hooks to handle daemon events -""" -from threading import Thread - -from .removefrominsertqueue import remove_from_insert_queue - -from typing import TYPE_CHECKING - -from gevent import sleep - -from communicatorutils.uploadblocks import mixmate -from communicatorutils import restarttor - -if TYPE_CHECKING: - from toomanyobjs import TooMany - from deadsimplekv import DeadSimpleKV - from communicator import OnionrCommunicatorDaemon - from httpapi.daemoneventsapi import DaemonEventsBP - from onionrtypes import BlockHash - from apiservers import PublicAPI -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def daemon_event_handlers(shared_state: 'TooMany'): - def _get_inst(class_name: str): - while True: - try: - return shared_state.get_by_string(class_name) - except KeyError: - sleep(0.2) - comm_inst = _get_inst('OnionrCommunicatorDaemon') - public_api: 'PublicAPI' = _get_inst('PublicAPI') - events_api: 'DaemonEventsBP' = _get_inst('DaemonEventsBP') - kv: 'DeadSimpleKV' = _get_inst('DeadSimpleKV') - - def remove_from_insert_queue_wrapper(block_hash: 'BlockHash'): - remove_from_insert_queue(comm_inst, block_hash) - return "removed" - - def print_test(text=''): - print("It works!", text) - return f"It works! {text}" - - def upload_event(block: 'BlockHash' = ''): - if not block: - raise ValueError - public_api.hideBlocks.append(block) - try: - mixmate.block_mixer(kv.get('blocksToUpload'), block) - except ValueError: - pass - return "removed" - - def restart_tor(): - restarttor.restart(shared_state) - kv.put('offlinePeers', []) - kv.put('onlinePeers', []) - - def test_runtime(): - Thread(target=comm_inst.shared_state.get_by_string( - "OnionrRunTestManager").run_tests).start() - - events_api.register_listener(remove_from_insert_queue_wrapper) - events_api.register_listener(restart_tor) - events_api.register_listener(print_test) - events_api.register_listener(upload_event) - events_api.register_listener(test_runtime) diff --git a/src/communicator/daemoneventhooks/removefrominsertqueue.py b/src/communicator/daemoneventhooks/removefrominsertqueue.py deleted file mode 100644 index d135ccd9..00000000 --- a/src/communicator/daemoneventhooks/removefrominsertqueue.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Onionr - P2P Anonymous Storage Network. - -Remove block hash from daemon's upload list. -""" -from typing import TYPE_CHECKING -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV - from communicator import OnionrCommunicatorDaemon - from onionrtypes import BlockHash -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def remove_from_insert_queue(comm_inst: "OnionrCommunicatorDaemon", - b_hash: "BlockHash"): - """Remove block hash from daemon's upload list.""" - kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV") - try: - kv.get('generating_blocks').remove(b_hash) - except ValueError: - pass diff --git a/src/communicator/onlinepeers/README.md b/src/communicator/onlinepeers/README.md deleted file mode 100644 index fe56995f..00000000 --- a/src/communicator/onlinepeers/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Online Peers - -Manages a pool of peers to perform actions with. Since Onionr does not maintain socket connections, it holds a list of peers. - - -## Files - -* \_\_init\_\_.py: exposes some functions to interact with the pool -* clearofflinepeer.py: Pop the oldest peer in the offline list -* onlinepeers.py: communicator timer to add new peers to the pool randomly -* pickonlinepeers.py: returns a random peer from the online pool -* removeonlinepeer.py: removes a specified peer from the online pool \ No newline at end of file diff --git a/src/communicator/onlinepeers/__init__.py b/src/communicator/onlinepeers/__init__.py deleted file mode 100644 index a3b107ee..00000000 --- a/src/communicator/onlinepeers/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from . import clearofflinepeer, onlinepeers, pickonlinepeers, removeonlinepeer - -clear_offline_peer = clearofflinepeer.clear_offline_peer -get_online_peers = onlinepeers.get_online_peers -pick_online_peer = pickonlinepeers.pick_online_peer -remove_online_peer = removeonlinepeer.remove_online_peer \ No newline at end of file diff --git a/src/communicator/onlinepeers/clearofflinepeer.py b/src/communicator/onlinepeers/clearofflinepeer.py deleted file mode 100644 index 47128a73..00000000 --- a/src/communicator/onlinepeers/clearofflinepeer.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Onionr - Private P2P Communication. - -clear offline peer in a communicator instance -""" -from typing import TYPE_CHECKING - -import logger -if TYPE_CHECKING: - from communicator import OnionrCommunicatorDaemon -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def clear_offline_peer(kv: 'DeadSimpleKV'): - """Remove the longest offline peer to retry later.""" - try: - removed = kv.get('offlinePeers').pop(0) - except IndexError: - pass - else: - logger.debug('Removed ' + removed + - ' from offline list, will try them again.') - diff --git a/src/communicator/onlinepeers/onlinepeers.py b/src/communicator/onlinepeers/onlinepeers.py deleted file mode 100644 index e9966308..00000000 --- a/src/communicator/onlinepeers/onlinepeers.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Onionr - Private P2P Communication. - -get online peers in a communicator instance -""" -import time -from typing import TYPE_CHECKING - -import config -from etc.humanreadabletime import human_readable_time -from communicatorutils.connectnewpeers import connect_new_peer_to_communicator -import logger -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def get_online_peers(shared_state): - """Manage the kv.get('onlinePeers') attribute list. - - Connect to more peers if we have none connected - """ - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - if config.get('general.offline_mode', False): - return - logger.info('Refreshing peer pool...') - max_peers = int(config.get('peers.max_connect', 10)) - needed = max_peers - len(kv.get('onlinePeers')) - - last_seen = 'never' - if not isinstance(kv.get('lastNodeSeen'), type(None)): - last_seen = human_readable_time(kv.get('lastNodeSeen')) - - for _ in range(needed): - if len(kv.get('onlinePeers')) == 0: - connect_new_peer_to_communicator(shared_state, useBootstrap=True) - else: - connect_new_peer_to_communicator(shared_state) - - if kv.get('shutdown'): - break - else: - if len(kv.get('onlinePeers')) == 0: - logger.debug('Couldn\'t connect to any peers.' + - f' Last node seen {last_seen} ago.') - try: - get_online_peers(shared_state) - except RecursionError: - pass - else: - kv.put('lastNodeSeen', time.time()) diff --git a/src/communicator/onlinepeers/pickonlinepeers.py b/src/communicator/onlinepeers/pickonlinepeers.py deleted file mode 100644 index dc11bceb..00000000 --- a/src/communicator/onlinepeers/pickonlinepeers.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Onionr - Private P2P Communication. - -pick online peers in a communicator instance -""" -import secrets -from typing import TYPE_CHECKING - -import onionrexceptions - -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def pick_online_peer(kv: 'DeadSimpleKV'): - """Randomly picks peer from pool without bias (using secrets module).""" - ret_data = '' - peer_length = len(kv.get('onlinePeers')) - if peer_length <= 0: - raise onionrexceptions.OnlinePeerNeeded - - while True: - peer_length = len(kv.get('onlinePeers')) - - try: - # Get a random online peer, securely. - # May get stuck in loop if network is lost - ret_data = kv.get('onlinePeers')[secrets.randbelow(peer_length)] - except IndexError: - pass - else: - break - return ret_data diff --git a/src/communicator/onlinepeers/removeonlinepeer.py b/src/communicator/onlinepeers/removeonlinepeer.py deleted file mode 100644 index 0eb0b338..00000000 --- a/src/communicator/onlinepeers/removeonlinepeer.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Onionr - Private P2P Communication. - -remove an online peer from the pool in a communicator instance -""" -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def remove_online_peer(kv, peer): - """Remove an online peer.""" - try: - del kv.get('connectTimes')[peer] - except KeyError: - pass - try: - del kv.get('dbTimestamps')[peer] - except KeyError: - pass - try: - kv.get('onlinePeers').remove(peer) - except ValueError: - pass diff --git a/src/communicator/peeraction.py b/src/communicator/peeraction.py deleted file mode 100644 index f09814f6..00000000 --- a/src/communicator/peeraction.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Onionr - Private P2P Communication. - -This file implements logic for performing requests to Onionr peers -""" -from typing import TYPE_CHECKING - -import streamedrequests -import logger -from onionrutils import epoch, basicrequests -from coredb import keydb -from . import onlinepeers -from onionrtypes import OnionAddressString -from onionrpeers.peerprofiles import PeerProfiles -from etc.waitforsetvar import wait_for_set_var -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def get_peer_profile(kv, address: OnionAddressString) -> 'PeerProfiles': - profile_inst_list = kv.get('peerProfiles') - for profile in profile_inst_list: - if profile.address == address: - return profile - p = PeerProfiles(address) - profile_inst_list.append(p) - return p - - -def peer_action(shared_state, peer, action, - returnHeaders=False, max_resp_size=5242880): - """Perform a get request to a peer.""" - penalty_score = -10 - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - if len(peer) == 0: - return False - url = 'http://%s/%s' % (peer, action) - - try: - ret_data = basicrequests.do_get_request(url, port=kv.get('proxyPort'), - max_size=max_resp_size) - except streamedrequests.exceptions.ResponseLimitReached: - logger.warn( - 'Request failed due to max response size being overflowed', - terminal=True) - ret_data = False - penalty_score = -100 - # if request failed, (error), mark peer offline - if ret_data is False: - try: - get_peer_profile(kv, peer).addScore(penalty_score) - onlinepeers.remove_online_peer(kv, peer) - keydb.transportinfo.set_address_info( - peer, 'lastConnectAttempt', epoch.get_epoch()) - if action != 'ping' and not kv.get('shutdown'): - logger.warn(f'Lost connection to {peer}', terminal=True) - # Will only add a new peer to pool if needed - onlinepeers.get_online_peers(kv) - except ValueError: - pass - else: - peer_profile = get_peer_profile(kv, peer) - peer_profile.update_connect_time() - peer_profile.addScore(1) - # If returnHeaders, returns tuple of data, headers. - # If not, just data string - return ret_data diff --git a/src/communicator/uploadqueue/__init__.py b/src/communicator/uploadqueue/__init__.py deleted file mode 100644 index 6b1331cc..00000000 --- a/src/communicator/uploadqueue/__init__.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Onionr - Private P2P Communication. - -Class to remember blocks that need to be uploaded -and not shared on startup/shutdown -""" -import atexit -import os -from typing import TYPE_CHECKING - -import deadsimplekv - -import filepaths -from onionrutils import localcommand -if TYPE_CHECKING: - from communicator import OnionrCommunicatorDaemon - from deadsimplekv import DeadSimpleKV -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - -UPLOAD_MEMORY_FILE = filepaths.upload_list - - -def _add_to_hidden_blocks(cache): - for bl in cache: - localcommand.local_command('waitforshare/' + bl, post=True) - - -class UploadQueue: - """Saves and loads block upload info from json file.""" - - def __init__(self, communicator: 'OnionrCommunicatorDaemon'): - """Start the UploadQueue object, loading left over uploads into queue. - - register save shutdown function - """ - self.communicator = communicator - cache: deadsimplekv.DeadSimpleKV = deadsimplekv.DeadSimpleKV( - UPLOAD_MEMORY_FILE) - self.kv: "DeadSimpleKV" = \ - communicator.shared_state.get_by_string("DeadSimpleKV") - self.store_obj = cache - cache = cache.get('uploads') - if cache is None: - cache = [] - - _add_to_hidden_blocks(cache) - self.kv.get('blocksToUpload').extend(cache) - - atexit.register(self.save) - - def save(self): - """Save to disk on shutdown or if called manually.""" - bl: deadsimplekv.DeadSimpleKV = self.kv.get('blocksToUpload') - if len(bl) == 0: - try: - os.remove(UPLOAD_MEMORY_FILE) - except FileNotFoundError: - pass - else: - self.store_obj.put('uploads', bl) - self.store_obj.flush() diff --git a/src/communicatorutils/README.md b/src/communicatorutils/README.md deleted file mode 100755 index 393328c3..00000000 --- a/src/communicatorutils/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# communicatorutils - -The files in this submodule handle various subtasks and utilities for the onionr communicator. - -## Files: - -announcenode.py: Uses a communicator instance to announce our transport address to connected nodes - -connectnewpeers.py: takes a communicator instance and has it connect to as many peers as needed, and/or to a new specified peer. - -cooldownpeer.py: randomly selects a connected peer in a communicator and disconnects them for the purpose of security and network balancing. - -daemonqueuehandler.py: checks for new commands in the daemon queue and processes them accordingly. - -deniableinserts.py: insert fake blocks with the communicator for plausible deniability - -downloadblocks.py: iterates a communicator instance's block download queue and attempts to download the blocks from online peers - -housekeeping.py: cleans old blocks and forward secrecy keys - -lookupadders.py: ask connected peers to share their list of peer transport addresses - -lookupblocks.py: lookup new blocks from connected peers from the communicator - -netcheck.py: check if the node is online based on communicator status and onion server ping results - -onionrcommunicataortimers.py: create a timer for a function to be launched on an interval. Control how many possible instances of a timer may be running a function at once and control if the timer should be ran in a thread or not. - -proxypicker.py: returns a string name for the appropriate proxy to be used with a particular peer transport address. - -servicecreator.py: iterate connection blocks and create new direct connection servers for them. - -uploadblocks.py: iterate a communicator's upload queue and upload the blocks to connected peers \ No newline at end of file diff --git a/src/communicatorutils/__init__.py b/src/communicatorutils/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/src/communicatorutils/announcenode.py b/src/communicatorutils/announcenode.py deleted file mode 100755 index 91caa987..00000000 --- a/src/communicatorutils/announcenode.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Onionr - Private P2P Communication. - -Use a communicator instance to announce -our transport address to connected nodes -""" -from typing import TYPE_CHECKING - -import logger -from onionrutils import basicrequests -from utils import gettransports -from netcontroller import NetController -from communicator import onlinepeers -from coredb import keydb -import onionrexceptions - -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def announce_node(shared_state): - """Announce our node to our peers.""" - ret_data = False - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - config = shared_state.get_by_string("OnionrCommunicatorDaemon").config - # Do not let announceCache get too large - if len(kv.get('announceCache')) >= 10000: - kv.get('announceCache').popitem() - - if config.get('general.security_level', 0) == 0: - # Announce to random online peers - for i in kv.get('onlinePeers'): - if i not in kv.get('announceCache'): - peer = i - break - else: - try: - peer = onlinepeers.pick_online_peer(kv) - except onionrexceptions.OnlinePeerNeeded: - peer = "" - - try: - ourID = gettransports.get()[0] - if not peer: - raise onionrexceptions.OnlinePeerNeeded - except (IndexError, onionrexceptions.OnlinePeerNeeded): - pass - else: - url = 'http://' + peer + '/announce' - data = {'node': ourID} - - logger.info('Announcing node to ' + url) - if basicrequests.do_post_request( - url, - data, - port=shared_state.get(NetController).socksPort)\ - == 'Success': - logger.info('Successfully introduced node to ' + peer, - terminal=True) - ret_data = True - keydb.transportinfo.set_address_info(peer, 'introduced', 1) - - return ret_data diff --git a/src/communicatorutils/connectnewpeers.py b/src/communicatorutils/connectnewpeers.py deleted file mode 100755 index 3278c569..00000000 --- a/src/communicatorutils/connectnewpeers.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Onionr - Private P2P Communication. - -Connect a new peer to our communicator instance. -Does so randomly if no peer is specified -""" -import time -import secrets - -import onionrexceptions -import logger -import onionrpeers -from utils import networkmerger, gettransports -from onionrutils import stringvalidators, epoch -from communicator import peeraction, bootstrappeers -from coredb import keydb -import config -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def connect_new_peer_to_communicator(shared_state, peer='', useBootstrap=False): - retData = False - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - tried = kv.get('offlinePeers') - transports = gettransports.get() - if peer != '': - if stringvalidators.validate_transport(peer): - peerList = [peer] - else: - raise onionrexceptions.InvalidAddress( - 'Will not attempt connection test to invalid address') - else: - peerList = keydb.listkeys.list_adders() - - mainPeerList = keydb.listkeys.list_adders() - if not peerList: - peerList = onionrpeers.get_score_sorted_peer_list() - - """ - If we don't have enough peers connected or random chance, - select new peers to try - """ - if len(peerList) < 8 or secrets.randbelow(4) == 3: - tryingNew = [] - for x in kv.get('newPeers'): - if x not in peerList: - peerList.append(x) - tryingNew.append(x) - for i in tryingNew: - kv.get('newPeers').remove(i) - - if len(peerList) == 0 or useBootstrap: - # Avoid duplicating bootstrap addresses in peerList - if config.get('general.use_bootstrap_list', True): - bootstrappeers.add_bootstrap_list_to_peer_list(kv, peerList) - - for address in peerList: - address = address.strip() - - # Don't connect to our own address - if address in transports: - continue - """Don't connect to invalid address or - if its already been tried/connected, or if its cooled down - """ - if len(address) == 0 or address in tried \ - or address in kv.get('onlinePeers') \ - or address in kv.get('cooldownPeer'): - continue - if kv.get('shutdown'): - return - # Ping a peer, - ret = peeraction.peer_action(shared_state, address, 'ping') - if ret == 'pong!': - time.sleep(0.1) - if address not in mainPeerList: - # Add a peer to our list if it isn't already since it connected - networkmerger.mergeAdders(address) - if address not in kv.get('onlinePeers'): - logger.info('Connected to ' + address, terminal=True) - kv.get('onlinePeers').append(address) - kv.get('connectTimes')[address] = epoch.get_epoch() - retData = address - - # add peer to profile list if they're not in it - for profile in kv.get('peerProfiles'): - if profile.address == address: - break - else: - kv.get('peerProfiles').append( - onionrpeers.PeerProfiles(address)) - try: - del kv.get('plaintextDisabledPeers')[address] - except KeyError: - pass - if peeraction.peer_action( - shared_state, address, 'plaintext') == 'false': - kv.get('plaintextDisabledPeers')[address] = True - break - - else: - # Mark a peer as tried if they failed to respond to ping - tried.append(address) - logger.debug('Failed to connect to %s: %s ' % (address, ret)) - return retData diff --git a/src/communicatorutils/cooldownpeer.py b/src/communicatorutils/cooldownpeer.py deleted file mode 100755 index 0e57db8b..00000000 --- a/src/communicatorutils/cooldownpeer.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Onionr - Private P2P Communication. - -Select random online peer in a communicator instance and have them "cool down" -""" -from typing import TYPE_CHECKING - -from onionrutils import epoch -from communicator import onlinepeers - -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def cooldown_peer(shared_state): - """Randomly add an online peer to cooldown, so we can connect a new one.""" - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - config = shared_state.get_by_string("OnionrCommunicatorDaemon").config - online_peer_amount = len(kv.get('onlinePeers')) - minTime = 300 - cooldown_time = 600 - to_cool = '' - tempConnectTimes = dict(kv.get('connectTimes')) - - # Remove peers from cooldown that have been there long enough - tempCooldown = dict(kv.get('cooldownPeer')) - for peer in tempCooldown: - if (epoch.get_epoch() - tempCooldown[peer]) >= cooldown_time: - del kv.get('cooldownPeer')[peer] - - # Cool down a peer, if we have max connections alive for long enough - if online_peer_amount >= config.get('peers.max_connect', 10, save=True): - finding = True - - while finding: - try: - to_cool = min(tempConnectTimes, key=tempConnectTimes.get) - if (epoch.get_epoch() - tempConnectTimes[to_cool]) < minTime: - del tempConnectTimes[to_cool] - else: - finding = False - except ValueError: - break - else: - onlinepeers.remove_online_peer(kv, to_cool) - kv.get('cooldownPeer')[to_cool] = epoch.get_epoch() - diff --git a/src/communicatorutils/deniableinserts.py b/src/communicatorutils/deniableinserts.py deleted file mode 100755 index 26c34163..00000000 --- a/src/communicatorutils/deniableinserts.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Onionr - Private P2P Communication. - -Use the communicator to insert fake mail messages -""" -import secrets - -from etc import onionrvalues -import onionrblocks -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def insert_deniable_block(): - """Insert a fake block to make it more difficult to track real blocks.""" - fakePeer = '' - chance = 10 - if secrets.randbelow(chance) == (chance - 1): - # This assumes on the libsodium primitives to have key-privacy - fakePeer = onionrvalues.DENIABLE_PEER_ADDRESS - data = secrets.token_hex(secrets.randbelow(5120) + 1) - onionrblocks.insert(data, header='pm', encryptType='asym', - asymPeer=fakePeer, disableForward=True, - meta={'subject': 'foo'}) diff --git a/src/communicatorutils/downloadblocks/__init__.py b/src/communicatorutils/downloadblocks/__init__.py deleted file mode 100755 index 26b38e52..00000000 --- a/src/communicatorutils/downloadblocks/__init__.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Onionr - Private P2P Communication. - -Download blocks using the communicator instance. -""" -from typing import TYPE_CHECKING -from secrets import SystemRandom - -if TYPE_CHECKING: - from communicator import OnionrCommunicatorDaemon - from deadsimplekv import DeadSimpleKV - -from gevent import spawn - -import onionrexceptions -import logger -import onionrpeers - -from communicator import peeraction -from communicator import onlinepeers -from onionrblocks import blockmetadata -from onionrutils import validatemetadata -from coredb import blockmetadb -from onionrutils.localcommand import local_command -import onionrcrypto -import onionrstorage -from onionrblocks import onionrblacklist -from onionrblocks import storagecounter -from onionrproofs import vdf -from . import shoulddownload -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - -storage_counter = storagecounter.StorageCounter() - - -def download_blocks_from_communicator(shared_state: "TooMany"): - """Use communicator instance to download blocks in the comms's queue""" - blacklist = onionrblacklist.OnionrBlackList() - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter - count: int = 0 - metadata_validation_result: bool = False - # Iterate the block queue in the communicator - for blockHash in list(kv.get('blockQueue')): - count += 1 - - try: - blockPeers = list(kv.get('blockQueue')[blockHash]) - except KeyError: - blockPeers = [] - removeFromQueue = True - - if not shoulddownload.should_download(shared_state, blockHash): - continue - - if kv.get('shutdown') or not kv.get('isOnline') or \ - storage_counter.is_full(): - # Exit loop if shutting down or offline, or disk allocation reached - break - # Do not download blocks being downloaded - if blockHash in kv.get('currentDownloading'): - continue - - if len(kv.get('onlinePeers')) == 0: - break - - # So we can avoid concurrent downloading in other threads of same block - kv.get('currentDownloading').append(blockHash) - if len(blockPeers) == 0: - try: - peerUsed = onlinepeers.pick_online_peer(kv) - except onionrexceptions.OnlinePeerNeeded: - continue - else: - SystemRandom().shuffle(blockPeers) - peerUsed = blockPeers.pop(0) - - if not kv.get('shutdown') and peerUsed.strip() != '': - logger.info( - f"Attempting to download %s from {peerUsed}..." % (blockHash[:12],)) - content = peeraction.peer_action( - shared_state, peerUsed, - 'getdata/' + blockHash, - max_resp_size=3000000) # block content from random peer - - if content is not False and len(content) > 0: - try: - content = content.encode() - except AttributeError: - pass - - if vdf.verify_vdf(blockHash, content): - #content = content.decode() # decode here because sha3Hash needs bytes above - metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata - metadata = metas[0] - try: - metadata_validation_result = \ - validatemetadata.validate_metadata(metadata, metas[2]) - except onionrexceptions.PlaintextNotSupported: - logger.debug(f"Not saving {blockHash} due to plaintext not enabled") - removeFromQueue = True - except onionrexceptions.DataExists: - metadata_validation_result = False - if metadata_validation_result: # check if metadata is valid, and verify nonce - save_block(blockHash, data) - else: - if blacklist.inBlacklist(blockHash): - logger.warn(f'Block {blockHash} is blacklisted.') - else: - logger.warn('Metadata for block %s is invalid.' % (blockHash,)) - blacklist.addToDB(blockHash) - else: - # if block didn't meet expected hash - tempHash = onionrcrypto.hashers.sha3_hash(content) # lazy hack, TODO use var - try: - tempHash = tempHash.decode() - except AttributeError: - pass - # Punish peer for sharing invalid block (not always malicious, but is bad regardless) - onionrpeers.PeerProfiles(peerUsed).addScore(-50) - if removeFromQueue: - try: - del kv.get('blockQueue')[blockHash] # remove from block queue both if success or false - if count == LOG_SKIP_COUNT: - logger.info('%s blocks remaining in queue' % - [len(kv.get('blockQueue'))], terminal=True) - count = 0 - except KeyError: - pass - kv.get('currentDownloading').remove(blockHash) diff --git a/src/communicatorutils/downloadblocks/shoulddownload.py b/src/communicatorutils/downloadblocks/shoulddownload.py deleted file mode 100644 index ddf11624..00000000 --- a/src/communicatorutils/downloadblocks/shoulddownload.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Onionr - Private P2P Communication. - -Check if a block should be downloaded -(if we already have it or its blacklisted or not) -""" -from coredb import blockmetadb -from onionrblocks import onionrblacklist -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def should_download(shared_state, block_hash) -> bool: - """Return bool for if a (assumed to exist) block should be downloaded.""" - blacklist = onionrblacklist.OnionrBlackList() - should = True - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - if block_hash in blockmetadb.get_block_list(): - # Don't download block we have - should = False - else: - if blacklist.inBlacklist(block_hash): - # Don't download blacklisted block - should = False - if should is False: - # Remove block from communicator queue if it shouldn't be downloaded - try: - del kv.get('blockQueue')[block_hash] - except KeyError: - pass - return should diff --git a/src/communicatorutils/housekeeping.py b/src/communicatorutils/housekeeping.py deleted file mode 100755 index a36f8556..00000000 --- a/src/communicatorutils/housekeeping.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Onionr - Private P2P Communication. - -Cleanup old Onionr blocks and forward secrecy keys using the communicator. -Ran from a communicator timer usually -""" -import sqlite3 -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV - -import logger -from onionrusers import onionrusers -from onionrutils import epoch -from coredb import blockmetadb, dbfiles -import onionrstorage -from onionrstorage import removeblock -from onionrblocks import onionrblacklist -from onionrblocks.storagecounter import StorageCounter -from etc.onionrvalues import DATABASE_LOCK_TIMEOUT -from onionrproofs import hashMeetsDifficulty -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - -storage_counter = StorageCounter() - - -def __remove_from_upload(shared_state, block_hash: str): - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - try: - kv.get('blocksToUpload').remove(block_hash) - except ValueError: - pass - - -def __purge_block(shared_state, block_hash, add_to_blacklist = True): - blacklist = None - - removeblock.remove_block(block_hash) - onionrstorage.deleteBlock(block_hash) - __remove_from_upload(shared_state, block_hash) - - if add_to_blacklist: - blacklist = onionrblacklist.OnionrBlackList() - blacklist.addToDB(block_hash) - - -def clean_old_blocks(shared_state): - """Delete expired blocks + old blocks if disk allocation is near full""" - blacklist = onionrblacklist.OnionrBlackList() - # Delete expired blocks - for bHash in blockmetadb.expiredblocks.get_expired_blocks(): - __purge_block(shared_state, bHash, add_to_blacklist=True) - logger.info('Deleted expired block: %s' % (bHash,)) - - while storage_counter.is_full(): - try: - oldest = blockmetadb.get_block_list()[0] - except IndexError: - break - else: - __purge_block(shared_state, bHash, add_to_blacklist=True) - logger.info('Deleted block because of full storage: %s' % (oldest,)) - - -def clean_keys(): - """Delete expired forward secrecy keys""" - conn = sqlite3.connect(dbfiles.user_id_info_db, - timeout=DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - time = epoch.get_epoch() - deleteKeys = [] - - for entry in c.execute( - "SELECT * FROM forwardKeys WHERE expire <= ?", (time,)): - logger.debug('Forward key: %s' % entry[1]) - deleteKeys.append(entry[1]) - - for key in deleteKeys: - logger.debug('Deleting forward key %s' % key) - c.execute("DELETE from forwardKeys where forwardKey = ?", (key,)) - conn.commit() - conn.close() - - onionrusers.deleteExpiredKeys() - - -def clean_blocks_not_meeting_pow(shared_state): - """Clean blocks not meeting min send/rec pow. Used if config.json POW changes""" - block_list = blockmetadb.get_block_list() - for block in block_list: - if not hashMeetsDifficulty(block): - logger.warn( - f"Deleting block {block} because it was stored" + - "with a POW level smaller than current.", terminal=True) - __purge_block(shared_state, block) diff --git a/src/communicatorutils/lookupadders.py b/src/communicatorutils/lookupadders.py deleted file mode 100755 index 788ffa12..00000000 --- a/src/communicatorutils/lookupadders.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Onionr - Private P2P Communication. - -Lookup new peer transport addresses using the communicator -""" -from typing import TYPE_CHECKING -import logger -from onionrutils import stringvalidators -from communicator import peeraction, onlinepeers -from utils import gettransports -import onionrexceptions - - -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def lookup_new_peer_transports_with_communicator(shared_state): - logger.info('Looking up new addresses...') - tryAmount = 1 - newPeers = [] - transports = gettransports.get() - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - - for i in range(tryAmount): - # Download new peer address list from random online peers - if len(newPeers) > 10000: - # Don't get new peers if we have too many queued up - break - try: - peer = onlinepeers.pick_online_peer(kv) - newAdders = peeraction.peer_action(shared_state, peer, action='pex') - except onionrexceptions.OnlinePeerNeeded: - continue - try: - newPeers = newAdders.split(',') - except AttributeError: - pass - else: - # Validate new peers are good format and not already in queue - invalid = [] - for x in newPeers: - x = x.strip() - if not stringvalidators.validate_transport(x) \ - or x in kv.get('newPeers') or x in transports: - # avoid adding if its our address - invalid.append(x) - for x in invalid: - try: - newPeers.remove(x) - except ValueError: - pass - kv.get('newPeers').extend(newPeers) \ No newline at end of file diff --git a/src/communicatorutils/lookupblocks.py b/src/communicatorutils/lookupblocks.py deleted file mode 100755 index 99574b89..00000000 --- a/src/communicatorutils/lookupblocks.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Onionr - Private P2P Communication. - -Lookup new blocks with the communicator using a random connected peer -""" -from typing import TYPE_CHECKING - -from gevent import time - -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV - -import logger -import onionrproofs -from onionrutils import stringvalidators, epoch -from communicator import peeraction, onlinepeers -from coredb.blockmetadb import get_block_list -from utils import reconstructhash -from onionrblocks import onionrblacklist -import onionrexceptions -import config -from etc import onionrvalues -from onionrblocks.storagecounter import StorageCounter -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - -blacklist = onionrblacklist.OnionrBlackList() -storage_counter = StorageCounter() - - -def lookup_blocks_from_communicator(shared_state: 'TooMany'): - logger.info('Looking up new blocks') - tryAmount = 2 - newBlocks = '' - # List of existing saved blocks - existingBlocks = get_block_list() - triedPeers = [] # list of peers we've tried this time around - # Max amount of *new* block hashes to have in queue - maxBacklog = 1560 - lastLookupTime = 0 # Last time we looked up a particular peer's list - new_block_count = 0 - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - for i in range(tryAmount): - # Defined here to reset it each time, time offset is added later - listLookupCommand = 'getblocklist' - if len(kv.get('blockQueue')) >= maxBacklog: - break - if not kv.get('isOnline'): - break - # check if disk allocation is used - if storage_counter.is_full(): - logger.debug( - 'Not looking up new blocks due to maximum amount of disk used') - break - try: - # select random online peer - peer = onlinepeers.pick_online_peer(kv) - except onionrexceptions.OnlinePeerNeeded: - time.sleep(1) - continue - # if we've already tried all the online peers this time around, stop - if peer in triedPeers: - if len(kv.get('onlinePeers')) == len(triedPeers): - break - else: - continue - triedPeers.append(peer) - - # Get the last time we looked up a peer's stamp, - # to only fetch blocks since then. - # Saved in memory only for privacy reasons - try: - lastLookupTime = kv.get('dbTimestamps')[peer] - except KeyError: - lastLookupTime = epoch.get_epoch() - onionrvalues.DEFAULT_EXPIRE - listLookupCommand += '?date=%s' % (lastLookupTime,) - try: - newBlocks = peeraction.peer_action( - shared_state, - peer, listLookupCommand) # get list of new block hashes - except Exception as error: - logger.warn( - f'Could not get new blocks from {peer}.', - error=error) - newBlocks = False - - if newBlocks != False: # noqa - # if request was a success - for i in newBlocks.split('\n'): - if stringvalidators.validate_hash(i): - i = reconstructhash.reconstruct_hash(i) - # if newline seperated string is valid hash - - # if block does not exist on disk + is not already in queue - if i not in existingBlocks: - if i not in kv.get('blockQueue'): - if onionrproofs.hashMeetsDifficulty(i) and \ - not blacklist.inBlacklist(i): - if len(kv.get('blockQueue')) <= 1000000: - # add blocks to download queue - kv.get('blockQueue')[i] = [peer] - new_block_count += 1 - kv.get('dbTimestamps')[peer] = \ - epoch.get_rounded_epoch(roundS=60) - else: - if peer not in kv.get('blockQueue')[i]: - if len(kv.get('blockQueue')[i]) < 10: - kv.get('blockQueue')[i].append(peer) - if new_block_count > 0: - block_string = "" - if new_block_count > 1: - block_string = "s" - logger.info( - f'Discovered {new_block_count} new block{block_string}', - terminal=True) diff --git a/src/communicatorutils/netcheck.py b/src/communicatorutils/netcheck.py deleted file mode 100755 index b45a4361..00000000 --- a/src/communicatorutils/netcheck.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -Onionr - Private P2P Communication. - -Determine if our node is able to use Tor based -on the status of a communicator instance -and the result of pinging onion http servers -""" -import logger -from utils import netutils -from onionrutils import localcommand, epoch -from . import restarttor - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def net_check(shared_state): - """Check if we are connected to the internet. - - or not when we can't connect to any peers - """ - # for detecting if we have received incoming connections recently - rec = False - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - proxy_port = shared_state.get_by_string("NetController").socksPort - - if len(kv.get('onlinePeers')) == 0: - try: - if (epoch.get_epoch() - int(localcommand.local_command - ('/lastconnect'))) <= 60: - kv.put('isOnline', True) - rec = True - except ValueError: - pass - if not rec and not netutils.check_network(torPort=proxy_port): - if not kv.get('shutdown'): - if not shared_state.get_by_string( - "OnionrCommunicatorDaemon").config.get( - 'general.offline_mode', False): - logger.warn('Network check failed, are you connected to ' + - 'the Internet, and is Tor working? ' + - 'This is usually temporary, but bugs and censorship can cause this to persist, in which case you should report it to beardog [at] mailbox.org', # noqa - terminal=True) - restarttor.restart(shared_state) - kv.put('offlinePeers', []) - kv.put('isOnline', False) - else: - kv.put('isOnline', True) diff --git a/src/communicatorutils/proxypicker.py b/src/communicatorutils/proxypicker.py deleted file mode 100755 index 85dc4d5b..00000000 --- a/src/communicatorutils/proxypicker.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -Onionr - Private P2P Communication. - -Pick a proxy to use based on a peer's address -""" -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def pick_proxy(peer_address): - if peer_address.endswith('.onion'): - return 'tor' - elif peer_address.endswith('.i2p'): - return 'i2p' - raise ValueError( - f"Peer address not ending with acceptable domain: {peer_address}") diff --git a/src/communicatorutils/restarttor.py b/src/communicatorutils/restarttor.py deleted file mode 100644 index 35388d05..00000000 --- a/src/communicatorutils/restarttor.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -Onionr - Private P2P Communication. - -Restart Onionr managed Tor -""" -import netcontroller -import config -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def restart(shared_state): - if not config.get('tor.use_existing_tor', False): - net = shared_state.get(netcontroller.NetController) - net.killTor() - net.startTor() diff --git a/src/communicatorutils/uploadblocks/__init__.py b/src/communicatorutils/uploadblocks/__init__.py deleted file mode 100755 index 8bf23be6..00000000 --- a/src/communicatorutils/uploadblocks/__init__.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Onionr - Private P2P Communication. - -Upload blocks in the upload queue to peers from the communicator -""" -from typing import TYPE_CHECKING -from time import sleep -from threading import Thread -from secrets import SystemRandom - -from . import sessionmanager - -from onionrtypes import UserID -import logger -from communicatorutils import proxypicker -import onionrexceptions -from onionrblocks import onionrblockapi as block -from onionrblocks.blockmetadata.fromdata import get_block_metadata_from_data -from onionrutils import stringvalidators, basicrequests -from onionrutils.validatemetadata import validate_metadata -from communicator import onlinepeers -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV - from communicator import OnionrCommunicatorDaemon -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def upload_blocks_from_communicator(shared_state: 'OnionrCommunicatorDaemon'): - """Accept a communicator instance + upload blocks from its upload queue.""" - """when inserting a block, we try to upload - it to a few peers to add some deniability & increase functionality""" - kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") - - session_manager: sessionmanager.BlockUploadSessionManager - session_manager = shared_state.get( - sessionmanager.BlockUploadSessionManager) - tried_peers: UserID = [] - finishedUploads = [] - - SystemRandom().shuffle(kv.get('blocksToUpload')) - - def remove_from_hidden(bl): - sleep(60) - try: - shared_state.get_by_string( - 'PublicAPI').hideBlocks.remove(bl) - except ValueError: - pass - - if len(kv.get('blocksToUpload')) != 0: - for bl in kv.get('blocksToUpload'): - if not stringvalidators.validate_hash(bl): - logger.warn('Requested to upload invalid block', terminal=True) - return - session = session_manager.add_session(bl) - for _ in range(min(len(kv.get('onlinePeers')), 6)): - try: - peer = onlinepeers.pick_online_peer(kv) - if not block.Block(bl).isEncrypted: - if peer in kv.get('plaintextDisabledPeers'): - logger.info(f"Cannot upload plaintext block to peer that denies it {peer}") # noqa - continue - except onionrexceptions.OnlinePeerNeeded: - continue - try: - session.peer_exists[peer] - continue - except KeyError: - pass - try: - if session.peer_fails[peer] > 3: - continue - except KeyError: - pass - if peer in tried_peers: - continue - tried_peers.append(peer) - url = f'http://{peer}/upload' - try: - data = block.Block(bl).getRaw() - if not data: - logger.warn( - f"Couldn't get data for block in upload list {bl}", - terminal=True) - raise onionrexceptions.NoDataAvailable - try: - def __check_metadata(): - metadata = get_block_metadata_from_data(data)[0] - if not validate_metadata(metadata, data): - logger.warn( - f"Metadata for uploading block not valid {bl}") - raise onionrexceptions.InvalidMetadata - __check_metadata() - except onionrexceptions.DataExists: - pass - except( # noqa - onionrexceptions.NoDataAvailable, - onionrexceptions.InvalidMetadata) as _: - finishedUploads.append(bl) - break - proxy_type = proxypicker.pick_proxy(peer) - logger.info( - f"Uploading block {bl[:8]} to {peer}", terminal=True) - resp = basicrequests.do_post_request( - url, data=data, proxyType=proxy_type, - content_type='application/octet-stream') - if resp is not False: - if resp == 'success': - Thread(target=remove_from_hidden, - args=[bl], daemon=True).start() - session.success() - session.peer_exists[peer] = True - elif resp == 'exists': - session.success() - session.peer_exists[peer] = True - else: - session.fail() - session.fail_peer(peer) - shared_state.get_by_string( - 'OnionrCommunicatorDaemon').getPeerProfileInstance( - peer).addScore(-5) - logger.warn( - f'Failed to upload {bl[:8]}, reason: {resp}', - terminal=True) - else: - session.fail() - session_manager.clean_session() - for x in finishedUploads: - try: - kv.get('blocksToUpload').remove(x) - - shared_state.get_by_string( - 'PublicAPI').hideBlocks.remove(x) - - except ValueError: - pass \ No newline at end of file diff --git a/src/communicatorutils/uploadblocks/mixmate/__init__.py b/src/communicatorutils/uploadblocks/mixmate/__init__.py deleted file mode 100644 index 3e63ea99..00000000 --- a/src/communicatorutils/uploadblocks/mixmate/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Onionr - Private P2P Communication. - -Perform block mixing -""" -import time -from typing import List - -import onionrtypes -from onionrblocks import onionrblockapi - -from .pool import UploadPool -from .pool import PoolFullException - -from etc import onionrvalues - -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" -upload_pool = UploadPool(4) - - -def block_mixer(upload_list: List[onionrtypes.BlockHash], - block_to_mix: onionrtypes.BlockHash): - """Delay and mix block inserts. - - Take a block list and a received/created block and add it - to the said block list - """ - bl = onionrblockapi.Block(block_to_mix) - - try: - if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE: - raise ValueError - except TypeError: - pass - if block_to_mix: - upload_list.append(block_to_mix) diff --git a/src/communicatorutils/uploadblocks/mixmate/pool.py b/src/communicatorutils/uploadblocks/mixmate/pool.py deleted file mode 100644 index f5600082..00000000 --- a/src/communicatorutils/uploadblocks/mixmate/pool.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Onionr - Private P2P Communication. - -Upload pool -""" -from typing import List -from secrets import SystemRandom - -import onionrutils -import onionrtypes -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -class PoolFullException(Exception): - """For when the UploadPool is full. - - Raise when a new hash is attempted to be added - """ - - -class PoolNotReady(Exception): - """Raise when UploadPool pool access is attempted without it being full.""" - - -class AlreadyInPool(Exception): - """Raise when a hash already in pool is attempted to be added again.""" - - -class UploadPool: - """Upload pool for mixing blocks together and delaying uploads.""" - - def __init__(self, pool_size: int): - """Create a new pool with a specified max size. - - Uses private var and getter to avoid direct adding - """ - self._pool: List[onionrtypes.BlockHash] = [] - self._pool_size = pool_size - self.birthday = onionrutils.epoch.get_epoch() - - def add_to_pool(self, item: List[onionrtypes.BlockHash]): - """Add a new hash to the pool. Raise PoolFullException if full.""" - if len(self._pool) >= self._pool_size: - raise PoolFullException - if not onionrutils.stringvalidators.validate_hash(item): - raise ValueError - self._pool.append(item) - - def get_pool(self) -> List[onionrtypes.BlockHash]: - """Get the hash pool in secure random order.""" - if len(self._pool) != self._pool_size: - raise PoolNotReady - - final_pool: List[onionrtypes.BlockHash] = list(self._pool) - SystemRandom().shuffle(final_pool) - - self._pool.clear() - self.birthday = onionrutils.epoch.get_epoch() - return final_pool diff --git a/src/communicatorutils/uploadblocks/session.py b/src/communicatorutils/uploadblocks/session.py deleted file mode 100644 index d9b715bf..00000000 --- a/src/communicatorutils/uploadblocks/session.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Onionr - Private P2P Communication. - -Virtual upload "sessions" for blocks -""" -from typing import Union, Dict - -from onionrtypes import UserID -from onionrutils import stringvalidators -from onionrutils import bytesconverter -from onionrutils import epoch -from utils import reconstructhash -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -class UploadSession: - """Manage statistics for an Onionr block upload session. - - accept a block hash (incl. unpadded) as an argument - """ - - def __init__(self, block_hash: Union[str, bytes]): - block_hash = bytesconverter.bytes_to_str(block_hash) - block_hash = reconstructhash.reconstruct_hash(block_hash) - if not stringvalidators.validate_hash(block_hash): - raise ValueError - - self.start_time = epoch.get_epoch() - self.block_hash = reconstructhash.deconstruct_hash(block_hash) - self.total_fail_count: int = 0 - self.total_success_count: int = 0 - self.peer_fails: Dict[UserID, int] = {} - self.peer_exists: Dict[UserID, bool] = {} - - def fail_peer(self, peer): - try: - self.peer_fails[peer] += 1 - except KeyError: - self.peer_fails[peer] = 0 - - def fail(self): - self.total_fail_count += 1 - - def success(self): - self.total_success_count += 1 diff --git a/src/communicatorutils/uploadblocks/sessionmanager.py b/src/communicatorutils/uploadblocks/sessionmanager.py deleted file mode 100644 index 486823a3..00000000 --- a/src/communicatorutils/uploadblocks/sessionmanager.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Onionr - Private P2P Communication. - -Manager for upload 'sessions' -""" -from typing import List, Union, TYPE_CHECKING -if TYPE_CHECKING: - from deadsimplekv import DeadSimpleKV - from session import UploadSession - -from onionrutils import bytesconverter -from etc import onionrvalues -from utils import reconstructhash - -from . import session -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -class BlockUploadSessionManager: - """Holds block UploadSession instances. - - Optionally accepts iterable of sessions to added on init - Arguments: old_session: iterable of old UploadSession objects - """ - - def __init__(self, old_sessions: List = None): - if old_sessions is None: - self.sessions = [] - else: - self.sessions = old_sessions - - def add_session(self, - session_or_block: Union[str, - bytes, - session.UploadSession - ] - ) -> session.UploadSession: - """Create (or add existing) block upload session. - - from a str/bytes block hex hash, existing UploadSession - """ - if isinstance(session_or_block, session.UploadSession): - if session_or_block not in self.sessions: - self.sessions.append(session_or_block) - return session_or_block - try: - return self.get_session(session_or_block) - except KeyError: - pass - # convert bytes hash to str - if isinstance(session_or_block, bytes): - session_or_block = bytesconverter.bytes_to_str(session_or_block) - # intentionally not elif - if isinstance(session_or_block, str): - new_session = session.UploadSession(session_or_block) - self.sessions.append(new_session) - return new_session - raise ValueError - - def get_session(self, - block_hash: Union[str, bytes] - ) -> session.UploadSession: - block_hash = reconstructhash.deconstruct_hash( - bytesconverter.bytes_to_str(block_hash)) - for sess in self.sessions: - if sess.block_hash == block_hash: - return sess - raise KeyError - - def clean_session(self, - specific_session: Union[str, 'UploadSession'] = None): - - comm_inst: 'OnionrCommunicatorDaemon' # type: ignore - comm_inst = self._too_many.get_by_string( # pylint: disable=E1101 type: ignore - "OnionrCommunicatorDaemon") - kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string( - "DeadSimpleKV") - sessions_to_delete = [] - if kv.get('startTime') < 120: - return - onlinePeerCount = len(kv.get('onlinePeers')) - - # If we have no online peers right now, - if onlinePeerCount == 0: - return - - for sess in self.sessions: - # if over 50% of peers that were online for a session have - # become unavailable, don't kill sessions - if sess.total_success_count > onlinePeerCount: - if onlinePeerCount / sess.total_success_count >= 0.5: - return - # Clean sessions if they have uploaded to enough online peers - if sess.total_success_count <= 0: - continue - if (sess.total_success_count / onlinePeerCount) >= \ - onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT: - sessions_to_delete.append(sess) - for sess in sessions_to_delete: - try: - self.sessions.remove(session) - except ValueError: - pass - # TODO cleanup to one round of search - # Remove the blocks from the sessions, upload list, - # and waitforshare list - try: - kv.get('blocksToUpload').remove( - reconstructhash.reconstruct_hash(sess.block_hash)) - except ValueError: - pass - try: - kv.get('blocksToUpload').remove(sess.block_hash) - except ValueError: - pass diff --git a/src/coredb/keydb/__init__.py b/src/coredb/keydb/__init__.py index 16d602d5..13b95148 100644 --- a/src/coredb/keydb/__init__.py +++ b/src/coredb/keydb/__init__.py @@ -1 +1 @@ -from . import addkeys, listkeys, removekeys, userinfo, transportinfo \ No newline at end of file +from . import addkeys, listkeys, removekeys, userinfo \ No newline at end of file diff --git a/src/coredb/keydb/addkeys.py b/src/coredb/keydb/addkeys.py index 9c21c78c..364b0a09 100644 --- a/src/coredb/keydb/addkeys.py +++ b/src/coredb/keydb/addkeys.py @@ -25,9 +25,9 @@ from etc import onionrvalues """ -def add_peer(peerID, name=''): +def add_pub_key(peerID, name=''): """Add a public key to the key database (misleading function name).""" - if peerID in listkeys.list_peers() or peerID == onionrcrypto.pub_key: + if peerID in listkeys.list_pub_keys() or peerID == onionrcrypto.pub_key: raise ValueError("specified id is already known") # This function simply adds a peer to the DB @@ -54,35 +54,3 @@ def add_peer(peerID, name=''): return True - -def add_address(address): - """Add an address to the address database (only tor currently)""" - - if type(address) is None or len(address) == 0: - return False - if stringvalidators.validate_transport(address): - if address in gettransports.get(): - return False - conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - # check if address is in database - # this is safe to do because the address is validated above, but we strip some chars here too just in case - address = address.replace('\'', '').replace(';', '').replace('"', '').replace('\\', '') - for i in c.execute("SELECT * FROM adders WHERE address = ?;", (address,)): - try: - if i[0] == address: - conn.close() - return False - except ValueError: - pass - except IndexError: - pass - - t = (address, 1) - c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t) - conn.commit() - conn.close() - - return True - else: - return False diff --git a/src/coredb/keydb/listkeys.py b/src/coredb/keydb/listkeys.py index ff56b15e..28d58657 100644 --- a/src/coredb/keydb/listkeys.py +++ b/src/coredb/keydb/listkeys.py @@ -22,8 +22,9 @@ import logger from onionrutils import epoch from etc import onionrvalues from .. import dbfiles -from . import userinfo, transportinfo -def list_peers(randomOrder=True, getPow=False, trust=0): +from . import userinfo + +def list_pub_keys(randomOrder=True, getPow=False, trust=0): ''' Return a list of public keys (misleading function name) @@ -59,28 +60,3 @@ def list_peers(randomOrder=True, getPow=False, trust=0): conn.close() return peerList - -def list_adders(randomOrder=True, i2p=True, recent=0): - ''' - Return a list of transport addresses - ''' - conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - if randomOrder: - addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();') - else: - addresses = c.execute('SELECT * FROM adders;') - addressList = [] - for i in addresses: - if len(i[0].strip()) == 0: - continue - addressList.append(i[0]) - conn.close() - testList = list(addressList) # create new list to iterate - for address in testList: - try: - if recent > 0 and (epoch.get_epoch() - transportinfo.get_address_info(address, 'lastConnect')) > recent: - raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0 - except TypeError: - addressList.remove(address) - return addressList \ No newline at end of file diff --git a/src/coredb/keydb/removekeys.py b/src/coredb/keydb/removekeys.py index 46d2d641..1dd0fb39 100644 --- a/src/coredb/keydb/removekeys.py +++ b/src/coredb/keydb/removekeys.py @@ -24,23 +24,7 @@ from onionrutils import mnemonickeys from .. import dbfiles from etc import onionrvalues -def remove_address(address): - ''' - Remove an address from the address database - ''' - if stringvalidators.validate_transport(address): - conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - t = (address,) - c.execute('Delete from adders where address=?;', t) - conn.commit() - conn.close() - - #events.event('address_remove', data = {'address': address}, onionr = core_inst.onionrInst) - return True - else: - return False def remove_user(pubkey: str)->bool: ''' diff --git a/src/coredb/keydb/transportinfo.py b/src/coredb/keydb/transportinfo.py deleted file mode 100644 index 9f03b323..00000000 --- a/src/coredb/keydb/transportinfo.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Onionr - Private P2P Communication. - -get or set transport address meta information -""" -import sqlite3 -from .. import dbfiles -from etc import onionrvalues -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" -info_numbers = { - 'address': 0, - 'type': 1, - 'knownPeer': 2, - 'speed': 3, - 'success': 4, - 'powValue': 5, - 'failure': 6, - 'lastConnect': 7, - 'trust': 8, - 'introduced': 9} - - -def get_address_info(address, info): - """Get info about an address from its database entry. - - address text, 0 - type int, 1 - knownPeer text, 2 - speed int, 3 - success int, 4 - powValue 5 - failure int 6 - lastConnect 7 - trust 8 - introduced 9 - """ - conn = sqlite3.connect( - dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - - command = (address,) - - info = info_numbers[info] - iter_count = 0 - retVal = '' - - for row in c.execute('SELECT * FROM adders WHERE address=?;', command): - for i in row: - if iter_count == info: - retVal = i - break - else: - iter_count += 1 - conn.close() - - return retVal - - -def set_address_info(address, key, data): - """Update an address for a key.""" - conn = sqlite3.connect( - dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT) - c = conn.cursor() - - command = (data, address) - - if key not in info_numbers.keys(): - raise ValueError( - "Got invalid database key when setting address info, must be in whitelist") - else: - c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command) - conn.commit() - conn.close() \ No newline at end of file diff --git a/src/httpapi/__init__.py b/src/httpapi/__init__.py index 101a20c6..c6f0ee16 100755 --- a/src/httpapi/__init__.py +++ b/src/httpapi/__init__.py @@ -1,24 +1,26 @@ """ - Onionr - Private P2P Communication +Onionr - Private P2P Communication - This file registers plugin's flask blueprints for the client http server -""" -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . +Register plugins flask blueprints for the client http server """ import onionrplugins import config +from .fdsafehandler import FDSafeHandler +""" +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + def load_plugin_blueprints(flaskapp, blueprint: str = 'flask_blueprint'): """Iterate enabled plugins and load any http endpoints they have""" diff --git a/src/httpapi/miscclientapi/addpeer.py b/src/httpapi/miscclientapi/addpeer.py deleted file mode 100644 index c01a8098..00000000 --- a/src/httpapi/miscclientapi/addpeer.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Onionr - Private P2P Communication. - -add a transport address to the db -""" -from onionrutils.stringvalidators import validate_transport -from coredb.keydb.addkeys import add_address -from coredb.keydb.listkeys import list_adders -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def add_peer(peer): - - if peer in list_adders(): - return "already added" - if add_address(peer): - return "success" - else: - return "failure, invalid address" diff --git a/src/httpapi/miscclientapi/endpoints.py b/src/httpapi/miscclientapi/endpoints.py index 6fbfcade..ed10d969 100644 --- a/src/httpapi/miscclientapi/endpoints.py +++ b/src/httpapi/miscclientapi/endpoints.py @@ -15,14 +15,10 @@ import unpaddedbase32 from httpapi import apiutils import onionrcrypto import config -from netcontroller import NetController -from onionrstatistics.serializeddata import SerializedData from onionrutils import mnemonickeys from onionrutils import bytesconverter from etc import onionrvalues from utils import reconstructhash -from utils.gettransports import get as get_tor -from .addpeer import add_peer """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -49,16 +45,6 @@ class PrivateEndpoints: private_endpoints_bp = Blueprint('privateendpoints', __name__) self.private_endpoints_bp = private_endpoints_bp - @private_endpoints_bp.route('/addpeer/', methods=['post']) - def add_peer_endpoint(name): - result = add_peer(name) - if result == "success": - return Response("success") - else: - if "already" in result: - return Response(result, 409) - else: - return Response(result, 400) @private_endpoints_bp.route('/www/', endpoint='www') def wwwPublic(path): @@ -75,33 +61,13 @@ class PrivateEndpoints: def get_is_atty(): return Response(str(sys_stdout.isatty()).lower()) - @private_endpoints_bp.route('/hitcount') - def get_hit_count(): - return Response(str(client_api.publicAPI.hitCount)) @private_endpoints_bp.route('/ping') def ping(): # Used to check if client api is working return Response("pong!") - @private_endpoints_bp.route('/lastconnect') - def last_connect(): - return Response(str(client_api.publicAPI.lastRequest)) - @private_endpoints_bp.route('/waitforshare/', methods=['post']) - def wait_for_share(name): - """Prevent the **public** api from sharing blocks. - - Used for blocks we created usually - """ - if not name.isalnum(): - raise ValueError('block hash needs to be alpha numeric') - name = reconstructhash.reconstruct_hash(name) - if name in client_api.publicAPI.hideBlocks: - return Response("will be removed") - else: - client_api.publicAPI.hideBlocks.append(name) - return Response("added") @private_endpoints_bp.route('/shutdown') def shutdown(): @@ -116,17 +82,6 @@ class PrivateEndpoints: def get_hidden_blocks(): return Response('\n'.join(client_api.publicAPI.hideBlocks)) - @private_endpoints_bp.route('/getstats') - def get_stats(): - """Return serialized node statistics.""" - while True: - try: - return Response(client_api._too_many.get( - SerializedData).get_stats()) - except AttributeError: - pass - except FileNotFoundError: - pass @private_endpoints_bp.route('/getuptime') def show_uptime(): @@ -159,32 +114,6 @@ class PrivateEndpoints: def get_os_system(): return Response(platform.system().lower()) - @private_endpoints_bp.route('/gettorsocks') - def get_tor_socks(): - while True: - try: - return Response( - str( - g.too_many.get_by_string( - 'NetController').socksPort)) - except KeyError: - sleep(0.1) - - @private_endpoints_bp.route('/torready') - def is_tor_ready(): - """If Tor is starting up, the web UI is not ready to be used.""" - try: - return Response( - str(g.too_many.get_by_string('NetController').readyState).lower()) - except KeyError: - return Response("false") - - @private_endpoints_bp.route('/gettoraddress') - def get_tor_address(): - """Return public Tor v3 Onion address for this node""" - if not config.get('general.security_level', 0) == 0: - abort(404) - return Response(get_tor()[0]) @private_endpoints_bp.route('/getgeneratingblocks') def get_generating_blocks() -> Response: @@ -195,10 +124,3 @@ class PrivateEndpoints: )) ) - @private_endpoints_bp.route('/getblockstoupload') - def get_blocks_to_upload() -> Response: - return Response( - ','.join( - g.too_many.get_by_string('DeadSimpleKV').get('blocksToUpload') - ) - ) diff --git a/src/httpapi/miscpublicapi/__init__.py b/src/httpapi/miscpublicapi/__init__.py deleted file mode 100755 index 2f5f9c56..00000000 --- a/src/httpapi/miscpublicapi/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from . import announce, upload, getblocks, endpoints - -announce = announce.handle_announce # endpoint handler for accepting peer announcements -upload = upload.accept_upload # endpoint handler for accepting public uploads -public_block_list = getblocks.get_public_block_list # endpoint handler for getting block lists -public_get_block_data = getblocks.get_block_data # endpoint handler for responding to peers requests for block data \ No newline at end of file diff --git a/src/httpapi/miscpublicapi/announce.py b/src/httpapi/miscpublicapi/announce.py deleted file mode 100755 index 29a78a43..00000000 --- a/src/httpapi/miscpublicapi/announce.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Onionr - Private P2P Communication. - -Handle announcements to the public API server -""" -from flask import Response, g -import deadsimplekv - -import logger -from etc import onionrvalues -from onionrutils import stringvalidators, bytesconverter -import filepaths -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def handle_announce(request): - """accept announcement posts, validating POW - clientAPI should be an instance of the clientAPI server running, - request is a instance of a flask request - """ - resp = 'failure' - newNode = '' - - try: - newNode = request.form['node'].encode() - except KeyError: - logger.warn('No node specified for upload') - else: - newNode = bytesconverter.bytes_to_str(newNode) - announce_queue = deadsimplekv.DeadSimpleKV(filepaths.announce_cache) - announce_queue_list = announce_queue.get('new_peers') - if announce_queue_list is None: - announce_queue_list = [] - else: - if len(announce_queue_list) >= onionrvalues.MAX_NEW_PEER_QUEUE: - newNode = '' - - if stringvalidators.validate_transport(newNode) and \ - newNode not in announce_queue_list: - g.shared_state.get( - deadsimplekv.DeadSimpleKV).get('newPeers').append(newNode) - announce_queue.put('new_peers', - announce_queue_list.append(newNode)) - announce_queue.flush() - resp = 'Success' - - resp = Response(resp) - if resp == 'failure': - return resp, 406 - return resp \ No newline at end of file diff --git a/src/httpapi/miscpublicapi/endpoints.py b/src/httpapi/miscpublicapi/endpoints.py deleted file mode 100644 index 9c575c07..00000000 --- a/src/httpapi/miscpublicapi/endpoints.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Onionr - Private P2P Communication. - -Misc public API endpoints too small to need their own file -and that need access to the public api inst -""" -from flask import Response, Blueprint, request, send_from_directory, abort, g -from . import getblocks, upload, announce -from coredb import keydb -import config -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -class PublicEndpoints: - def __init__(self, public_api): - - public_endpoints_bp = Blueprint('publicendpoints', __name__) - self.public_endpoints_bp = public_endpoints_bp - - @public_endpoints_bp.route('/') - def banner(): - # Display info to people who visit a node address in their browser - try: - with open('../static-data/index.html', 'r') as html: - resp = Response(html.read(), mimetype='text/html') - except FileNotFoundError: - resp = Response("") - return resp - - @public_endpoints_bp.route('/getblocklist') - def get_block_list(): - """Get a list of blocks, optionally filtered by epoch time stamp, - excluding those hidden""" - return getblocks.get_public_block_list(public_api, request) - - @public_endpoints_bp.route('/getdata/') - def get_block_data(name): - # Share data for a block if we have it and it isn't hidden - return getblocks.get_block_data(public_api, name) - - @public_endpoints_bp.route('/www/') - def www_public(path): - # A way to share files directly over your .onion - if not config.get("www.public.run", True): - abort(403) - return send_from_directory( - config.get('www.public.path', 'static-data/www/public/'), path) - - - @public_endpoints_bp.route('/plaintext') - def plaintext_enabled_endpoint(): - return Response(str(config.get("general.store_plaintext_blocks", True)).lower()) - - @public_endpoints_bp.route('/ping') - def ping(): - # Endpoint to test if nodes are up - return Response("pong!") - - @public_endpoints_bp.route('/pex') - def peer_exchange(): - response = ','.join(keydb.listkeys.list_adders(recent=3600)) - if len(response) == 0: - response = '' - return Response(response) - - @public_endpoints_bp.route('/announce', methods=['post']) - def accept_announce(): - """Accept announcements with pow token to prevent spam""" - g.shared_state = public_api._too_many - resp = announce.handle_announce(request) - return resp - - @public_endpoints_bp.route('/upload', methods=['post']) - def upload_endpoint(): - """Accept file uploads. - In the future this will be done more often than on creation - to speed up block sync - """ - return upload.accept_upload(request) diff --git a/src/httpapi/miscpublicapi/getblocks.py b/src/httpapi/miscpublicapi/getblocks.py deleted file mode 100755 index cc64e6c9..00000000 --- a/src/httpapi/miscpublicapi/getblocks.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Onionr - Private P2P Communication. - -Public endpoints to get block data and lists -""" -from flask import Response, abort - -import config -from onionrutils import bytesconverter, stringvalidators -from coredb import blockmetadb -from utils import reconstructhash -from onionrblocks import BlockList -from onionrblocks.onionrblockapi import Block -from .. import apiutils -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def get_public_block_list(public_API, request): - # Provide a list of our blocks, with a date offset - date_adjust = request.args.get('date') - type_filter = request.args.get('type') - b_list = blockmetadb.get_block_list(date_rec=date_adjust) - share_list = '' - if config.get('general.hide_created_blocks', True): - for b in public_API.hideBlocks: - if b in b_list: - # Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block - b_list.remove(b) - for b in b_list: - if type_filter: - if Block(b, decrypt=False).getType() != type_filter: - continue - share_list += '%s\n' % (reconstructhash.deconstruct_hash(b),) - return Response(share_list) - - -def get_block_data(public_API, b_hash): - """return block data by hash unless we are hiding it""" - resp = '' - b_hash = reconstructhash.reconstruct_hash(b_hash) - if stringvalidators.validate_hash(b_hash): - if not config.get('general.hide_created_blocks', True) \ - or b_hash not in public_API.hideBlocks: - if b_hash in public_API._too_many.get(BlockList).get(): - block = apiutils.GetBlockData().get_block_data( - b_hash, raw=True, decrypt=False) - try: - # Encode in case data is binary - block = block.encode('utf-8') - except AttributeError: - # 404 if no block data - if not block: - abort(404) - if not len(block): - abort(404) - resp = block - if len(resp) == 0: - abort(404) - resp = "" - # Has to be octet stream, otherwise binary data fails hash check - return Response(resp, mimetype='application/octet-stream') diff --git a/src/httpapi/miscpublicapi/upload.py b/src/httpapi/miscpublicapi/upload.py deleted file mode 100755 index 9e57f1e4..00000000 --- a/src/httpapi/miscpublicapi/upload.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Onionr - Private P2P Communication. - -Accept block uploads to the public API server -""" -import sys - -from gevent import spawn -from flask import Response -from flask import abort -from flask import g - -from onionrutils import localcommand -from onionrblocks import blockimporter -import onionrexceptions -import logger -import config - -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -def accept_upload(request): - """Accept uploaded blocks to our public Onionr protocol API server""" - resp = 'failure' - data = request.get_data() - data_size = sys.getsizeof(data) - b_hash = None - if data_size < 30: - resp = 'size' - elif data_size < 100000000: - try: - b_hash = blockimporter.import_block_from_data(data) - if b_hash: - # Upload mixing is where a node will hide and reupload a block - # to act like it is also a creator - # This adds deniability but is very slow - if g.too_many.get_by_string( - "DeadSimpleKV").get('onlinePeers') and \ - config.get('general.upload_mixing', False): - spawn( - localcommand.local_command, - '/daemon-event/upload_event', - post=True, - is_json=True, - post_data={'block': b_hash} - ).get(timeout=10) - resp = 'success' - else: - resp = 'failure' - logger.warn( - f'Error encountered importing uploaded block {b_hash}') - except onionrexceptions.BlacklistedBlock: - logger.debug('uploaded block is blacklisted') - resp = 'failure' - except onionrexceptions.InvalidProof: - resp = 'proof' - except onionrexceptions.DataExists: - resp = 'exists' - except onionrexceptions.PlaintextNotSupported: - logger.debug(f"attempted plaintext upload to us: {b_hash}") - resp = 'failure' - except onionrexceptions.InvalidMetadata: - logger.debug( - f'uploaded block {b_hash} has invalid metadata') - resp = 'failure' - if resp == 'failure': - abort(400) - elif resp == 'size': - resp = Response(resp, 400) - logger.warn( - f'Error importing uploaded block, invalid size {b_hash}') - elif resp == 'proof': - resp = Response(resp, 400) - if b_hash: - logger.warn( - f'Error importing uploaded block, invalid proof {b_hash}') - else: - logger.warn( - 'Error importing uploaded block, invalid proof') - else: - resp = Response(resp) - return resp diff --git a/src/httpapi/sse/private/__init__.py b/src/httpapi/sse/private/__init__.py index 4213fd4f..9d9ffaef 100644 --- a/src/httpapi/sse/private/__init__.py +++ b/src/httpapi/sse/private/__init__.py @@ -13,7 +13,6 @@ from onionrblocks.onionrblockapi import Block from coredb.dbfiles import block_meta_db from coredb.blockmetadb import get_block_list from onionrutils.epoch import get_epoch -from onionrstatistics.transports.tor import TorStats from .. import wrapper """ This program is free software: you can redistribute it and/or modify @@ -44,15 +43,6 @@ def stream_hello(): return SSEWrapper.handle_sse_request(print_hello) -@private_sse_blueprint.route('/torcircuits') -def stream_tor_circuits(): - tor_stats = g.too_many.get(TorStats) - def circuit_stat_stream(): - while True: - yield "data: " + tor_stats.get_json() + "\n\n" - sleep(10) - return SSEWrapper.handle_sse_request(circuit_stat_stream) - @private_sse_blueprint.route('/recentblocks') def stream_recent_blocks(): def _compile_json(b_list): diff --git a/src/lan/__init__.py b/src/lan/__init__.py deleted file mode 100644 index ea30efd5..00000000 --- a/src/lan/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Onionr - Private P2P Communication. - -LAN manager -""" -from typing import TYPE_CHECKING -from threading import Thread -if TYPE_CHECKING: - from toomanyobjs import TooMany - -from .discover import learn_services, advertise_service -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - - -class LANManager: - """Initialize and start/top LAN transport threads.""" - - def __init__(self, too_many: "TooMany"): - self.too_many = too_many - self.peers: "exploded IP Address string" = [] - - def start(self): - Thread(target=learn_services, daemon=True).start() - Thread(target=advertise_service, daemon=True).start() - diff --git a/src/lan/client/__init__.py b/src/lan/client/__init__.py deleted file mode 100644 index 6b1dab66..00000000 --- a/src/lan/client/__init__.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Onionr - Private P2P Communication. - -LAN transport client thread -""" -import requests - -from typing import Set - -from onionrtypes import LANIP -import logger -from coredb.blockmetadb import get_block_list -from onionrblocks.blockimporter import import_block_from_data -import onionrexceptions -from ..server import ports -from onionrproofs import hashMeetsDifficulty - -from threading import Thread -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - -connected_lan_peers: Set[LANIP] = set([]) - - -def _lan_work(peer: LANIP): - def _sync_peer(url): - our_blocks = get_block_list() - blocks = requests.get(url + 'blist/0').text.splitlines() - for block in blocks: - if block not in our_blocks and hashMeetsDifficulty(block): - try: - import_block_from_data( - requests.get( - url + f'get/{block}', stream=True).raw.read(6000000)) - except onionrexceptions.InvalidMetadata: - logger.warn(f"Could not get {block} from lan peer") - except onionrexceptions.InvalidProof: - logger.warn( - f"Invalid proof for {block} from lan peer {peer}", terminal=True) - break - - for port in ports: - try: - root = f'http://{peer}:{port}/' - if requests.get(f'{root}ping').text != 'onionr!': - connected_lan_peers.remove(peer) - else: - logger.info(f'[LAN] Connected to {peer}:{port}', terminal=True) - while True: - try: - _sync_peer(root) - except requests.exceptions.ConnectionError: - break - break - except requests.exceptions.ConnectionError: - pass - else: - connected_lan_peers.remove(peer) - - -def connect_peer(peer: LANIP): - if peer not in connected_lan_peers: - connected_lan_peers.add(peer) - Thread(target=_lan_work, args=[peer], daemon=True).start() - diff --git a/src/lan/discover.py b/src/lan/discover.py deleted file mode 100644 index 42827c4f..00000000 --- a/src/lan/discover.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Onionr - Private P2P Communication. - -Discover and publish private-network -""" -import socket -import struct -from ipaddress import ip_address - -from .getip import lan_ips, best_ip -from utils.bettersleep import better_sleep -from . import client -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" -MCAST_GRP = '224.0.0.112' -MCAST_PORT = 1337 -IS_ALL_GROUPS = True -ANNOUNCE_LOOP_SLEEP = 30 - - -def learn_services(): - """Take a list to infintely add lan service info to.""" - - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if IS_ALL_GROUPS: - # on this port, receives ALL multicast groups - sock.bind(('', MCAST_PORT)) - else: - # on this port, listen ONLY to MCAST_GRP - sock.bind((MCAST_GRP, MCAST_PORT)) - mreq = struct.pack("4sl", socket.inet_aton(MCAST_GRP), socket.INADDR_ANY) - - sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) - - while True: - service_ips = sock.recv(200).decode('utf-8') - if 'onionr' not in service_ips: - continue - service_ips = service_ips.replace('onionr-', '').split('-') - - for service in service_ips: - try: - ip_address(service) - if not ip_address(service).is_private: - raise ValueError - if service in lan_ips: - raise ValueError - except ValueError: - pass - else: - client.connect_peer(service) - - -def advertise_service(specific_ips=None): - # regarding socket.IP_MULTICAST_TTL - # --------------------------------- - # for all packets sent, after three hops on the network the packet will not - # be re-sent/broadcast - # (see https://www.tldp.org/HOWTO/Multicast-HOWTO-6.html) - MULTICAST_TTL = 3 - - ips = best_ip - if not ips: - return - - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) - sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, MULTICAST_TTL) - while True: - sock.sendto(f"onionr-{ips}".encode('utf-8'), (MCAST_GRP, MCAST_PORT)) - better_sleep(ANNOUNCE_LOOP_SLEEP) - diff --git a/src/lan/getip.py b/src/lan/getip.py deleted file mode 100644 index 0031734b..00000000 --- a/src/lan/getip.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Onionr - Private P2P Communication. - -Identify LAN ip addresses and determine the best one -""" -from ipaddress import IPv4Address - -from psutil import net_if_addrs -from socket import AF_INET - -import logger -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" - -lan_ips = [] - -# https://psutil.readthedocs.io/en/latest/#psutil.net_if_addrs -def _get_lan_ips(): - for interface in net_if_addrs().keys(): - for address in net_if_addrs()[interface]: - # Don't see benefit in ipv6, so just check for v4 addresses - if address[0] == AF_INET: - # Mark the address for use in LAN if it is a private address - if IPv4Address(address[1]).is_private and not IPv4Address(address[1]).is_loopback: - lan_ips.append(address[1]) -try: - _get_lan_ips() -except OSError: - logger.warn("Could not identify LAN ips due to OSError.") - -# These are more likely to be actual local subnets rather than VPNs -for ip in lan_ips: - if '192.168' in ip: - best_ip = ip - break -else: - try: - best_ip = lan_ips[0] - except IndexError: - best_ip = "" diff --git a/src/lan/server/__init__.py b/src/lan/server/__init__.py deleted file mode 100644 index ce52a8c2..00000000 --- a/src/lan/server/__init__.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Onionr - Private P2P Communication. - -LAN transport server thread -""" -import ipaddress -import time -from threading import Thread - -from gevent.pywsgi import WSGIServer -from flask import Flask -from flask import Response -from flask import request -from flask import abort - -from onionrblocks.onionrblockapi import Block -from httpapi.fdsafehandler import FDSafeHandler -from netcontroller import get_open_port -import config -from coredb.blockmetadb import get_block_list -from lan.getip import best_ip, lan_ips -from onionrutils import stringvalidators -from httpapi.miscpublicapi.upload import accept_upload -import logger -from utils.bettersleep import better_sleep -""" - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -""" -ports = range(1337, 1340) -_start_time = time.time() - - -class LANServer: - def __init__(self, shared_state): - app = Flask(__name__) - self.app = app - self.host = config.get('lan.bind_ip', '') - self.server = None - if self.host == '': - self.host = best_ip - self.port = None - - @app.before_request - def dns_rebinding_prevention(): - if request.remote_addr in lan_ips or \ - ipaddress.ip_address(request.remote_addr).is_loopback: - if time.time() - _start_time > 600: - abort(403) - if request.host != f'{self.host}:{self.port}': - logger.warn('Potential DNS rebinding attack on LAN server:') - logger.warn( - f'Hostname {request.host} was used instead of {self.host}:{self.port}') # noqa - abort(403) - - @app.route('/blist/