diff --git a/src/apiservers/__init__.py b/src/apiservers/__init__.py
index 82599cc7..f9926150 100755
--- a/src/apiservers/__init__.py
+++ b/src/apiservers/__init__.py
@@ -4,7 +4,6 @@ Public is net-facing server meant for other nodes
Private is meant for controlling and accessing this node
"""
-from . import public, private
+from . import private
-PublicAPI = public.PublicAPI
ClientAPI = private.PrivateAPI
diff --git a/src/apiservers/private/__init__.py b/src/apiservers/private/__init__.py
index 657c30b7..a480ce96 100644
--- a/src/apiservers/private/__init__.py
+++ b/src/apiservers/private/__init__.py
@@ -16,20 +16,20 @@ import logger
from etc import waitforsetvar
from . import register_private_blueprints
import config
-from .. import public
+
"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
"""
@@ -50,7 +50,7 @@ class PrivateAPI:
self.startTime = epoch.get_epoch()
app = flask.Flask(__name__)
-
+
bind_port = int(config.get('client.client.port', 59496))
self.bindPort = bind_port
@@ -77,8 +77,7 @@ class PrivateAPI:
"""Start client gevent API web server with flask client app."""
waitforsetvar.wait_for_set_var(self, "_too_many")
fd_handler = httpapi.fdsafehandler.FDSafeHandler
- self.publicAPI = self._too_many.get( # pylint: disable=E1101
- public.PublicAPI)
+
self.httpServer = WSGIServer((self.host, self.bindPort),
self.app, log=None,
handler_class=fd_handler)
diff --git a/src/apiservers/public/__init__.py b/src/apiservers/public/__init__.py
deleted file mode 100644
index 2a2c610a..00000000
--- a/src/apiservers/public/__init__.py
+++ /dev/null
@@ -1,76 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-This file handles all incoming http requests
-to the public api server, using Flask
-"""
-import time
-import threading
-import flask
-from gevent.pywsgi import WSGIServer
-from httpapi import apiutils, security, fdsafehandler, miscpublicapi
-import logger
-import config
-import filepaths
-from utils import gettransports
-from etc import onionrvalues, waitforsetvar
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def _get_tor_adder(pub_api):
- transports = []
- while len(transports) == 0:
- transports = gettransports.get()
- time.sleep(0.3)
- pub_api.torAdder = transports[0]
-
-
-class PublicAPI:
- """The new client api server, isolated from the public api."""
-
- def __init__(self):
- """Setup the public api app."""
- app = flask.Flask('PublicAPI')
- app.config['MAX_CONTENT_LENGTH'] = 5 * 1024 * 1024
- self.i2pEnabled = config.get('i2p.host', False)
- self.hideBlocks = [] # Blocks to be denied sharing
- self.host = apiutils.setbindip.set_bind_IP(
- filepaths.public_API_host_file)
-
- threading.Thread(target=_get_tor_adder,
- args=[self], daemon=True).start()
-
- self.torAdder = ""
- self.bindPort = config.get('client.public.port')
- self.lastRequest = 0
- # total rec requests to public api since server started
- self.hitCount = 0
- self.config = config
- self.API_VERSION = onionrvalues.API_VERSION
- logger.info('Running public api on %s:%s' % (self.host, self.bindPort))
-
- app.register_blueprint(
- security.public.PublicAPISecurity(self).public_api_security_bp)
- app.register_blueprint(
- miscpublicapi.endpoints.PublicEndpoints(self).public_endpoints_bp)
- self.app = app
-
- def start(self):
- """Start the Public API server."""
- waitforsetvar.wait_for_set_var(self, "_too_many")
- self.httpServer = WSGIServer((self.host, self.bindPort),
- self.app, log=None,
- handler_class=fdsafehandler.FDSafeHandler)
- self.httpServer.serve_forever()
diff --git a/src/communicator/README.md b/src/communicator/README.md
deleted file mode 100644
index 9925f247..00000000
--- a/src/communicator/README.md
+++ /dev/null
@@ -1,15 +0,0 @@
-# Onionr Communicator
-
-Onionr communicator is the Onionr client. It "connects" to remote Onionr peers and does things such as:
-
-* Finding new peers
-* Uploading blocks
-* Downloading blocks
-* Daemon maintenance/housekeeping
-
-## Files
-
-* \_\_init\_\_.py: Contains the main communicator code. Inits and launches the communicator and sets up the timers
-* peeraction.py: contains a function to send commands to remote peers
-* bootstrappers.py: adds peers from the bootstrap list to the communicator to try to connect to them
-* onlinepeers: management of the online peer pool for the communicator
\ No newline at end of file
diff --git a/src/communicator/__init__.py b/src/communicator/__init__.py
deleted file mode 100755
index 9a84cc8b..00000000
--- a/src/communicator/__init__.py
+++ /dev/null
@@ -1,206 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-This file contains both the OnionrCommunicate class for
-communcating with peers and code to operate as a daemon,
-getting commands from the command queue database
-"""
-import time
-
-import config
-import logger
-import onionrpeers
-import onionrplugins as plugins
-from . import onlinepeers
-from . import uploadqueue
-from communicatorutils import downloadblocks
-from communicatorutils import lookupblocks
-from communicatorutils import lookupadders
-from communicatorutils import connectnewpeers
-from communicatorutils import uploadblocks
-from communicatorutils import announcenode, deniableinserts
-from communicatorutils import cooldownpeer
-from communicatorutils import housekeeping
-from communicatorutils import netcheck
-from onionrthreads import add_onionr_thread
-from onionrcommands.openwebinterface import get_url
-from netcontroller import NetController
-from . import bootstrappeers
-from . import daemoneventhooks
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-config.reload()
-
-
-class OnionrCommunicatorDaemon:
- def __init__(self, shared_state, developmentMode=None):
- if developmentMode is None:
- developmentMode = config.get(
- 'general.dev_mode', False)
-
- # configure logger and stuff
- self.config = config
- self.shared_state = shared_state # TooManyObjects module
- shared_state.add(self)
-
- # populate kv values
- self.kv = self.shared_state.get_by_string('DeadSimpleKV')
-
- if config.get('general.offline_mode', False):
- self.kv.put('isOnline', False)
-
- # initialize core with Tor socks port being 3rd argument
- self.proxyPort = shared_state.get(NetController).socksPort
-
- self.upload_session_manager = self.shared_state.get(
- uploadblocks.sessionmanager.BlockUploadSessionManager)
- self.shared_state.share_object()
-
- # loop time.sleep delay in seconds
- self.delay = 5
-
- # amount of threads running by name, used to prevent too many
- self.threadCounts = {}
-
- # Loads in and starts the enabled plugins
- plugins.reload()
-
- # extends our upload list and saves our list when Onionr exits
- uploadqueue.UploadQueue(self)
-
- add_onionr_thread(
- lookupblocks.lookup_blocks_from_communicator,
- [self.shared_state], 25, 3)
-
- add_onionr_thread(
- downloadblocks.download_blocks_from_communicator,
- [self.shared_state],
- config.get('timers.getBlocks', 10), 1)
-
- add_onionr_thread(onlinepeers.clear_offline_peer, [self.kv], 58)
-
- add_onionr_thread(
- housekeeping.clean_old_blocks, [self.shared_state], 10, 1)
-
- # Discover new peers
- add_onionr_thread(
- lookupadders.lookup_new_peer_transports_with_communicator,
- [shared_state], 60, 3)
-
- # Timer for adjusting which peers
- # we actively communicate to at any given time,
- # to avoid over-using peers
- add_onionr_thread(
- cooldownpeer.cooldown_peer, [self.shared_state], 30, 60)
-
- # Timer to read the upload queue and upload the entries to peers
- add_onionr_thread(
- uploadblocks.upload_blocks_from_communicator,
- [self.shared_state], 5, 1)
-
- # This timer creates deniable blocks,
- # in an attempt to further obfuscate block insertion metadata
- if config.get('general.insert_deniable_blocks', True):
- add_onionr_thread(
- deniableinserts.insert_deniable_block, [], 180, 10)
-
- if config.get('transports.tor', True):
- # Timer to check for connectivity,
- # through Tor to various high-profile onion services
- add_onionr_thread(netcheck.net_check, [shared_state], 500, 60)
-
- # Announce the public API server transport address
- # to other nodes if security level allows
- if config.get('general.security_level', 1) == 0 \
- and config.get('general.announce_node', True):
- # Default to high security level incase config breaks
- add_onionr_thread(
- announcenode.announce_node, [self.shared_state], 600, 60)
- else:
- logger.debug('Will not announce node.')
-
- add_onionr_thread(onionrpeers.peer_cleanup, [], 300, 300)
-
- add_onionr_thread(housekeeping.clean_keys, [], 15, 1)
-
- if config.get('general.use_bootstrap_list', True):
- bootstrappeers.add_bootstrap_list_to_peer_list(
- self.kv, [], db_only=True)
-
- daemoneventhooks.daemon_event_handlers(shared_state)
-
- get_url()
- if not config.get('onboarding.done', True):
- logger.info(
- 'First run detected. Run openhome to get setup.',
- terminal=True)
-
- while not config.get('onboarding.done', True) and \
- not self.shared_state.get_by_string(
- 'DeadSimpleKV').get('shutdown'):
- try:
- time.sleep(2)
- except KeyboardInterrupt:
- self.shared_state.get_by_string(
- 'DeadSimpleKV').put('shutdown', True)
-
- # Main daemon loop, mainly for calling timers,
- # don't do any complex operations here to avoid locking
- try:
- while not self.shared_state.get_by_string(
- 'DeadSimpleKV').get('shutdown'):
- time.sleep(self.delay)
- except KeyboardInterrupt:
- self.shared_state.get_by_string(
- 'DeadSimpleKV').put('shutdown', True)
-
- logger.info(
- 'Goodbye. (Onionr is cleaning up, and will exit)', terminal=True)
-
- def decrementThreadCount(self, threadName):
- """Decrement amount of a thread name if more than zero.
-
- called when a function meant to be run in a thread ends
- """
- try:
- if self.threadCounts[threadName] > 0:
- self.threadCounts[threadName] -= 1
- except KeyError:
- pass
-
- def peerCleanup(self):
- """This just calls onionrpeers.cleanupPeers.
-
- Remove dead or bad peers (offline too long, too slow)"""
- onionrpeers.peer_cleanup()
- self.decrementThreadCount('peerCleanup')
-
- def getPeerProfileInstance(self, peer):
- """Gets a peer profile instance from the list of profiles"""
- for i in self.kv.get('peerProfiles'):
- # if the peer's profile is already loaded, return that
- if i.address == peer:
- retData = i
- break
- else:
- # if the peer's profile is not loaded, return a new one.
- # connectNewPeer also adds it to the list on connect
- retData = onionrpeers.PeerProfiles(peer)
- self.kv.get('peerProfiles').append(retData)
- return retData
-
-
-def startCommunicator(shared_state):
- OnionrCommunicatorDaemon(shared_state)
diff --git a/src/communicator/bootstrappeers.py b/src/communicator/bootstrappeers.py
deleted file mode 100644
index 191c3165..00000000
--- a/src/communicator/bootstrappeers.py
+++ /dev/null
@@ -1,36 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-add bootstrap peers to the communicator peer list
-"""
-from typing import TYPE_CHECKING
-
-from utils import readstatic, gettransports
-from coredb import keydb
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-bootstrap_peers = readstatic.read_static('bootstrap-nodes.txt').split(',')
-
-
-def add_bootstrap_list_to_peer_list(kv, peerList, db_only=False):
- """Add the bootstrap list to the peer list (no duplicates)."""
- for i in bootstrap_peers:
- # Add bootstrap peers to peerList (does not save them)
- # Don't add them if they're already added or in the offline list
- if i not in peerList and i not in kv.get('offlinePeers') \
- and i not in gettransports.get() and len(str(i).strip()) > 0:
- if not db_only:
- peerList.append(i)
- keydb.addkeys.add_address(i)
diff --git a/src/communicator/daemoneventhooks/__init__.py b/src/communicator/daemoneventhooks/__init__.py
deleted file mode 100644
index f43ac54f..00000000
--- a/src/communicator/daemoneventhooks/__init__.py
+++ /dev/null
@@ -1,82 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Hooks to handle daemon events
-"""
-from threading import Thread
-
-from .removefrominsertqueue import remove_from_insert_queue
-
-from typing import TYPE_CHECKING
-
-from gevent import sleep
-
-from communicatorutils.uploadblocks import mixmate
-from communicatorutils import restarttor
-
-if TYPE_CHECKING:
- from toomanyobjs import TooMany
- from deadsimplekv import DeadSimpleKV
- from communicator import OnionrCommunicatorDaemon
- from httpapi.daemoneventsapi import DaemonEventsBP
- from onionrtypes import BlockHash
- from apiservers import PublicAPI
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def daemon_event_handlers(shared_state: 'TooMany'):
- def _get_inst(class_name: str):
- while True:
- try:
- return shared_state.get_by_string(class_name)
- except KeyError:
- sleep(0.2)
- comm_inst = _get_inst('OnionrCommunicatorDaemon')
- public_api: 'PublicAPI' = _get_inst('PublicAPI')
- events_api: 'DaemonEventsBP' = _get_inst('DaemonEventsBP')
- kv: 'DeadSimpleKV' = _get_inst('DeadSimpleKV')
-
- def remove_from_insert_queue_wrapper(block_hash: 'BlockHash'):
- remove_from_insert_queue(comm_inst, block_hash)
- return "removed"
-
- def print_test(text=''):
- print("It works!", text)
- return f"It works! {text}"
-
- def upload_event(block: 'BlockHash' = ''):
- if not block:
- raise ValueError
- public_api.hideBlocks.append(block)
- try:
- mixmate.block_mixer(kv.get('blocksToUpload'), block)
- except ValueError:
- pass
- return "removed"
-
- def restart_tor():
- restarttor.restart(shared_state)
- kv.put('offlinePeers', [])
- kv.put('onlinePeers', [])
-
- def test_runtime():
- Thread(target=comm_inst.shared_state.get_by_string(
- "OnionrRunTestManager").run_tests).start()
-
- events_api.register_listener(remove_from_insert_queue_wrapper)
- events_api.register_listener(restart_tor)
- events_api.register_listener(print_test)
- events_api.register_listener(upload_event)
- events_api.register_listener(test_runtime)
diff --git a/src/communicator/daemoneventhooks/removefrominsertqueue.py b/src/communicator/daemoneventhooks/removefrominsertqueue.py
deleted file mode 100644
index d135ccd9..00000000
--- a/src/communicator/daemoneventhooks/removefrominsertqueue.py
+++ /dev/null
@@ -1,33 +0,0 @@
-"""Onionr - P2P Anonymous Storage Network.
-
-Remove block hash from daemon's upload list.
-"""
-from typing import TYPE_CHECKING
-if TYPE_CHECKING:
- from deadsimplekv import DeadSimpleKV
- from communicator import OnionrCommunicatorDaemon
- from onionrtypes import BlockHash
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def remove_from_insert_queue(comm_inst: "OnionrCommunicatorDaemon",
- b_hash: "BlockHash"):
- """Remove block hash from daemon's upload list."""
- kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV")
- try:
- kv.get('generating_blocks').remove(b_hash)
- except ValueError:
- pass
diff --git a/src/communicator/onlinepeers/README.md b/src/communicator/onlinepeers/README.md
deleted file mode 100644
index fe56995f..00000000
--- a/src/communicator/onlinepeers/README.md
+++ /dev/null
@@ -1,12 +0,0 @@
-# Online Peers
-
-Manages a pool of peers to perform actions with. Since Onionr does not maintain socket connections, it holds a list of peers.
-
-
-## Files
-
-* \_\_init\_\_.py: exposes some functions to interact with the pool
-* clearofflinepeer.py: Pop the oldest peer in the offline list
-* onlinepeers.py: communicator timer to add new peers to the pool randomly
-* pickonlinepeers.py: returns a random peer from the online pool
-* removeonlinepeer.py: removes a specified peer from the online pool
\ No newline at end of file
diff --git a/src/communicator/onlinepeers/__init__.py b/src/communicator/onlinepeers/__init__.py
deleted file mode 100644
index a3b107ee..00000000
--- a/src/communicator/onlinepeers/__init__.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from . import clearofflinepeer, onlinepeers, pickonlinepeers, removeonlinepeer
-
-clear_offline_peer = clearofflinepeer.clear_offline_peer
-get_online_peers = onlinepeers.get_online_peers
-pick_online_peer = pickonlinepeers.pick_online_peer
-remove_online_peer = removeonlinepeer.remove_online_peer
\ No newline at end of file
diff --git a/src/communicator/onlinepeers/clearofflinepeer.py b/src/communicator/onlinepeers/clearofflinepeer.py
deleted file mode 100644
index 47128a73..00000000
--- a/src/communicator/onlinepeers/clearofflinepeer.py
+++ /dev/null
@@ -1,35 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-clear offline peer in a communicator instance
-"""
-from typing import TYPE_CHECKING
-
-import logger
-if TYPE_CHECKING:
- from communicator import OnionrCommunicatorDaemon
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def clear_offline_peer(kv: 'DeadSimpleKV'):
- """Remove the longest offline peer to retry later."""
- try:
- removed = kv.get('offlinePeers').pop(0)
- except IndexError:
- pass
- else:
- logger.debug('Removed ' + removed +
- ' from offline list, will try them again.')
-
diff --git a/src/communicator/onlinepeers/onlinepeers.py b/src/communicator/onlinepeers/onlinepeers.py
deleted file mode 100644
index e9966308..00000000
--- a/src/communicator/onlinepeers/onlinepeers.py
+++ /dev/null
@@ -1,63 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-get online peers in a communicator instance
-"""
-import time
-from typing import TYPE_CHECKING
-
-import config
-from etc.humanreadabletime import human_readable_time
-from communicatorutils.connectnewpeers import connect_new_peer_to_communicator
-import logger
-if TYPE_CHECKING:
- from deadsimplekv import DeadSimpleKV
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def get_online_peers(shared_state):
- """Manage the kv.get('onlinePeers') attribute list.
-
- Connect to more peers if we have none connected
- """
- kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
- if config.get('general.offline_mode', False):
- return
- logger.info('Refreshing peer pool...')
- max_peers = int(config.get('peers.max_connect', 10))
- needed = max_peers - len(kv.get('onlinePeers'))
-
- last_seen = 'never'
- if not isinstance(kv.get('lastNodeSeen'), type(None)):
- last_seen = human_readable_time(kv.get('lastNodeSeen'))
-
- for _ in range(needed):
- if len(kv.get('onlinePeers')) == 0:
- connect_new_peer_to_communicator(shared_state, useBootstrap=True)
- else:
- connect_new_peer_to_communicator(shared_state)
-
- if kv.get('shutdown'):
- break
- else:
- if len(kv.get('onlinePeers')) == 0:
- logger.debug('Couldn\'t connect to any peers.' +
- f' Last node seen {last_seen} ago.')
- try:
- get_online_peers(shared_state)
- except RecursionError:
- pass
- else:
- kv.put('lastNodeSeen', time.time())
diff --git a/src/communicator/onlinepeers/pickonlinepeers.py b/src/communicator/onlinepeers/pickonlinepeers.py
deleted file mode 100644
index dc11bceb..00000000
--- a/src/communicator/onlinepeers/pickonlinepeers.py
+++ /dev/null
@@ -1,47 +0,0 @@
-"""
-Onionr - Private P2P Communication.
-
-pick online peers in a communicator instance
-"""
-import secrets
-from typing import TYPE_CHECKING
-
-import onionrexceptions
-
-if TYPE_CHECKING:
- from deadsimplekv import DeadSimpleKV
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def pick_online_peer(kv: 'DeadSimpleKV'):
- """Randomly picks peer from pool without bias (using secrets module)."""
- ret_data = ''
- peer_length = len(kv.get('onlinePeers'))
- if peer_length <= 0:
- raise onionrexceptions.OnlinePeerNeeded
-
- while True:
- peer_length = len(kv.get('onlinePeers'))
-
- try:
- # Get a random online peer, securely.
- # May get stuck in loop if network is lost
- ret_data = kv.get('onlinePeers')[secrets.randbelow(peer_length)]
- except IndexError:
- pass
- else:
- break
- return ret_data
diff --git a/src/communicator/onlinepeers/removeonlinepeer.py b/src/communicator/onlinepeers/removeonlinepeer.py
deleted file mode 100644
index 0eb0b338..00000000
--- a/src/communicator/onlinepeers/removeonlinepeer.py
+++ /dev/null
@@ -1,38 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-remove an online peer from the pool in a communicator instance
-"""
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
- from deadsimplekv import DeadSimpleKV
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def remove_online_peer(kv, peer):
- """Remove an online peer."""
- try:
- del kv.get('connectTimes')[peer]
- except KeyError:
- pass
- try:
- del kv.get('dbTimestamps')[peer]
- except KeyError:
- pass
- try:
- kv.get('onlinePeers').remove(peer)
- except ValueError:
- pass
diff --git a/src/communicator/peeraction.py b/src/communicator/peeraction.py
deleted file mode 100644
index f09814f6..00000000
--- a/src/communicator/peeraction.py
+++ /dev/null
@@ -1,78 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-This file implements logic for performing requests to Onionr peers
-"""
-from typing import TYPE_CHECKING
-
-import streamedrequests
-import logger
-from onionrutils import epoch, basicrequests
-from coredb import keydb
-from . import onlinepeers
-from onionrtypes import OnionAddressString
-from onionrpeers.peerprofiles import PeerProfiles
-from etc.waitforsetvar import wait_for_set_var
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def get_peer_profile(kv, address: OnionAddressString) -> 'PeerProfiles':
- profile_inst_list = kv.get('peerProfiles')
- for profile in profile_inst_list:
- if profile.address == address:
- return profile
- p = PeerProfiles(address)
- profile_inst_list.append(p)
- return p
-
-
-def peer_action(shared_state, peer, action,
- returnHeaders=False, max_resp_size=5242880):
- """Perform a get request to a peer."""
- penalty_score = -10
- kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
- if len(peer) == 0:
- return False
- url = 'http://%s/%s' % (peer, action)
-
- try:
- ret_data = basicrequests.do_get_request(url, port=kv.get('proxyPort'),
- max_size=max_resp_size)
- except streamedrequests.exceptions.ResponseLimitReached:
- logger.warn(
- 'Request failed due to max response size being overflowed',
- terminal=True)
- ret_data = False
- penalty_score = -100
- # if request failed, (error), mark peer offline
- if ret_data is False:
- try:
- get_peer_profile(kv, peer).addScore(penalty_score)
- onlinepeers.remove_online_peer(kv, peer)
- keydb.transportinfo.set_address_info(
- peer, 'lastConnectAttempt', epoch.get_epoch())
- if action != 'ping' and not kv.get('shutdown'):
- logger.warn(f'Lost connection to {peer}', terminal=True)
- # Will only add a new peer to pool if needed
- onlinepeers.get_online_peers(kv)
- except ValueError:
- pass
- else:
- peer_profile = get_peer_profile(kv, peer)
- peer_profile.update_connect_time()
- peer_profile.addScore(1)
- # If returnHeaders, returns tuple of data, headers.
- # If not, just data string
- return ret_data
diff --git a/src/communicator/uploadqueue/__init__.py b/src/communicator/uploadqueue/__init__.py
deleted file mode 100644
index 6b1331cc..00000000
--- a/src/communicator/uploadqueue/__init__.py
+++ /dev/null
@@ -1,73 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Class to remember blocks that need to be uploaded
-and not shared on startup/shutdown
-"""
-import atexit
-import os
-from typing import TYPE_CHECKING
-
-import deadsimplekv
-
-import filepaths
-from onionrutils import localcommand
-if TYPE_CHECKING:
- from communicator import OnionrCommunicatorDaemon
- from deadsimplekv import DeadSimpleKV
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-UPLOAD_MEMORY_FILE = filepaths.upload_list
-
-
-def _add_to_hidden_blocks(cache):
- for bl in cache:
- localcommand.local_command('waitforshare/' + bl, post=True)
-
-
-class UploadQueue:
- """Saves and loads block upload info from json file."""
-
- def __init__(self, communicator: 'OnionrCommunicatorDaemon'):
- """Start the UploadQueue object, loading left over uploads into queue.
-
- register save shutdown function
- """
- self.communicator = communicator
- cache: deadsimplekv.DeadSimpleKV = deadsimplekv.DeadSimpleKV(
- UPLOAD_MEMORY_FILE)
- self.kv: "DeadSimpleKV" = \
- communicator.shared_state.get_by_string("DeadSimpleKV")
- self.store_obj = cache
- cache = cache.get('uploads')
- if cache is None:
- cache = []
-
- _add_to_hidden_blocks(cache)
- self.kv.get('blocksToUpload').extend(cache)
-
- atexit.register(self.save)
-
- def save(self):
- """Save to disk on shutdown or if called manually."""
- bl: deadsimplekv.DeadSimpleKV = self.kv.get('blocksToUpload')
- if len(bl) == 0:
- try:
- os.remove(UPLOAD_MEMORY_FILE)
- except FileNotFoundError:
- pass
- else:
- self.store_obj.put('uploads', bl)
- self.store_obj.flush()
diff --git a/src/communicatorutils/README.md b/src/communicatorutils/README.md
deleted file mode 100755
index 393328c3..00000000
--- a/src/communicatorutils/README.md
+++ /dev/null
@@ -1,33 +0,0 @@
-# communicatorutils
-
-The files in this submodule handle various subtasks and utilities for the onionr communicator.
-
-## Files:
-
-announcenode.py: Uses a communicator instance to announce our transport address to connected nodes
-
-connectnewpeers.py: takes a communicator instance and has it connect to as many peers as needed, and/or to a new specified peer.
-
-cooldownpeer.py: randomly selects a connected peer in a communicator and disconnects them for the purpose of security and network balancing.
-
-daemonqueuehandler.py: checks for new commands in the daemon queue and processes them accordingly.
-
-deniableinserts.py: insert fake blocks with the communicator for plausible deniability
-
-downloadblocks.py: iterates a communicator instance's block download queue and attempts to download the blocks from online peers
-
-housekeeping.py: cleans old blocks and forward secrecy keys
-
-lookupadders.py: ask connected peers to share their list of peer transport addresses
-
-lookupblocks.py: lookup new blocks from connected peers from the communicator
-
-netcheck.py: check if the node is online based on communicator status and onion server ping results
-
-onionrcommunicataortimers.py: create a timer for a function to be launched on an interval. Control how many possible instances of a timer may be running a function at once and control if the timer should be ran in a thread or not.
-
-proxypicker.py: returns a string name for the appropriate proxy to be used with a particular peer transport address.
-
-servicecreator.py: iterate connection blocks and create new direct connection servers for them.
-
-uploadblocks.py: iterate a communicator's upload queue and upload the blocks to connected peers
\ No newline at end of file
diff --git a/src/communicatorutils/__init__.py b/src/communicatorutils/__init__.py
deleted file mode 100755
index e69de29b..00000000
diff --git a/src/communicatorutils/announcenode.py b/src/communicatorutils/announcenode.py
deleted file mode 100755
index 91caa987..00000000
--- a/src/communicatorutils/announcenode.py
+++ /dev/null
@@ -1,77 +0,0 @@
-"""
-Onionr - Private P2P Communication.
-
-Use a communicator instance to announce
-our transport address to connected nodes
-"""
-from typing import TYPE_CHECKING
-
-import logger
-from onionrutils import basicrequests
-from utils import gettransports
-from netcontroller import NetController
-from communicator import onlinepeers
-from coredb import keydb
-import onionrexceptions
-
-if TYPE_CHECKING:
- from deadsimplekv import DeadSimpleKV
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def announce_node(shared_state):
- """Announce our node to our peers."""
- ret_data = False
- kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
- config = shared_state.get_by_string("OnionrCommunicatorDaemon").config
- # Do not let announceCache get too large
- if len(kv.get('announceCache')) >= 10000:
- kv.get('announceCache').popitem()
-
- if config.get('general.security_level', 0) == 0:
- # Announce to random online peers
- for i in kv.get('onlinePeers'):
- if i not in kv.get('announceCache'):
- peer = i
- break
- else:
- try:
- peer = onlinepeers.pick_online_peer(kv)
- except onionrexceptions.OnlinePeerNeeded:
- peer = ""
-
- try:
- ourID = gettransports.get()[0]
- if not peer:
- raise onionrexceptions.OnlinePeerNeeded
- except (IndexError, onionrexceptions.OnlinePeerNeeded):
- pass
- else:
- url = 'http://' + peer + '/announce'
- data = {'node': ourID}
-
- logger.info('Announcing node to ' + url)
- if basicrequests.do_post_request(
- url,
- data,
- port=shared_state.get(NetController).socksPort)\
- == 'Success':
- logger.info('Successfully introduced node to ' + peer,
- terminal=True)
- ret_data = True
- keydb.transportinfo.set_address_info(peer, 'introduced', 1)
-
- return ret_data
diff --git a/src/communicatorutils/connectnewpeers.py b/src/communicatorutils/connectnewpeers.py
deleted file mode 100755
index 3278c569..00000000
--- a/src/communicatorutils/connectnewpeers.py
+++ /dev/null
@@ -1,117 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Connect a new peer to our communicator instance.
-Does so randomly if no peer is specified
-"""
-import time
-import secrets
-
-import onionrexceptions
-import logger
-import onionrpeers
-from utils import networkmerger, gettransports
-from onionrutils import stringvalidators, epoch
-from communicator import peeraction, bootstrappeers
-from coredb import keydb
-import config
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def connect_new_peer_to_communicator(shared_state, peer='', useBootstrap=False):
- retData = False
- kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
- tried = kv.get('offlinePeers')
- transports = gettransports.get()
- if peer != '':
- if stringvalidators.validate_transport(peer):
- peerList = [peer]
- else:
- raise onionrexceptions.InvalidAddress(
- 'Will not attempt connection test to invalid address')
- else:
- peerList = keydb.listkeys.list_adders()
-
- mainPeerList = keydb.listkeys.list_adders()
- if not peerList:
- peerList = onionrpeers.get_score_sorted_peer_list()
-
- """
- If we don't have enough peers connected or random chance,
- select new peers to try
- """
- if len(peerList) < 8 or secrets.randbelow(4) == 3:
- tryingNew = []
- for x in kv.get('newPeers'):
- if x not in peerList:
- peerList.append(x)
- tryingNew.append(x)
- for i in tryingNew:
- kv.get('newPeers').remove(i)
-
- if len(peerList) == 0 or useBootstrap:
- # Avoid duplicating bootstrap addresses in peerList
- if config.get('general.use_bootstrap_list', True):
- bootstrappeers.add_bootstrap_list_to_peer_list(kv, peerList)
-
- for address in peerList:
- address = address.strip()
-
- # Don't connect to our own address
- if address in transports:
- continue
- """Don't connect to invalid address or
- if its already been tried/connected, or if its cooled down
- """
- if len(address) == 0 or address in tried \
- or address in kv.get('onlinePeers') \
- or address in kv.get('cooldownPeer'):
- continue
- if kv.get('shutdown'):
- return
- # Ping a peer,
- ret = peeraction.peer_action(shared_state, address, 'ping')
- if ret == 'pong!':
- time.sleep(0.1)
- if address not in mainPeerList:
- # Add a peer to our list if it isn't already since it connected
- networkmerger.mergeAdders(address)
- if address not in kv.get('onlinePeers'):
- logger.info('Connected to ' + address, terminal=True)
- kv.get('onlinePeers').append(address)
- kv.get('connectTimes')[address] = epoch.get_epoch()
- retData = address
-
- # add peer to profile list if they're not in it
- for profile in kv.get('peerProfiles'):
- if profile.address == address:
- break
- else:
- kv.get('peerProfiles').append(
- onionrpeers.PeerProfiles(address))
- try:
- del kv.get('plaintextDisabledPeers')[address]
- except KeyError:
- pass
- if peeraction.peer_action(
- shared_state, address, 'plaintext') == 'false':
- kv.get('plaintextDisabledPeers')[address] = True
- break
-
- else:
- # Mark a peer as tried if they failed to respond to ping
- tried.append(address)
- logger.debug('Failed to connect to %s: %s ' % (address, ret))
- return retData
diff --git a/src/communicatorutils/cooldownpeer.py b/src/communicatorutils/cooldownpeer.py
deleted file mode 100755
index 0e57db8b..00000000
--- a/src/communicatorutils/cooldownpeer.py
+++ /dev/null
@@ -1,60 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Select random online peer in a communicator instance and have them "cool down"
-"""
-from typing import TYPE_CHECKING
-
-from onionrutils import epoch
-from communicator import onlinepeers
-
-if TYPE_CHECKING:
- from deadsimplekv import DeadSimpleKV
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def cooldown_peer(shared_state):
- """Randomly add an online peer to cooldown, so we can connect a new one."""
- kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
- config = shared_state.get_by_string("OnionrCommunicatorDaemon").config
- online_peer_amount = len(kv.get('onlinePeers'))
- minTime = 300
- cooldown_time = 600
- to_cool = ''
- tempConnectTimes = dict(kv.get('connectTimes'))
-
- # Remove peers from cooldown that have been there long enough
- tempCooldown = dict(kv.get('cooldownPeer'))
- for peer in tempCooldown:
- if (epoch.get_epoch() - tempCooldown[peer]) >= cooldown_time:
- del kv.get('cooldownPeer')[peer]
-
- # Cool down a peer, if we have max connections alive for long enough
- if online_peer_amount >= config.get('peers.max_connect', 10, save=True):
- finding = True
-
- while finding:
- try:
- to_cool = min(tempConnectTimes, key=tempConnectTimes.get)
- if (epoch.get_epoch() - tempConnectTimes[to_cool]) < minTime:
- del tempConnectTimes[to_cool]
- else:
- finding = False
- except ValueError:
- break
- else:
- onlinepeers.remove_online_peer(kv, to_cool)
- kv.get('cooldownPeer')[to_cool] = epoch.get_epoch()
-
diff --git a/src/communicatorutils/deniableinserts.py b/src/communicatorutils/deniableinserts.py
deleted file mode 100755
index 26c34163..00000000
--- a/src/communicatorutils/deniableinserts.py
+++ /dev/null
@@ -1,35 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Use the communicator to insert fake mail messages
-"""
-import secrets
-
-from etc import onionrvalues
-import onionrblocks
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def insert_deniable_block():
- """Insert a fake block to make it more difficult to track real blocks."""
- fakePeer = ''
- chance = 10
- if secrets.randbelow(chance) == (chance - 1):
- # This assumes on the libsodium primitives to have key-privacy
- fakePeer = onionrvalues.DENIABLE_PEER_ADDRESS
- data = secrets.token_hex(secrets.randbelow(5120) + 1)
- onionrblocks.insert(data, header='pm', encryptType='asym',
- asymPeer=fakePeer, disableForward=True,
- meta={'subject': 'foo'})
diff --git a/src/communicatorutils/downloadblocks/__init__.py b/src/communicatorutils/downloadblocks/__init__.py
deleted file mode 100755
index 26b38e52..00000000
--- a/src/communicatorutils/downloadblocks/__init__.py
+++ /dev/null
@@ -1,142 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Download blocks using the communicator instance.
-"""
-from typing import TYPE_CHECKING
-from secrets import SystemRandom
-
-if TYPE_CHECKING:
- from communicator import OnionrCommunicatorDaemon
- from deadsimplekv import DeadSimpleKV
-
-from gevent import spawn
-
-import onionrexceptions
-import logger
-import onionrpeers
-
-from communicator import peeraction
-from communicator import onlinepeers
-from onionrblocks import blockmetadata
-from onionrutils import validatemetadata
-from coredb import blockmetadb
-from onionrutils.localcommand import local_command
-import onionrcrypto
-import onionrstorage
-from onionrblocks import onionrblacklist
-from onionrblocks import storagecounter
-from onionrproofs import vdf
-from . import shoulddownload
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-storage_counter = storagecounter.StorageCounter()
-
-
-def download_blocks_from_communicator(shared_state: "TooMany"):
- """Use communicator instance to download blocks in the comms's queue"""
- blacklist = onionrblacklist.OnionrBlackList()
- kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
- LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
- count: int = 0
- metadata_validation_result: bool = False
- # Iterate the block queue in the communicator
- for blockHash in list(kv.get('blockQueue')):
- count += 1
-
- try:
- blockPeers = list(kv.get('blockQueue')[blockHash])
- except KeyError:
- blockPeers = []
- removeFromQueue = True
-
- if not shoulddownload.should_download(shared_state, blockHash):
- continue
-
- if kv.get('shutdown') or not kv.get('isOnline') or \
- storage_counter.is_full():
- # Exit loop if shutting down or offline, or disk allocation reached
- break
- # Do not download blocks being downloaded
- if blockHash in kv.get('currentDownloading'):
- continue
-
- if len(kv.get('onlinePeers')) == 0:
- break
-
- # So we can avoid concurrent downloading in other threads of same block
- kv.get('currentDownloading').append(blockHash)
- if len(blockPeers) == 0:
- try:
- peerUsed = onlinepeers.pick_online_peer(kv)
- except onionrexceptions.OnlinePeerNeeded:
- continue
- else:
- SystemRandom().shuffle(blockPeers)
- peerUsed = blockPeers.pop(0)
-
- if not kv.get('shutdown') and peerUsed.strip() != '':
- logger.info(
- f"Attempting to download %s from {peerUsed}..." % (blockHash[:12],))
- content = peeraction.peer_action(
- shared_state, peerUsed,
- 'getdata/' + blockHash,
- max_resp_size=3000000) # block content from random peer
-
- if content is not False and len(content) > 0:
- try:
- content = content.encode()
- except AttributeError:
- pass
-
- if vdf.verify_vdf(blockHash, content):
- #content = content.decode() # decode here because sha3Hash needs bytes above
- metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
- metadata = metas[0]
- try:
- metadata_validation_result = \
- validatemetadata.validate_metadata(metadata, metas[2])
- except onionrexceptions.PlaintextNotSupported:
- logger.debug(f"Not saving {blockHash} due to plaintext not enabled")
- removeFromQueue = True
- except onionrexceptions.DataExists:
- metadata_validation_result = False
- if metadata_validation_result: # check if metadata is valid, and verify nonce
- save_block(blockHash, data)
- else:
- if blacklist.inBlacklist(blockHash):
- logger.warn(f'Block {blockHash} is blacklisted.')
- else:
- logger.warn('Metadata for block %s is invalid.' % (blockHash,))
- blacklist.addToDB(blockHash)
- else:
- # if block didn't meet expected hash
- tempHash = onionrcrypto.hashers.sha3_hash(content) # lazy hack, TODO use var
- try:
- tempHash = tempHash.decode()
- except AttributeError:
- pass
- # Punish peer for sharing invalid block (not always malicious, but is bad regardless)
- onionrpeers.PeerProfiles(peerUsed).addScore(-50)
- if removeFromQueue:
- try:
- del kv.get('blockQueue')[blockHash] # remove from block queue both if success or false
- if count == LOG_SKIP_COUNT:
- logger.info('%s blocks remaining in queue' %
- [len(kv.get('blockQueue'))], terminal=True)
- count = 0
- except KeyError:
- pass
- kv.get('currentDownloading').remove(blockHash)
diff --git a/src/communicatorutils/downloadblocks/shoulddownload.py b/src/communicatorutils/downloadblocks/shoulddownload.py
deleted file mode 100644
index ddf11624..00000000
--- a/src/communicatorutils/downloadblocks/shoulddownload.py
+++ /dev/null
@@ -1,42 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Check if a block should be downloaded
-(if we already have it or its blacklisted or not)
-"""
-from coredb import blockmetadb
-from onionrblocks import onionrblacklist
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def should_download(shared_state, block_hash) -> bool:
- """Return bool for if a (assumed to exist) block should be downloaded."""
- blacklist = onionrblacklist.OnionrBlackList()
- should = True
- kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
- if block_hash in blockmetadb.get_block_list():
- # Don't download block we have
- should = False
- else:
- if blacklist.inBlacklist(block_hash):
- # Don't download blacklisted block
- should = False
- if should is False:
- # Remove block from communicator queue if it shouldn't be downloaded
- try:
- del kv.get('blockQueue')[block_hash]
- except KeyError:
- pass
- return should
diff --git a/src/communicatorutils/housekeeping.py b/src/communicatorutils/housekeeping.py
deleted file mode 100755
index a36f8556..00000000
--- a/src/communicatorutils/housekeeping.py
+++ /dev/null
@@ -1,108 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Cleanup old Onionr blocks and forward secrecy keys using the communicator.
-Ran from a communicator timer usually
-"""
-import sqlite3
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
- from deadsimplekv import DeadSimpleKV
-
-import logger
-from onionrusers import onionrusers
-from onionrutils import epoch
-from coredb import blockmetadb, dbfiles
-import onionrstorage
-from onionrstorage import removeblock
-from onionrblocks import onionrblacklist
-from onionrblocks.storagecounter import StorageCounter
-from etc.onionrvalues import DATABASE_LOCK_TIMEOUT
-from onionrproofs import hashMeetsDifficulty
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-storage_counter = StorageCounter()
-
-
-def __remove_from_upload(shared_state, block_hash: str):
- kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
- try:
- kv.get('blocksToUpload').remove(block_hash)
- except ValueError:
- pass
-
-
-def __purge_block(shared_state, block_hash, add_to_blacklist = True):
- blacklist = None
-
- removeblock.remove_block(block_hash)
- onionrstorage.deleteBlock(block_hash)
- __remove_from_upload(shared_state, block_hash)
-
- if add_to_blacklist:
- blacklist = onionrblacklist.OnionrBlackList()
- blacklist.addToDB(block_hash)
-
-
-def clean_old_blocks(shared_state):
- """Delete expired blocks + old blocks if disk allocation is near full"""
- blacklist = onionrblacklist.OnionrBlackList()
- # Delete expired blocks
- for bHash in blockmetadb.expiredblocks.get_expired_blocks():
- __purge_block(shared_state, bHash, add_to_blacklist=True)
- logger.info('Deleted expired block: %s' % (bHash,))
-
- while storage_counter.is_full():
- try:
- oldest = blockmetadb.get_block_list()[0]
- except IndexError:
- break
- else:
- __purge_block(shared_state, bHash, add_to_blacklist=True)
- logger.info('Deleted block because of full storage: %s' % (oldest,))
-
-
-def clean_keys():
- """Delete expired forward secrecy keys"""
- conn = sqlite3.connect(dbfiles.user_id_info_db,
- timeout=DATABASE_LOCK_TIMEOUT)
- c = conn.cursor()
- time = epoch.get_epoch()
- deleteKeys = []
-
- for entry in c.execute(
- "SELECT * FROM forwardKeys WHERE expire <= ?", (time,)):
- logger.debug('Forward key: %s' % entry[1])
- deleteKeys.append(entry[1])
-
- for key in deleteKeys:
- logger.debug('Deleting forward key %s' % key)
- c.execute("DELETE from forwardKeys where forwardKey = ?", (key,))
- conn.commit()
- conn.close()
-
- onionrusers.deleteExpiredKeys()
-
-
-def clean_blocks_not_meeting_pow(shared_state):
- """Clean blocks not meeting min send/rec pow. Used if config.json POW changes"""
- block_list = blockmetadb.get_block_list()
- for block in block_list:
- if not hashMeetsDifficulty(block):
- logger.warn(
- f"Deleting block {block} because it was stored" +
- "with a POW level smaller than current.", terminal=True)
- __purge_block(shared_state, block)
diff --git a/src/communicatorutils/lookupadders.py b/src/communicatorutils/lookupadders.py
deleted file mode 100755
index 788ffa12..00000000
--- a/src/communicatorutils/lookupadders.py
+++ /dev/null
@@ -1,66 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Lookup new peer transport addresses using the communicator
-"""
-from typing import TYPE_CHECKING
-import logger
-from onionrutils import stringvalidators
-from communicator import peeraction, onlinepeers
-from utils import gettransports
-import onionrexceptions
-
-
-if TYPE_CHECKING:
- from deadsimplekv import DeadSimpleKV
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def lookup_new_peer_transports_with_communicator(shared_state):
- logger.info('Looking up new addresses...')
- tryAmount = 1
- newPeers = []
- transports = gettransports.get()
- kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
-
- for i in range(tryAmount):
- # Download new peer address list from random online peers
- if len(newPeers) > 10000:
- # Don't get new peers if we have too many queued up
- break
- try:
- peer = onlinepeers.pick_online_peer(kv)
- newAdders = peeraction.peer_action(shared_state, peer, action='pex')
- except onionrexceptions.OnlinePeerNeeded:
- continue
- try:
- newPeers = newAdders.split(',')
- except AttributeError:
- pass
- else:
- # Validate new peers are good format and not already in queue
- invalid = []
- for x in newPeers:
- x = x.strip()
- if not stringvalidators.validate_transport(x) \
- or x in kv.get('newPeers') or x in transports:
- # avoid adding if its our address
- invalid.append(x)
- for x in invalid:
- try:
- newPeers.remove(x)
- except ValueError:
- pass
- kv.get('newPeers').extend(newPeers)
\ No newline at end of file
diff --git a/src/communicatorutils/lookupblocks.py b/src/communicatorutils/lookupblocks.py
deleted file mode 100755
index 99574b89..00000000
--- a/src/communicatorutils/lookupblocks.py
+++ /dev/null
@@ -1,126 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Lookup new blocks with the communicator using a random connected peer
-"""
-from typing import TYPE_CHECKING
-
-from gevent import time
-
-if TYPE_CHECKING:
- from deadsimplekv import DeadSimpleKV
-
-import logger
-import onionrproofs
-from onionrutils import stringvalidators, epoch
-from communicator import peeraction, onlinepeers
-from coredb.blockmetadb import get_block_list
-from utils import reconstructhash
-from onionrblocks import onionrblacklist
-import onionrexceptions
-import config
-from etc import onionrvalues
-from onionrblocks.storagecounter import StorageCounter
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-blacklist = onionrblacklist.OnionrBlackList()
-storage_counter = StorageCounter()
-
-
-def lookup_blocks_from_communicator(shared_state: 'TooMany'):
- logger.info('Looking up new blocks')
- tryAmount = 2
- newBlocks = ''
- # List of existing saved blocks
- existingBlocks = get_block_list()
- triedPeers = [] # list of peers we've tried this time around
- # Max amount of *new* block hashes to have in queue
- maxBacklog = 1560
- lastLookupTime = 0 # Last time we looked up a particular peer's list
- new_block_count = 0
- kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
- for i in range(tryAmount):
- # Defined here to reset it each time, time offset is added later
- listLookupCommand = 'getblocklist'
- if len(kv.get('blockQueue')) >= maxBacklog:
- break
- if not kv.get('isOnline'):
- break
- # check if disk allocation is used
- if storage_counter.is_full():
- logger.debug(
- 'Not looking up new blocks due to maximum amount of disk used')
- break
- try:
- # select random online peer
- peer = onlinepeers.pick_online_peer(kv)
- except onionrexceptions.OnlinePeerNeeded:
- time.sleep(1)
- continue
- # if we've already tried all the online peers this time around, stop
- if peer in triedPeers:
- if len(kv.get('onlinePeers')) == len(triedPeers):
- break
- else:
- continue
- triedPeers.append(peer)
-
- # Get the last time we looked up a peer's stamp,
- # to only fetch blocks since then.
- # Saved in memory only for privacy reasons
- try:
- lastLookupTime = kv.get('dbTimestamps')[peer]
- except KeyError:
- lastLookupTime = epoch.get_epoch() - onionrvalues.DEFAULT_EXPIRE
- listLookupCommand += '?date=%s' % (lastLookupTime,)
- try:
- newBlocks = peeraction.peer_action(
- shared_state,
- peer, listLookupCommand) # get list of new block hashes
- except Exception as error:
- logger.warn(
- f'Could not get new blocks from {peer}.',
- error=error)
- newBlocks = False
-
- if newBlocks != False: # noqa
- # if request was a success
- for i in newBlocks.split('\n'):
- if stringvalidators.validate_hash(i):
- i = reconstructhash.reconstruct_hash(i)
- # if newline seperated string is valid hash
-
- # if block does not exist on disk + is not already in queue
- if i not in existingBlocks:
- if i not in kv.get('blockQueue'):
- if onionrproofs.hashMeetsDifficulty(i) and \
- not blacklist.inBlacklist(i):
- if len(kv.get('blockQueue')) <= 1000000:
- # add blocks to download queue
- kv.get('blockQueue')[i] = [peer]
- new_block_count += 1
- kv.get('dbTimestamps')[peer] = \
- epoch.get_rounded_epoch(roundS=60)
- else:
- if peer not in kv.get('blockQueue')[i]:
- if len(kv.get('blockQueue')[i]) < 10:
- kv.get('blockQueue')[i].append(peer)
- if new_block_count > 0:
- block_string = ""
- if new_block_count > 1:
- block_string = "s"
- logger.info(
- f'Discovered {new_block_count} new block{block_string}',
- terminal=True)
diff --git a/src/communicatorutils/netcheck.py b/src/communicatorutils/netcheck.py
deleted file mode 100755
index b45a4361..00000000
--- a/src/communicatorutils/netcheck.py
+++ /dev/null
@@ -1,64 +0,0 @@
-"""
-Onionr - Private P2P Communication.
-
-Determine if our node is able to use Tor based
-on the status of a communicator instance
-and the result of pinging onion http servers
-"""
-import logger
-from utils import netutils
-from onionrutils import localcommand, epoch
-from . import restarttor
-
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
- from deadsimplekv import DeadSimpleKV
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def net_check(shared_state):
- """Check if we are connected to the internet.
-
- or not when we can't connect to any peers
- """
- # for detecting if we have received incoming connections recently
- rec = False
- kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
- proxy_port = shared_state.get_by_string("NetController").socksPort
-
- if len(kv.get('onlinePeers')) == 0:
- try:
- if (epoch.get_epoch() - int(localcommand.local_command
- ('/lastconnect'))) <= 60:
- kv.put('isOnline', True)
- rec = True
- except ValueError:
- pass
- if not rec and not netutils.check_network(torPort=proxy_port):
- if not kv.get('shutdown'):
- if not shared_state.get_by_string(
- "OnionrCommunicatorDaemon").config.get(
- 'general.offline_mode', False):
- logger.warn('Network check failed, are you connected to ' +
- 'the Internet, and is Tor working? ' +
- 'This is usually temporary, but bugs and censorship can cause this to persist, in which case you should report it to beardog [at] mailbox.org', # noqa
- terminal=True)
- restarttor.restart(shared_state)
- kv.put('offlinePeers', [])
- kv.put('isOnline', False)
- else:
- kv.put('isOnline', True)
diff --git a/src/communicatorutils/proxypicker.py b/src/communicatorutils/proxypicker.py
deleted file mode 100755
index 85dc4d5b..00000000
--- a/src/communicatorutils/proxypicker.py
+++ /dev/null
@@ -1,28 +0,0 @@
-"""
-Onionr - Private P2P Communication.
-
-Pick a proxy to use based on a peer's address
-"""
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def pick_proxy(peer_address):
- if peer_address.endswith('.onion'):
- return 'tor'
- elif peer_address.endswith('.i2p'):
- return 'i2p'
- raise ValueError(
- f"Peer address not ending with acceptable domain: {peer_address}")
diff --git a/src/communicatorutils/restarttor.py b/src/communicatorutils/restarttor.py
deleted file mode 100644
index 35388d05..00000000
--- a/src/communicatorutils/restarttor.py
+++ /dev/null
@@ -1,28 +0,0 @@
-"""
-Onionr - Private P2P Communication.
-
-Restart Onionr managed Tor
-"""
-import netcontroller
-import config
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def restart(shared_state):
- if not config.get('tor.use_existing_tor', False):
- net = shared_state.get(netcontroller.NetController)
- net.killTor()
- net.startTor()
diff --git a/src/communicatorutils/uploadblocks/__init__.py b/src/communicatorutils/uploadblocks/__init__.py
deleted file mode 100755
index 8bf23be6..00000000
--- a/src/communicatorutils/uploadblocks/__init__.py
+++ /dev/null
@@ -1,148 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Upload blocks in the upload queue to peers from the communicator
-"""
-from typing import TYPE_CHECKING
-from time import sleep
-from threading import Thread
-from secrets import SystemRandom
-
-from . import sessionmanager
-
-from onionrtypes import UserID
-import logger
-from communicatorutils import proxypicker
-import onionrexceptions
-from onionrblocks import onionrblockapi as block
-from onionrblocks.blockmetadata.fromdata import get_block_metadata_from_data
-from onionrutils import stringvalidators, basicrequests
-from onionrutils.validatemetadata import validate_metadata
-from communicator import onlinepeers
-if TYPE_CHECKING:
- from deadsimplekv import DeadSimpleKV
- from communicator import OnionrCommunicatorDaemon
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def upload_blocks_from_communicator(shared_state: 'OnionrCommunicatorDaemon'):
- """Accept a communicator instance + upload blocks from its upload queue."""
- """when inserting a block, we try to upload
- it to a few peers to add some deniability & increase functionality"""
- kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
-
- session_manager: sessionmanager.BlockUploadSessionManager
- session_manager = shared_state.get(
- sessionmanager.BlockUploadSessionManager)
- tried_peers: UserID = []
- finishedUploads = []
-
- SystemRandom().shuffle(kv.get('blocksToUpload'))
-
- def remove_from_hidden(bl):
- sleep(60)
- try:
- shared_state.get_by_string(
- 'PublicAPI').hideBlocks.remove(bl)
- except ValueError:
- pass
-
- if len(kv.get('blocksToUpload')) != 0:
- for bl in kv.get('blocksToUpload'):
- if not stringvalidators.validate_hash(bl):
- logger.warn('Requested to upload invalid block', terminal=True)
- return
- session = session_manager.add_session(bl)
- for _ in range(min(len(kv.get('onlinePeers')), 6)):
- try:
- peer = onlinepeers.pick_online_peer(kv)
- if not block.Block(bl).isEncrypted:
- if peer in kv.get('plaintextDisabledPeers'):
- logger.info(f"Cannot upload plaintext block to peer that denies it {peer}") # noqa
- continue
- except onionrexceptions.OnlinePeerNeeded:
- continue
- try:
- session.peer_exists[peer]
- continue
- except KeyError:
- pass
- try:
- if session.peer_fails[peer] > 3:
- continue
- except KeyError:
- pass
- if peer in tried_peers:
- continue
- tried_peers.append(peer)
- url = f'http://{peer}/upload'
- try:
- data = block.Block(bl).getRaw()
- if not data:
- logger.warn(
- f"Couldn't get data for block in upload list {bl}",
- terminal=True)
- raise onionrexceptions.NoDataAvailable
- try:
- def __check_metadata():
- metadata = get_block_metadata_from_data(data)[0]
- if not validate_metadata(metadata, data):
- logger.warn(
- f"Metadata for uploading block not valid {bl}")
- raise onionrexceptions.InvalidMetadata
- __check_metadata()
- except onionrexceptions.DataExists:
- pass
- except( # noqa
- onionrexceptions.NoDataAvailable,
- onionrexceptions.InvalidMetadata) as _:
- finishedUploads.append(bl)
- break
- proxy_type = proxypicker.pick_proxy(peer)
- logger.info(
- f"Uploading block {bl[:8]} to {peer}", terminal=True)
- resp = basicrequests.do_post_request(
- url, data=data, proxyType=proxy_type,
- content_type='application/octet-stream')
- if resp is not False:
- if resp == 'success':
- Thread(target=remove_from_hidden,
- args=[bl], daemon=True).start()
- session.success()
- session.peer_exists[peer] = True
- elif resp == 'exists':
- session.success()
- session.peer_exists[peer] = True
- else:
- session.fail()
- session.fail_peer(peer)
- shared_state.get_by_string(
- 'OnionrCommunicatorDaemon').getPeerProfileInstance(
- peer).addScore(-5)
- logger.warn(
- f'Failed to upload {bl[:8]}, reason: {resp}',
- terminal=True)
- else:
- session.fail()
- session_manager.clean_session()
- for x in finishedUploads:
- try:
- kv.get('blocksToUpload').remove(x)
-
- shared_state.get_by_string(
- 'PublicAPI').hideBlocks.remove(x)
-
- except ValueError:
- pass
\ No newline at end of file
diff --git a/src/communicatorutils/uploadblocks/mixmate/__init__.py b/src/communicatorutils/uploadblocks/mixmate/__init__.py
deleted file mode 100644
index 3e63ea99..00000000
--- a/src/communicatorutils/uploadblocks/mixmate/__init__.py
+++ /dev/null
@@ -1,48 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Perform block mixing
-"""
-import time
-from typing import List
-
-import onionrtypes
-from onionrblocks import onionrblockapi
-
-from .pool import UploadPool
-from .pool import PoolFullException
-
-from etc import onionrvalues
-
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-upload_pool = UploadPool(4)
-
-
-def block_mixer(upload_list: List[onionrtypes.BlockHash],
- block_to_mix: onionrtypes.BlockHash):
- """Delay and mix block inserts.
-
- Take a block list and a received/created block and add it
- to the said block list
- """
- bl = onionrblockapi.Block(block_to_mix)
-
- try:
- if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE:
- raise ValueError
- except TypeError:
- pass
- if block_to_mix:
- upload_list.append(block_to_mix)
diff --git a/src/communicatorutils/uploadblocks/mixmate/pool.py b/src/communicatorutils/uploadblocks/mixmate/pool.py
deleted file mode 100644
index f5600082..00000000
--- a/src/communicatorutils/uploadblocks/mixmate/pool.py
+++ /dev/null
@@ -1,71 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Upload pool
-"""
-from typing import List
-from secrets import SystemRandom
-
-import onionrutils
-import onionrtypes
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-class PoolFullException(Exception):
- """For when the UploadPool is full.
-
- Raise when a new hash is attempted to be added
- """
-
-
-class PoolNotReady(Exception):
- """Raise when UploadPool pool access is attempted without it being full."""
-
-
-class AlreadyInPool(Exception):
- """Raise when a hash already in pool is attempted to be added again."""
-
-
-class UploadPool:
- """Upload pool for mixing blocks together and delaying uploads."""
-
- def __init__(self, pool_size: int):
- """Create a new pool with a specified max size.
-
- Uses private var and getter to avoid direct adding
- """
- self._pool: List[onionrtypes.BlockHash] = []
- self._pool_size = pool_size
- self.birthday = onionrutils.epoch.get_epoch()
-
- def add_to_pool(self, item: List[onionrtypes.BlockHash]):
- """Add a new hash to the pool. Raise PoolFullException if full."""
- if len(self._pool) >= self._pool_size:
- raise PoolFullException
- if not onionrutils.stringvalidators.validate_hash(item):
- raise ValueError
- self._pool.append(item)
-
- def get_pool(self) -> List[onionrtypes.BlockHash]:
- """Get the hash pool in secure random order."""
- if len(self._pool) != self._pool_size:
- raise PoolNotReady
-
- final_pool: List[onionrtypes.BlockHash] = list(self._pool)
- SystemRandom().shuffle(final_pool)
-
- self._pool.clear()
- self.birthday = onionrutils.epoch.get_epoch()
- return final_pool
diff --git a/src/communicatorutils/uploadblocks/session.py b/src/communicatorutils/uploadblocks/session.py
deleted file mode 100644
index d9b715bf..00000000
--- a/src/communicatorutils/uploadblocks/session.py
+++ /dev/null
@@ -1,57 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Virtual upload "sessions" for blocks
-"""
-from typing import Union, Dict
-
-from onionrtypes import UserID
-from onionrutils import stringvalidators
-from onionrutils import bytesconverter
-from onionrutils import epoch
-from utils import reconstructhash
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-class UploadSession:
- """Manage statistics for an Onionr block upload session.
-
- accept a block hash (incl. unpadded) as an argument
- """
-
- def __init__(self, block_hash: Union[str, bytes]):
- block_hash = bytesconverter.bytes_to_str(block_hash)
- block_hash = reconstructhash.reconstruct_hash(block_hash)
- if not stringvalidators.validate_hash(block_hash):
- raise ValueError
-
- self.start_time = epoch.get_epoch()
- self.block_hash = reconstructhash.deconstruct_hash(block_hash)
- self.total_fail_count: int = 0
- self.total_success_count: int = 0
- self.peer_fails: Dict[UserID, int] = {}
- self.peer_exists: Dict[UserID, bool] = {}
-
- def fail_peer(self, peer):
- try:
- self.peer_fails[peer] += 1
- except KeyError:
- self.peer_fails[peer] = 0
-
- def fail(self):
- self.total_fail_count += 1
-
- def success(self):
- self.total_success_count += 1
diff --git a/src/communicatorutils/uploadblocks/sessionmanager.py b/src/communicatorutils/uploadblocks/sessionmanager.py
deleted file mode 100644
index 486823a3..00000000
--- a/src/communicatorutils/uploadblocks/sessionmanager.py
+++ /dev/null
@@ -1,127 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Manager for upload 'sessions'
-"""
-from typing import List, Union, TYPE_CHECKING
-if TYPE_CHECKING:
- from deadsimplekv import DeadSimpleKV
- from session import UploadSession
-
-from onionrutils import bytesconverter
-from etc import onionrvalues
-from utils import reconstructhash
-
-from . import session
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-class BlockUploadSessionManager:
- """Holds block UploadSession instances.
-
- Optionally accepts iterable of sessions to added on init
- Arguments: old_session: iterable of old UploadSession objects
- """
-
- def __init__(self, old_sessions: List = None):
- if old_sessions is None:
- self.sessions = []
- else:
- self.sessions = old_sessions
-
- def add_session(self,
- session_or_block: Union[str,
- bytes,
- session.UploadSession
- ]
- ) -> session.UploadSession:
- """Create (or add existing) block upload session.
-
- from a str/bytes block hex hash, existing UploadSession
- """
- if isinstance(session_or_block, session.UploadSession):
- if session_or_block not in self.sessions:
- self.sessions.append(session_or_block)
- return session_or_block
- try:
- return self.get_session(session_or_block)
- except KeyError:
- pass
- # convert bytes hash to str
- if isinstance(session_or_block, bytes):
- session_or_block = bytesconverter.bytes_to_str(session_or_block)
- # intentionally not elif
- if isinstance(session_or_block, str):
- new_session = session.UploadSession(session_or_block)
- self.sessions.append(new_session)
- return new_session
- raise ValueError
-
- def get_session(self,
- block_hash: Union[str, bytes]
- ) -> session.UploadSession:
- block_hash = reconstructhash.deconstruct_hash(
- bytesconverter.bytes_to_str(block_hash))
- for sess in self.sessions:
- if sess.block_hash == block_hash:
- return sess
- raise KeyError
-
- def clean_session(self,
- specific_session: Union[str, 'UploadSession'] = None):
-
- comm_inst: 'OnionrCommunicatorDaemon' # type: ignore
- comm_inst = self._too_many.get_by_string( # pylint: disable=E1101 type: ignore
- "OnionrCommunicatorDaemon")
- kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string(
- "DeadSimpleKV")
- sessions_to_delete = []
- if kv.get('startTime') < 120:
- return
- onlinePeerCount = len(kv.get('onlinePeers'))
-
- # If we have no online peers right now,
- if onlinePeerCount == 0:
- return
-
- for sess in self.sessions:
- # if over 50% of peers that were online for a session have
- # become unavailable, don't kill sessions
- if sess.total_success_count > onlinePeerCount:
- if onlinePeerCount / sess.total_success_count >= 0.5:
- return
- # Clean sessions if they have uploaded to enough online peers
- if sess.total_success_count <= 0:
- continue
- if (sess.total_success_count / onlinePeerCount) >= \
- onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
- sessions_to_delete.append(sess)
- for sess in sessions_to_delete:
- try:
- self.sessions.remove(session)
- except ValueError:
- pass
- # TODO cleanup to one round of search
- # Remove the blocks from the sessions, upload list,
- # and waitforshare list
- try:
- kv.get('blocksToUpload').remove(
- reconstructhash.reconstruct_hash(sess.block_hash))
- except ValueError:
- pass
- try:
- kv.get('blocksToUpload').remove(sess.block_hash)
- except ValueError:
- pass
diff --git a/src/coredb/keydb/__init__.py b/src/coredb/keydb/__init__.py
index 16d602d5..13b95148 100644
--- a/src/coredb/keydb/__init__.py
+++ b/src/coredb/keydb/__init__.py
@@ -1 +1 @@
-from . import addkeys, listkeys, removekeys, userinfo, transportinfo
\ No newline at end of file
+from . import addkeys, listkeys, removekeys, userinfo
\ No newline at end of file
diff --git a/src/coredb/keydb/addkeys.py b/src/coredb/keydb/addkeys.py
index 9c21c78c..364b0a09 100644
--- a/src/coredb/keydb/addkeys.py
+++ b/src/coredb/keydb/addkeys.py
@@ -25,9 +25,9 @@ from etc import onionrvalues
"""
-def add_peer(peerID, name=''):
+def add_pub_key(peerID, name=''):
"""Add a public key to the key database (misleading function name)."""
- if peerID in listkeys.list_peers() or peerID == onionrcrypto.pub_key:
+ if peerID in listkeys.list_pub_keys() or peerID == onionrcrypto.pub_key:
raise ValueError("specified id is already known")
# This function simply adds a peer to the DB
@@ -54,35 +54,3 @@ def add_peer(peerID, name=''):
return True
-
-def add_address(address):
- """Add an address to the address database (only tor currently)"""
-
- if type(address) is None or len(address) == 0:
- return False
- if stringvalidators.validate_transport(address):
- if address in gettransports.get():
- return False
- conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
- c = conn.cursor()
- # check if address is in database
- # this is safe to do because the address is validated above, but we strip some chars here too just in case
- address = address.replace('\'', '').replace(';', '').replace('"', '').replace('\\', '')
- for i in c.execute("SELECT * FROM adders WHERE address = ?;", (address,)):
- try:
- if i[0] == address:
- conn.close()
- return False
- except ValueError:
- pass
- except IndexError:
- pass
-
- t = (address, 1)
- c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t)
- conn.commit()
- conn.close()
-
- return True
- else:
- return False
diff --git a/src/coredb/keydb/listkeys.py b/src/coredb/keydb/listkeys.py
index ff56b15e..28d58657 100644
--- a/src/coredb/keydb/listkeys.py
+++ b/src/coredb/keydb/listkeys.py
@@ -22,8 +22,9 @@ import logger
from onionrutils import epoch
from etc import onionrvalues
from .. import dbfiles
-from . import userinfo, transportinfo
-def list_peers(randomOrder=True, getPow=False, trust=0):
+from . import userinfo
+
+def list_pub_keys(randomOrder=True, getPow=False, trust=0):
'''
Return a list of public keys (misleading function name)
@@ -59,28 +60,3 @@ def list_peers(randomOrder=True, getPow=False, trust=0):
conn.close()
return peerList
-
-def list_adders(randomOrder=True, i2p=True, recent=0):
- '''
- Return a list of transport addresses
- '''
- conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
- c = conn.cursor()
- if randomOrder:
- addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();')
- else:
- addresses = c.execute('SELECT * FROM adders;')
- addressList = []
- for i in addresses:
- if len(i[0].strip()) == 0:
- continue
- addressList.append(i[0])
- conn.close()
- testList = list(addressList) # create new list to iterate
- for address in testList:
- try:
- if recent > 0 and (epoch.get_epoch() - transportinfo.get_address_info(address, 'lastConnect')) > recent:
- raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0
- except TypeError:
- addressList.remove(address)
- return addressList
\ No newline at end of file
diff --git a/src/coredb/keydb/removekeys.py b/src/coredb/keydb/removekeys.py
index 46d2d641..1dd0fb39 100644
--- a/src/coredb/keydb/removekeys.py
+++ b/src/coredb/keydb/removekeys.py
@@ -24,23 +24,7 @@ from onionrutils import mnemonickeys
from .. import dbfiles
from etc import onionrvalues
-def remove_address(address):
- '''
- Remove an address from the address database
- '''
- if stringvalidators.validate_transport(address):
- conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
- c = conn.cursor()
- t = (address,)
- c.execute('Delete from adders where address=?;', t)
- conn.commit()
- conn.close()
-
- #events.event('address_remove', data = {'address': address}, onionr = core_inst.onionrInst)
- return True
- else:
- return False
def remove_user(pubkey: str)->bool:
'''
diff --git a/src/coredb/keydb/transportinfo.py b/src/coredb/keydb/transportinfo.py
deleted file mode 100644
index 9f03b323..00000000
--- a/src/coredb/keydb/transportinfo.py
+++ /dev/null
@@ -1,85 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-get or set transport address meta information
-"""
-import sqlite3
-from .. import dbfiles
-from etc import onionrvalues
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-info_numbers = {
- 'address': 0,
- 'type': 1,
- 'knownPeer': 2,
- 'speed': 3,
- 'success': 4,
- 'powValue': 5,
- 'failure': 6,
- 'lastConnect': 7,
- 'trust': 8,
- 'introduced': 9}
-
-
-def get_address_info(address, info):
- """Get info about an address from its database entry.
-
- address text, 0
- type int, 1
- knownPeer text, 2
- speed int, 3
- success int, 4
- powValue 5
- failure int 6
- lastConnect 7
- trust 8
- introduced 9
- """
- conn = sqlite3.connect(
- dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
- c = conn.cursor()
-
- command = (address,)
-
- info = info_numbers[info]
- iter_count = 0
- retVal = ''
-
- for row in c.execute('SELECT * FROM adders WHERE address=?;', command):
- for i in row:
- if iter_count == info:
- retVal = i
- break
- else:
- iter_count += 1
- conn.close()
-
- return retVal
-
-
-def set_address_info(address, key, data):
- """Update an address for a key."""
- conn = sqlite3.connect(
- dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
- c = conn.cursor()
-
- command = (data, address)
-
- if key not in info_numbers.keys():
- raise ValueError(
- "Got invalid database key when setting address info, must be in whitelist")
- else:
- c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command)
- conn.commit()
- conn.close()
\ No newline at end of file
diff --git a/src/httpapi/__init__.py b/src/httpapi/__init__.py
index 101a20c6..c6f0ee16 100755
--- a/src/httpapi/__init__.py
+++ b/src/httpapi/__init__.py
@@ -1,24 +1,26 @@
"""
- Onionr - Private P2P Communication
+Onionr - Private P2P Communication
- This file registers plugin's flask blueprints for the client http server
-"""
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
+Register plugins flask blueprints for the client http server
"""
import onionrplugins
import config
+from .fdsafehandler import FDSafeHandler
+"""
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see .
+"""
+
def load_plugin_blueprints(flaskapp, blueprint: str = 'flask_blueprint'):
"""Iterate enabled plugins and load any http endpoints they have"""
diff --git a/src/httpapi/miscclientapi/addpeer.py b/src/httpapi/miscclientapi/addpeer.py
deleted file mode 100644
index c01a8098..00000000
--- a/src/httpapi/miscclientapi/addpeer.py
+++ /dev/null
@@ -1,31 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-add a transport address to the db
-"""
-from onionrutils.stringvalidators import validate_transport
-from coredb.keydb.addkeys import add_address
-from coredb.keydb.listkeys import list_adders
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def add_peer(peer):
-
- if peer in list_adders():
- return "already added"
- if add_address(peer):
- return "success"
- else:
- return "failure, invalid address"
diff --git a/src/httpapi/miscclientapi/endpoints.py b/src/httpapi/miscclientapi/endpoints.py
index 6fbfcade..ed10d969 100644
--- a/src/httpapi/miscclientapi/endpoints.py
+++ b/src/httpapi/miscclientapi/endpoints.py
@@ -15,14 +15,10 @@ import unpaddedbase32
from httpapi import apiutils
import onionrcrypto
import config
-from netcontroller import NetController
-from onionrstatistics.serializeddata import SerializedData
from onionrutils import mnemonickeys
from onionrutils import bytesconverter
from etc import onionrvalues
from utils import reconstructhash
-from utils.gettransports import get as get_tor
-from .addpeer import add_peer
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -49,16 +45,6 @@ class PrivateEndpoints:
private_endpoints_bp = Blueprint('privateendpoints', __name__)
self.private_endpoints_bp = private_endpoints_bp
- @private_endpoints_bp.route('/addpeer/', methods=['post'])
- def add_peer_endpoint(name):
- result = add_peer(name)
- if result == "success":
- return Response("success")
- else:
- if "already" in result:
- return Response(result, 409)
- else:
- return Response(result, 400)
@private_endpoints_bp.route('/www/', endpoint='www')
def wwwPublic(path):
@@ -75,33 +61,13 @@ class PrivateEndpoints:
def get_is_atty():
return Response(str(sys_stdout.isatty()).lower())
- @private_endpoints_bp.route('/hitcount')
- def get_hit_count():
- return Response(str(client_api.publicAPI.hitCount))
@private_endpoints_bp.route('/ping')
def ping():
# Used to check if client api is working
return Response("pong!")
- @private_endpoints_bp.route('/lastconnect')
- def last_connect():
- return Response(str(client_api.publicAPI.lastRequest))
- @private_endpoints_bp.route('/waitforshare/', methods=['post'])
- def wait_for_share(name):
- """Prevent the **public** api from sharing blocks.
-
- Used for blocks we created usually
- """
- if not name.isalnum():
- raise ValueError('block hash needs to be alpha numeric')
- name = reconstructhash.reconstruct_hash(name)
- if name in client_api.publicAPI.hideBlocks:
- return Response("will be removed")
- else:
- client_api.publicAPI.hideBlocks.append(name)
- return Response("added")
@private_endpoints_bp.route('/shutdown')
def shutdown():
@@ -116,17 +82,6 @@ class PrivateEndpoints:
def get_hidden_blocks():
return Response('\n'.join(client_api.publicAPI.hideBlocks))
- @private_endpoints_bp.route('/getstats')
- def get_stats():
- """Return serialized node statistics."""
- while True:
- try:
- return Response(client_api._too_many.get(
- SerializedData).get_stats())
- except AttributeError:
- pass
- except FileNotFoundError:
- pass
@private_endpoints_bp.route('/getuptime')
def show_uptime():
@@ -159,32 +114,6 @@ class PrivateEndpoints:
def get_os_system():
return Response(platform.system().lower())
- @private_endpoints_bp.route('/gettorsocks')
- def get_tor_socks():
- while True:
- try:
- return Response(
- str(
- g.too_many.get_by_string(
- 'NetController').socksPort))
- except KeyError:
- sleep(0.1)
-
- @private_endpoints_bp.route('/torready')
- def is_tor_ready():
- """If Tor is starting up, the web UI is not ready to be used."""
- try:
- return Response(
- str(g.too_many.get_by_string('NetController').readyState).lower())
- except KeyError:
- return Response("false")
-
- @private_endpoints_bp.route('/gettoraddress')
- def get_tor_address():
- """Return public Tor v3 Onion address for this node"""
- if not config.get('general.security_level', 0) == 0:
- abort(404)
- return Response(get_tor()[0])
@private_endpoints_bp.route('/getgeneratingblocks')
def get_generating_blocks() -> Response:
@@ -195,10 +124,3 @@ class PrivateEndpoints:
))
)
- @private_endpoints_bp.route('/getblockstoupload')
- def get_blocks_to_upload() -> Response:
- return Response(
- ','.join(
- g.too_many.get_by_string('DeadSimpleKV').get('blocksToUpload')
- )
- )
diff --git a/src/httpapi/miscpublicapi/__init__.py b/src/httpapi/miscpublicapi/__init__.py
deleted file mode 100755
index 2f5f9c56..00000000
--- a/src/httpapi/miscpublicapi/__init__.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from . import announce, upload, getblocks, endpoints
-
-announce = announce.handle_announce # endpoint handler for accepting peer announcements
-upload = upload.accept_upload # endpoint handler for accepting public uploads
-public_block_list = getblocks.get_public_block_list # endpoint handler for getting block lists
-public_get_block_data = getblocks.get_block_data # endpoint handler for responding to peers requests for block data
\ No newline at end of file
diff --git a/src/httpapi/miscpublicapi/announce.py b/src/httpapi/miscpublicapi/announce.py
deleted file mode 100755
index 29a78a43..00000000
--- a/src/httpapi/miscpublicapi/announce.py
+++ /dev/null
@@ -1,62 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Handle announcements to the public API server
-"""
-from flask import Response, g
-import deadsimplekv
-
-import logger
-from etc import onionrvalues
-from onionrutils import stringvalidators, bytesconverter
-import filepaths
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def handle_announce(request):
- """accept announcement posts, validating POW
- clientAPI should be an instance of the clientAPI server running,
- request is a instance of a flask request
- """
- resp = 'failure'
- newNode = ''
-
- try:
- newNode = request.form['node'].encode()
- except KeyError:
- logger.warn('No node specified for upload')
- else:
- newNode = bytesconverter.bytes_to_str(newNode)
- announce_queue = deadsimplekv.DeadSimpleKV(filepaths.announce_cache)
- announce_queue_list = announce_queue.get('new_peers')
- if announce_queue_list is None:
- announce_queue_list = []
- else:
- if len(announce_queue_list) >= onionrvalues.MAX_NEW_PEER_QUEUE:
- newNode = ''
-
- if stringvalidators.validate_transport(newNode) and \
- newNode not in announce_queue_list:
- g.shared_state.get(
- deadsimplekv.DeadSimpleKV).get('newPeers').append(newNode)
- announce_queue.put('new_peers',
- announce_queue_list.append(newNode))
- announce_queue.flush()
- resp = 'Success'
-
- resp = Response(resp)
- if resp == 'failure':
- return resp, 406
- return resp
\ No newline at end of file
diff --git a/src/httpapi/miscpublicapi/endpoints.py b/src/httpapi/miscpublicapi/endpoints.py
deleted file mode 100644
index 9c575c07..00000000
--- a/src/httpapi/miscpublicapi/endpoints.py
+++ /dev/null
@@ -1,91 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Misc public API endpoints too small to need their own file
-and that need access to the public api inst
-"""
-from flask import Response, Blueprint, request, send_from_directory, abort, g
-from . import getblocks, upload, announce
-from coredb import keydb
-import config
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-class PublicEndpoints:
- def __init__(self, public_api):
-
- public_endpoints_bp = Blueprint('publicendpoints', __name__)
- self.public_endpoints_bp = public_endpoints_bp
-
- @public_endpoints_bp.route('/')
- def banner():
- # Display info to people who visit a node address in their browser
- try:
- with open('../static-data/index.html', 'r') as html:
- resp = Response(html.read(), mimetype='text/html')
- except FileNotFoundError:
- resp = Response("")
- return resp
-
- @public_endpoints_bp.route('/getblocklist')
- def get_block_list():
- """Get a list of blocks, optionally filtered by epoch time stamp,
- excluding those hidden"""
- return getblocks.get_public_block_list(public_api, request)
-
- @public_endpoints_bp.route('/getdata/')
- def get_block_data(name):
- # Share data for a block if we have it and it isn't hidden
- return getblocks.get_block_data(public_api, name)
-
- @public_endpoints_bp.route('/www/')
- def www_public(path):
- # A way to share files directly over your .onion
- if not config.get("www.public.run", True):
- abort(403)
- return send_from_directory(
- config.get('www.public.path', 'static-data/www/public/'), path)
-
-
- @public_endpoints_bp.route('/plaintext')
- def plaintext_enabled_endpoint():
- return Response(str(config.get("general.store_plaintext_blocks", True)).lower())
-
- @public_endpoints_bp.route('/ping')
- def ping():
- # Endpoint to test if nodes are up
- return Response("pong!")
-
- @public_endpoints_bp.route('/pex')
- def peer_exchange():
- response = ','.join(keydb.listkeys.list_adders(recent=3600))
- if len(response) == 0:
- response = ''
- return Response(response)
-
- @public_endpoints_bp.route('/announce', methods=['post'])
- def accept_announce():
- """Accept announcements with pow token to prevent spam"""
- g.shared_state = public_api._too_many
- resp = announce.handle_announce(request)
- return resp
-
- @public_endpoints_bp.route('/upload', methods=['post'])
- def upload_endpoint():
- """Accept file uploads.
- In the future this will be done more often than on creation
- to speed up block sync
- """
- return upload.accept_upload(request)
diff --git a/src/httpapi/miscpublicapi/getblocks.py b/src/httpapi/miscpublicapi/getblocks.py
deleted file mode 100755
index cc64e6c9..00000000
--- a/src/httpapi/miscpublicapi/getblocks.py
+++ /dev/null
@@ -1,73 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Public endpoints to get block data and lists
-"""
-from flask import Response, abort
-
-import config
-from onionrutils import bytesconverter, stringvalidators
-from coredb import blockmetadb
-from utils import reconstructhash
-from onionrblocks import BlockList
-from onionrblocks.onionrblockapi import Block
-from .. import apiutils
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def get_public_block_list(public_API, request):
- # Provide a list of our blocks, with a date offset
- date_adjust = request.args.get('date')
- type_filter = request.args.get('type')
- b_list = blockmetadb.get_block_list(date_rec=date_adjust)
- share_list = ''
- if config.get('general.hide_created_blocks', True):
- for b in public_API.hideBlocks:
- if b in b_list:
- # Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block
- b_list.remove(b)
- for b in b_list:
- if type_filter:
- if Block(b, decrypt=False).getType() != type_filter:
- continue
- share_list += '%s\n' % (reconstructhash.deconstruct_hash(b),)
- return Response(share_list)
-
-
-def get_block_data(public_API, b_hash):
- """return block data by hash unless we are hiding it"""
- resp = ''
- b_hash = reconstructhash.reconstruct_hash(b_hash)
- if stringvalidators.validate_hash(b_hash):
- if not config.get('general.hide_created_blocks', True) \
- or b_hash not in public_API.hideBlocks:
- if b_hash in public_API._too_many.get(BlockList).get():
- block = apiutils.GetBlockData().get_block_data(
- b_hash, raw=True, decrypt=False)
- try:
- # Encode in case data is binary
- block = block.encode('utf-8')
- except AttributeError:
- # 404 if no block data
- if not block:
- abort(404)
- if not len(block):
- abort(404)
- resp = block
- if len(resp) == 0:
- abort(404)
- resp = ""
- # Has to be octet stream, otherwise binary data fails hash check
- return Response(resp, mimetype='application/octet-stream')
diff --git a/src/httpapi/miscpublicapi/upload.py b/src/httpapi/miscpublicapi/upload.py
deleted file mode 100755
index 9e57f1e4..00000000
--- a/src/httpapi/miscpublicapi/upload.py
+++ /dev/null
@@ -1,94 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Accept block uploads to the public API server
-"""
-import sys
-
-from gevent import spawn
-from flask import Response
-from flask import abort
-from flask import g
-
-from onionrutils import localcommand
-from onionrblocks import blockimporter
-import onionrexceptions
-import logger
-import config
-
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-def accept_upload(request):
- """Accept uploaded blocks to our public Onionr protocol API server"""
- resp = 'failure'
- data = request.get_data()
- data_size = sys.getsizeof(data)
- b_hash = None
- if data_size < 30:
- resp = 'size'
- elif data_size < 100000000:
- try:
- b_hash = blockimporter.import_block_from_data(data)
- if b_hash:
- # Upload mixing is where a node will hide and reupload a block
- # to act like it is also a creator
- # This adds deniability but is very slow
- if g.too_many.get_by_string(
- "DeadSimpleKV").get('onlinePeers') and \
- config.get('general.upload_mixing', False):
- spawn(
- localcommand.local_command,
- '/daemon-event/upload_event',
- post=True,
- is_json=True,
- post_data={'block': b_hash}
- ).get(timeout=10)
- resp = 'success'
- else:
- resp = 'failure'
- logger.warn(
- f'Error encountered importing uploaded block {b_hash}')
- except onionrexceptions.BlacklistedBlock:
- logger.debug('uploaded block is blacklisted')
- resp = 'failure'
- except onionrexceptions.InvalidProof:
- resp = 'proof'
- except onionrexceptions.DataExists:
- resp = 'exists'
- except onionrexceptions.PlaintextNotSupported:
- logger.debug(f"attempted plaintext upload to us: {b_hash}")
- resp = 'failure'
- except onionrexceptions.InvalidMetadata:
- logger.debug(
- f'uploaded block {b_hash} has invalid metadata')
- resp = 'failure'
- if resp == 'failure':
- abort(400)
- elif resp == 'size':
- resp = Response(resp, 400)
- logger.warn(
- f'Error importing uploaded block, invalid size {b_hash}')
- elif resp == 'proof':
- resp = Response(resp, 400)
- if b_hash:
- logger.warn(
- f'Error importing uploaded block, invalid proof {b_hash}')
- else:
- logger.warn(
- 'Error importing uploaded block, invalid proof')
- else:
- resp = Response(resp)
- return resp
diff --git a/src/httpapi/sse/private/__init__.py b/src/httpapi/sse/private/__init__.py
index 4213fd4f..9d9ffaef 100644
--- a/src/httpapi/sse/private/__init__.py
+++ b/src/httpapi/sse/private/__init__.py
@@ -13,7 +13,6 @@ from onionrblocks.onionrblockapi import Block
from coredb.dbfiles import block_meta_db
from coredb.blockmetadb import get_block_list
from onionrutils.epoch import get_epoch
-from onionrstatistics.transports.tor import TorStats
from .. import wrapper
"""
This program is free software: you can redistribute it and/or modify
@@ -44,15 +43,6 @@ def stream_hello():
return SSEWrapper.handle_sse_request(print_hello)
-@private_sse_blueprint.route('/torcircuits')
-def stream_tor_circuits():
- tor_stats = g.too_many.get(TorStats)
- def circuit_stat_stream():
- while True:
- yield "data: " + tor_stats.get_json() + "\n\n"
- sleep(10)
- return SSEWrapper.handle_sse_request(circuit_stat_stream)
-
@private_sse_blueprint.route('/recentblocks')
def stream_recent_blocks():
def _compile_json(b_list):
diff --git a/src/lan/__init__.py b/src/lan/__init__.py
deleted file mode 100644
index ea30efd5..00000000
--- a/src/lan/__init__.py
+++ /dev/null
@@ -1,37 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-LAN manager
-"""
-from typing import TYPE_CHECKING
-from threading import Thread
-if TYPE_CHECKING:
- from toomanyobjs import TooMany
-
-from .discover import learn_services, advertise_service
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-
-class LANManager:
- """Initialize and start/top LAN transport threads."""
-
- def __init__(self, too_many: "TooMany"):
- self.too_many = too_many
- self.peers: "exploded IP Address string" = []
-
- def start(self):
- Thread(target=learn_services, daemon=True).start()
- Thread(target=advertise_service, daemon=True).start()
-
diff --git a/src/lan/client/__init__.py b/src/lan/client/__init__.py
deleted file mode 100644
index 6b1dab66..00000000
--- a/src/lan/client/__init__.py
+++ /dev/null
@@ -1,76 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-LAN transport client thread
-"""
-import requests
-
-from typing import Set
-
-from onionrtypes import LANIP
-import logger
-from coredb.blockmetadb import get_block_list
-from onionrblocks.blockimporter import import_block_from_data
-import onionrexceptions
-from ..server import ports
-from onionrproofs import hashMeetsDifficulty
-
-from threading import Thread
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-connected_lan_peers: Set[LANIP] = set([])
-
-
-def _lan_work(peer: LANIP):
- def _sync_peer(url):
- our_blocks = get_block_list()
- blocks = requests.get(url + 'blist/0').text.splitlines()
- for block in blocks:
- if block not in our_blocks and hashMeetsDifficulty(block):
- try:
- import_block_from_data(
- requests.get(
- url + f'get/{block}', stream=True).raw.read(6000000))
- except onionrexceptions.InvalidMetadata:
- logger.warn(f"Could not get {block} from lan peer")
- except onionrexceptions.InvalidProof:
- logger.warn(
- f"Invalid proof for {block} from lan peer {peer}", terminal=True)
- break
-
- for port in ports:
- try:
- root = f'http://{peer}:{port}/'
- if requests.get(f'{root}ping').text != 'onionr!':
- connected_lan_peers.remove(peer)
- else:
- logger.info(f'[LAN] Connected to {peer}:{port}', terminal=True)
- while True:
- try:
- _sync_peer(root)
- except requests.exceptions.ConnectionError:
- break
- break
- except requests.exceptions.ConnectionError:
- pass
- else:
- connected_lan_peers.remove(peer)
-
-
-def connect_peer(peer: LANIP):
- if peer not in connected_lan_peers:
- connected_lan_peers.add(peer)
- Thread(target=_lan_work, args=[peer], daemon=True).start()
-
diff --git a/src/lan/discover.py b/src/lan/discover.py
deleted file mode 100644
index 42827c4f..00000000
--- a/src/lan/discover.py
+++ /dev/null
@@ -1,83 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Discover and publish private-network
-"""
-import socket
-import struct
-from ipaddress import ip_address
-
-from .getip import lan_ips, best_ip
-from utils.bettersleep import better_sleep
-from . import client
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-MCAST_GRP = '224.0.0.112'
-MCAST_PORT = 1337
-IS_ALL_GROUPS = True
-ANNOUNCE_LOOP_SLEEP = 30
-
-
-def learn_services():
- """Take a list to infintely add lan service info to."""
-
- sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
- sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- if IS_ALL_GROUPS:
- # on this port, receives ALL multicast groups
- sock.bind(('', MCAST_PORT))
- else:
- # on this port, listen ONLY to MCAST_GRP
- sock.bind((MCAST_GRP, MCAST_PORT))
- mreq = struct.pack("4sl", socket.inet_aton(MCAST_GRP), socket.INADDR_ANY)
-
- sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
-
- while True:
- service_ips = sock.recv(200).decode('utf-8')
- if 'onionr' not in service_ips:
- continue
- service_ips = service_ips.replace('onionr-', '').split('-')
-
- for service in service_ips:
- try:
- ip_address(service)
- if not ip_address(service).is_private:
- raise ValueError
- if service in lan_ips:
- raise ValueError
- except ValueError:
- pass
- else:
- client.connect_peer(service)
-
-
-def advertise_service(specific_ips=None):
- # regarding socket.IP_MULTICAST_TTL
- # ---------------------------------
- # for all packets sent, after three hops on the network the packet will not
- # be re-sent/broadcast
- # (see https://www.tldp.org/HOWTO/Multicast-HOWTO-6.html)
- MULTICAST_TTL = 3
-
- ips = best_ip
- if not ips:
- return
-
- sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
- sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, MULTICAST_TTL)
- while True:
- sock.sendto(f"onionr-{ips}".encode('utf-8'), (MCAST_GRP, MCAST_PORT))
- better_sleep(ANNOUNCE_LOOP_SLEEP)
-
diff --git a/src/lan/getip.py b/src/lan/getip.py
deleted file mode 100644
index 0031734b..00000000
--- a/src/lan/getip.py
+++ /dev/null
@@ -1,51 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-Identify LAN ip addresses and determine the best one
-"""
-from ipaddress import IPv4Address
-
-from psutil import net_if_addrs
-from socket import AF_INET
-
-import logger
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-
-lan_ips = []
-
-# https://psutil.readthedocs.io/en/latest/#psutil.net_if_addrs
-def _get_lan_ips():
- for interface in net_if_addrs().keys():
- for address in net_if_addrs()[interface]:
- # Don't see benefit in ipv6, so just check for v4 addresses
- if address[0] == AF_INET:
- # Mark the address for use in LAN if it is a private address
- if IPv4Address(address[1]).is_private and not IPv4Address(address[1]).is_loopback:
- lan_ips.append(address[1])
-try:
- _get_lan_ips()
-except OSError:
- logger.warn("Could not identify LAN ips due to OSError.")
-
-# These are more likely to be actual local subnets rather than VPNs
-for ip in lan_ips:
- if '192.168' in ip:
- best_ip = ip
- break
-else:
- try:
- best_ip = lan_ips[0]
- except IndexError:
- best_ip = ""
diff --git a/src/lan/server/__init__.py b/src/lan/server/__init__.py
deleted file mode 100644
index ce52a8c2..00000000
--- a/src/lan/server/__init__.py
+++ /dev/null
@@ -1,111 +0,0 @@
-"""Onionr - Private P2P Communication.
-
-LAN transport server thread
-"""
-import ipaddress
-import time
-from threading import Thread
-
-from gevent.pywsgi import WSGIServer
-from flask import Flask
-from flask import Response
-from flask import request
-from flask import abort
-
-from onionrblocks.onionrblockapi import Block
-from httpapi.fdsafehandler import FDSafeHandler
-from netcontroller import get_open_port
-import config
-from coredb.blockmetadb import get_block_list
-from lan.getip import best_ip, lan_ips
-from onionrutils import stringvalidators
-from httpapi.miscpublicapi.upload import accept_upload
-import logger
-from utils.bettersleep import better_sleep
-"""
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see .
-"""
-ports = range(1337, 1340)
-_start_time = time.time()
-
-
-class LANServer:
- def __init__(self, shared_state):
- app = Flask(__name__)
- self.app = app
- self.host = config.get('lan.bind_ip', '')
- self.server = None
- if self.host == '':
- self.host = best_ip
- self.port = None
-
- @app.before_request
- def dns_rebinding_prevention():
- if request.remote_addr in lan_ips or \
- ipaddress.ip_address(request.remote_addr).is_loopback:
- if time.time() - _start_time > 600:
- abort(403)
- if request.host != f'{self.host}:{self.port}':
- logger.warn('Potential DNS rebinding attack on LAN server:')
- logger.warn(
- f'Hostname {request.host} was used instead of {self.host}:{self.port}') # noqa
- abort(403)
-
- @app.route('/blist/