Browse Source

mass removal for redesigns

integrate-new-blocks
Kevin Froman 11 months ago
parent
commit
81c8c4f124
  1. 12
      docs/dev/http-api.md
  2. 14
      scripts/disable-dev-config.py
  3. 21
      scripts/enable-dev-config.py
  4. 3
      src/__init__.py
  5. 1
      src/apiservers/README.md
  6. 3
      src/apiservers/__init__.py
  7. 16
      src/apiservers/private/__init__.py
  8. 9
      src/apiservers/private/register_private_blueprints.py
  9. 76
      src/apiservers/public/__init__.py
  10. 1
      src/bigbrother/ministry/ofexec.py
  11. 53
      src/communicator/__init__.py
  12. 36
      src/communicator/bootstrappeers.py
  13. 28
      src/communicator/daemoneventhooks/__init__.py
  14. 33
      src/communicator/daemoneventhooks/removefrominsertqueue.py
  15. 12
      src/communicator/onlinepeers/README.md
  16. 6
      src/communicator/onlinepeers/__init__.py
  17. 35
      src/communicator/onlinepeers/clearofflinepeer.py
  18. 63
      src/communicator/onlinepeers/onlinepeers.py
  19. 47
      src/communicator/onlinepeers/pickonlinepeers.py
  20. 38
      src/communicator/onlinepeers/removeonlinepeer.py
  21. 78
      src/communicator/peeraction.py
  22. 73
      src/communicator/uploadqueue/__init__.py
  23. 33
      src/communicatorutils/README.md
  24. 0
      src/communicatorutils/__init__.py
  25. 77
      src/communicatorutils/announcenode.py
  26. 117
      src/communicatorutils/connectnewpeers.py
  27. 60
      src/communicatorutils/cooldownpeer.py
  28. 35
      src/communicatorutils/deniableinserts.py
  29. 173
      src/communicatorutils/downloadblocks/__init__.py
  30. 42
      src/communicatorutils/downloadblocks/shoulddownload.py
  31. 108
      src/communicatorutils/housekeeping.py
  32. 66
      src/communicatorutils/lookupadders.py
  33. 126
      src/communicatorutils/lookupblocks.py
  34. 64
      src/communicatorutils/netcheck.py
  35. 28
      src/communicatorutils/proxypicker.py
  36. 28
      src/communicatorutils/restarttor.py
  37. 148
      src/communicatorutils/uploadblocks/__init__.py
  38. 48
      src/communicatorutils/uploadblocks/mixmate/__init__.py
  39. 71
      src/communicatorutils/uploadblocks/mixmate/pool.py
  40. 57
      src/communicatorutils/uploadblocks/session.py
  41. 127
      src/communicatorutils/uploadblocks/sessionmanager.py
  42. 1
      src/coredb/__init__.py
  43. 84
      src/coredb/blockmetadb/__init__.py
  44. 49
      src/coredb/blockmetadb/add.py
  45. 41
      src/coredb/blockmetadb/expiredblocks.py
  46. 52
      src/coredb/blockmetadb/updateblockinfo.py
  47. 11
      src/coredb/dbfiles.py
  48. 1
      src/coredb/keydb/__init__.py
  49. 88
      src/coredb/keydb/addkeys.py
  50. 86
      src/coredb/keydb/listkeys.py
  51. 60
      src/coredb/keydb/removekeys.py
  52. 85
      src/coredb/keydb/transportinfo.py
  53. 73
      src/coredb/keydb/userinfo.py
  54. 1
      src/etc/cleanup/__init__.py
  55. 40
      src/etc/humanreadabletime.py
  56. 31
      src/etc/onionrvalues.py
  57. 17
      src/filepaths/__init__.py
  58. 2
      src/httpapi/README.md
  59. 1
      src/httpapi/__init__.py
  60. 4
      src/httpapi/apiutils/__init__.py
  61. 38
      src/httpapi/apiutils/getblockdata.py
  62. 1
      src/httpapi/apiutils/shutdown.py
  63. 75
      src/httpapi/friendsapi/__init__.py
  64. 91
      src/httpapi/insertblock.py
  65. 2
      src/httpapi/miscclientapi/__init__.py
  66. 31
      src/httpapi/miscclientapi/addpeer.py
  67. 123
      src/httpapi/miscclientapi/endpoints.py
  68. 65
      src/httpapi/miscclientapi/getblocks.py
  69. 27
      src/httpapi/miscclientapi/motd/__init__.py
  70. 6
      src/httpapi/miscpublicapi/__init__.py
  71. 62
      src/httpapi/miscpublicapi/announce.py
  72. 91
      src/httpapi/miscpublicapi/endpoints.py
  73. 73
      src/httpapi/miscpublicapi/getblocks.py
  74. 94
      src/httpapi/miscpublicapi/upload.py
  75. 94
      src/httpapi/onionrsitesapi/__init__.py
  76. 49
      src/httpapi/onionrsitesapi/findsite.py
  77. 79
      src/httpapi/onionrsitesapi/sitefiles.py
  78. 2
      src/httpapi/security/__init__.py
  79. 88
      src/httpapi/security/public.py
  80. 20
      src/httpapi/sse/__init__.py
  81. 38
      src/httpapi/sse/private/__init__.py
  82. 81
      src/keymanager.py
  83. 37
      src/lan/__init__.py
  84. 76
      src/lan/client/__init__.py
  85. 83
      src/lan/discover.py
  86. 51
      src/lan/getip.py
  87. 113
      src/lan/server/__init__.py
  88. 7
      src/netcontroller/__init__.py
  89. 40
      src/netcontroller/cleanephemeral.py
  90. 29
      src/netcontroller/getopenport.py
  91. 175
      src/netcontroller/torcontrol/__init__.py
  92. 36
      src/netcontroller/torcontrol/addbridges.py
  93. 44
      src/netcontroller/torcontrol/customtorrc.py
  94. 93
      src/netcontroller/torcontrol/gentorrc.py
  95. 63
      src/netcontroller/torcontrol/onionservice/__init__.py
  96. 81
      src/netcontroller/torcontrol/onionservice/servicecontrol.py
  97. 43
      src/netcontroller/torcontrol/onionservicecreator.py
  98. 48
      src/netcontroller/torcontrol/onionserviceonline.py
  99. 35
      src/netcontroller/torcontrol/rebuildtor.py
  100. 11
      src/netcontroller/torcontrol/togglenetwork.py

12
docs/dev/http-api.md

@ -64,18 +64,6 @@ Please note: endpoints that simply provide static web app files are not document
* /getuptime
- Methods: GET
- Returns uptime in seconds
* /getActivePubkey
- Methods: GET
- Returns the current active public key in base32 format
* /getHumanReadable/pubkey
- Methods: GET
- Echos the specified public key in mnemonic format
* /insertblock
- Methods: POST
- Accepts JSON data for creating a new block. 'message' contains the block data, 'to' specifies the peer's public key to encrypt the data to, 'sign' is a boolean for signing the message.
* /torready
- Methods: POST
- Returns boolean if Tor is started or not
# Public API

14
scripts/disable-dev-config.py

@ -6,26 +6,12 @@ import json
conf = json.load(open('static-data/default_config.json', 'r'))
conf['tor']['use_existing_tor'] = False
conf['tor']['existing_control_port'] = 0
conf['tor']['existing_control_password'] = ""
conf['tor']['existing_socks_port'] = 0
conf['general']['dev_mode'] = False
conf['general']['insert_deniable_blocks'] = True
conf['general']['random_bind_ip'] = True
conf['general']['display_header'] = True
conf['general']['security_level'] = 0
conf['general']['use_bootstrap_list'] = True
conf['onboarding']['done'] = False
conf['general']['minimum_block_pow'] = 5
conf['general']['minimum_send_pow'] = 5
conf['log']['file']['remove_on_exit'] = True
conf['transports']['lan'] = True
conf['transports']['tor'] = True
conf['transports']['sneakernet'] = True
conf['statistics']['i_dont_want_privacy'] = False
conf['statistics']['server'] = ''
conf['ui']['animated_background'] = True
conf['runtests']['skip_slow'] = False

21
scripts/enable-dev-config.py

@ -8,33 +8,16 @@ input("enter to continue") # hack to avoid vscode term input
conf = json.load(open('static-data/default_config.json', 'r'))
block_pow = int(input("Block POW level:"))
conf['general']['security_level'] = int(input("Security level:"))
conf['transports']['tor'] = False
if input('Use Tor? y/n').lower() == 'y':
conf['transports']['tor'] = True
if input("Reuse Tor? y/n:").lower() == 'y':
conf['tor']['use_existing_tor'] = True
conf['tor']['existing_control_port'] = int(input("Enter existing control port:"))
conf['tor']['existing_control_password'] = input("Tor pass:")
conf['tor']['existing_socks_port'] = int(input("Existing socks port:"))
conf['general']['dev_mode'] = True
conf['general']['insert_deniable_blocks'] = False
conf['general']['random_bind_ip'] = False
conf['onboarding']['done'] = True
conf['general']['minimum_block_pow'] = block_pow
conf['general']['minimum_send_pow'] = block_pow
conf['general']['use_bootstrap_list'] = False
if input("Use bootstrap list? y/n").lower() == 'y':
conf['general']['use_bootstrap_list'] = True
conf['log']['file']['remove_on_exit'] = False
conf['ui']['animated_background'] = False
conf['runtests']['skip_slow'] = True
if input('Stat reporting? y/n') == 'y':
conf['statistics']['i_dont_want_privacy'] = True
conf['statistics']['server'] = input('Statistics server')
json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4)

3
src/__init__.py

@ -70,7 +70,6 @@ createdirs.create_dirs()
import bigbrother # noqa
from onionrcommands import parser # noqa
from onionrplugins import onionrevents as events # noqa
from oldblocks.deleteplaintext import delete_plaintext_no_blacklist # noqa
setup.setup_config()
@ -84,8 +83,6 @@ if config.get('advanced.security_auditing', True):
except onionrexceptions.PythonVersion:
pass
if not config.get('general.store_plaintext_blocks', True):
delete_plaintext_no_blacklist()
setup.setup_default_plugins()

1
src/apiservers/README.md

@ -6,4 +6,3 @@ Contains the WSGI servers Onionr uses for remote peer communication and local da
* \_\_init\_\_.py: Exposes the server classes
* private: Contains the client API (the server used to interact with the local Onionr daemon, and view the web UI)
* public: Contains the public API (the server used by remote peers to talk to our daemon)

3
src/apiservers/__init__.py

@ -4,7 +4,6 @@ Public is net-facing server meant for other nodes
Private is meant for controlling and accessing this node
"""
from . import public, private
from . import private
PublicAPI = public.PublicAPI
ClientAPI = private.PrivateAPI

16
src/apiservers/private/__init__.py

@ -17,7 +17,6 @@ import logger
from etc import waitforsetvar
from . import register_private_blueprints
import config
from .. import public
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -69,7 +68,7 @@ class PrivateAPI:
self.httpServer = ''
self.queueResponse = {}
self.get_block_data = httpapi.apiutils.GetBlockData(self)
register_private_blueprints.register_private_blueprints(self, app)
httpapi.load_plugin_blueprints(app)
self.app = app
@ -79,17 +78,11 @@ class PrivateAPI:
waitforsetvar.wait_for_set_var(self, "_too_many")
fd_handler = httpapi.fdsafehandler.FDSafeHandler
self._too_many.add(httpapi.wrappedfunctions.SubProcVDFGenerator(self._too_many))
self.publicAPI = self._too_many.get( # pylint: disable=E1101
public.PublicAPI)
self.httpServer = WSGIServer((self.host, self.bindPort),
self.app, log=None,
handler_class=fd_handler)
self.httpServer.serve_forever()
def setPublicAPIInstance(self, inst):
"""Dynamically set public API instance."""
self.publicAPI = inst
def validateToken(self, token):
"""Validate that the client token matches the given token.
@ -112,10 +105,3 @@ class PrivateAPI:
# Don't error on race condition with startup
pass
def getBlockData(self, bHash, decrypt=False, raw=False,
headerOnly=False) -> bytes:
"""Returns block data bytes."""
return self.get_block_data.get_block_data(bHash,
decrypt=decrypt,
raw=raw,
headerOnly=headerOnly)

9
src/apiservers/private/register_private_blueprints.py

@ -5,8 +5,8 @@ This file registers blueprints for the private api server
from threading import Thread
from gevent import sleep
from httpapi import security, friendsapi, configapi, insertblock
from httpapi import miscclientapi, onionrsitesapi, apiutils
from httpapi import security, configapi
from httpapi import miscclientapi, apiutils
from httpapi import themeapi
from httpapi import fileoffsetreader
from httpapi.sse.private import private_sse_blueprint
@ -31,14 +31,9 @@ def register_private_blueprints(private_api, app):
"""Register private API plask blueprints."""
app.register_blueprint(security.client.ClientAPISecurity(
private_api).client_api_security_bp)
app.register_blueprint(friendsapi.friends)
app.register_blueprint(configapi.config_BP)
app.register_blueprint(insertblock.ib)
app.register_blueprint(miscclientapi.getblocks.client_get_blocks)
app.register_blueprint(miscclientapi.endpoints.PrivateEndpoints(
private_api).private_endpoints_bp)
app.register_blueprint(miscclientapi.motd.bp)
app.register_blueprint(onionrsitesapi.site_api)
app.register_blueprint(apiutils.shutdown.shutdown_bp)
app.register_blueprint(miscclientapi.staticfiles.static_files_bp)
app.register_blueprint(themeapi.theme_blueprint)

76
src/apiservers/public/__init__.py

@ -1,76 +0,0 @@
"""Onionr - Private P2P Communication.
This file handles all incoming http requests
to the public api server, using Flask
"""
import time
import threading
import flask
from gevent.pywsgi import WSGIServer
from httpapi import apiutils, security, fdsafehandler, miscpublicapi
import logger
import config
import filepaths
from utils import gettransports
from etc import onionrvalues, waitforsetvar
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def _get_tor_adder(pub_api):
transports = []
while len(transports) == 0:
transports = gettransports.get()
time.sleep(0.3)
pub_api.torAdder = transports[0]
class PublicAPI:
"""The new client api server, isolated from the public api."""
def __init__(self):
"""Setup the public api app."""
app = flask.Flask('PublicAPI')
app.config['MAX_CONTENT_LENGTH'] = 5 * 1024 * 1024
self.i2pEnabled = config.get('i2p.host', False)
self.hideBlocks = [] # Blocks to be denied sharing
self.host = apiutils.setbindip.set_bind_IP(
filepaths.public_API_host_file)
threading.Thread(target=_get_tor_adder,
args=[self], daemon=True).start()
self.torAdder = ""
self.bindPort = config.get('client.public.port')
self.lastRequest = 0
# total rec requests to public api since server started
self.hitCount = 0
self.config = config
self.API_VERSION = onionrvalues.API_VERSION
logger.info('Running public api on %s:%s' % (self.host, self.bindPort))
app.register_blueprint(
security.public.PublicAPISecurity(self).public_api_security_bp)
app.register_blueprint(
miscpublicapi.endpoints.PublicEndpoints(self).public_endpoints_bp)
self.app = app
def start(self):
"""Start the Public API server."""
waitforsetvar.wait_for_set_var(self, "_too_many")
self.httpServer = WSGIServer((self.host, self.bindPort),
self.app, log=None,
handler_class=fdsafehandler.FDSafeHandler)
self.httpServer.serve_forever()

1
src/bigbrother/ministry/ofexec.py

@ -37,6 +37,7 @@ def block_exec(event, info):
# because libraries have stupid amounts of compile/exec/eval,
# We have to use a whitelist where it can be tolerated
# Generally better than nothing, not a silver bullet
return
whitelisted_code = [
'netrc.py',
'shlex.py',

53
src/communicator/__init__.py

@ -9,26 +9,21 @@ import time
import config
import logger
import onionrplugins as plugins
from communicatorutils import uploadblocks
from . import uploadqueue
from onionrthreads import add_onionr_thread
from onionrcommands.openwebinterface import get_url
from netcontroller import NetController
from . import bootstrappeers
from . import daemoneventhooks
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
config.reload()
@ -51,11 +46,7 @@ class OnionrCommunicatorDaemon:
if config.get('general.offline_mode', False):
self.kv.put('isOnline', False)
# initialize core with Tor socks port being 3rd argument
self.proxyPort = shared_state.get(NetController).socksPort
self.upload_session_manager = self.shared_state.get(
uploadblocks.sessionmanager.BlockUploadSessionManager)
self.shared_state.share_object()
# loop time.sleep delay in seconds
@ -67,12 +58,6 @@ class OnionrCommunicatorDaemon:
# Loads in and starts the enabled plugins
plugins.reload()
# extends our upload list and saves our list when Onionr exits
uploadqueue.UploadQueue(self)
if config.get('general.use_bootstrap_list', True):
bootstrappeers.add_bootstrap_list_to_peer_list(
self.kv, [], db_only=True)
daemoneventhooks.daemon_event_handlers(shared_state)
@ -104,20 +89,6 @@ class OnionrCommunicatorDaemon:
logger.info(
'Goodbye. (Onionr is cleaning up, and will exit)', terminal=True)
def getPeerProfileInstance(self, peer):
"""Gets a peer profile instance from the list of profiles"""
for i in self.kv.get('peerProfiles'):
# if the peer's profile is already loaded, return that
if i.address == peer:
retData = i
break
else:
# if the peer's profile is not loaded, return a new one.
# connectNewPeer also adds it to the list on connect
retData = onionrpeers.PeerProfiles(peer)
self.kv.get('peerProfiles').append(retData)
return retData
def startCommunicator(shared_state):
OnionrCommunicatorDaemon(shared_state)

36
src/communicator/bootstrappeers.py

@ -1,36 +0,0 @@
"""Onionr - Private P2P Communication.
add bootstrap peers to the communicator peer list
"""
from typing import TYPE_CHECKING
from utils import readstatic, gettransports
from coredb import keydb
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
bootstrap_peers = readstatic.read_static('bootstrap-nodes.txt').split(',')
def add_bootstrap_list_to_peer_list(kv, peerList, db_only=False):
"""Add the bootstrap list to the peer list (no duplicates)."""
for i in bootstrap_peers:
# Add bootstrap peers to peerList (does not save them)
# Don't add them if they're already added or in the offline list
if i not in peerList and i not in kv.get('offlinePeers') \
and i not in gettransports.get() and len(str(i).strip()) > 0:
if not db_only:
peerList.append(i)
keydb.addkeys.add_address(i)

28
src/communicator/daemoneventhooks/__init__.py

@ -4,14 +4,10 @@ Hooks to handle daemon events
"""
from threading import Thread
from .removefrominsertqueue import remove_from_insert_queue
from typing import TYPE_CHECKING
from gevent import sleep
from communicatorutils.uploadblocks import mixmate
from communicatorutils import restarttor
if TYPE_CHECKING:
from toomanyobjs import TooMany
@ -19,7 +15,6 @@ if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
from httpapi.daemoneventsapi import DaemonEventsBP
from onionrtypes import BlockHash
from apiservers import PublicAPI
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -44,39 +39,16 @@ def daemon_event_handlers(shared_state: 'TooMany'):
except KeyError:
sleep(0.2)
comm_inst = _get_inst('OnionrCommunicatorDaemon')
public_api: 'PublicAPI' = _get_inst('PublicAPI')
events_api: 'DaemonEventsBP' = _get_inst('DaemonEventsBP')
kv: 'DeadSimpleKV' = _get_inst('DeadSimpleKV')
def remove_from_insert_queue_wrapper(block_hash: 'BlockHash'):
remove_from_insert_queue(comm_inst, block_hash)
return "removed"
def print_test(text=''):
print("It works!", text)
return f"It works! {text}"
def upload_event(block: 'BlockHash' = ''):
if not block:
raise ValueError
public_api.hideBlocks.append(block)
try:
mixmate.block_mixer(kv.get('blocksToUpload'), block)
except ValueError:
pass
return "removed"
def restart_tor():
restarttor.restart(shared_state)
kv.put('offlinePeers', [])
kv.put('onlinePeers', [])
def test_runtime():
Thread(target=comm_inst.shared_state.get_by_string(
"OnionrRunTestManager").run_tests).start()
events_api.register_listener(remove_from_insert_queue_wrapper)
events_api.register_listener(restart_tor)
events_api.register_listener(print_test)
events_api.register_listener(upload_event)
events_api.register_listener(test_runtime)

33
src/communicator/daemoneventhooks/removefrominsertqueue.py

@ -1,33 +0,0 @@
"""Onionr - P2P Anonymous Storage Network.
Remove block hash from daemon's upload list.
"""
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
from communicator import OnionrCommunicatorDaemon
from onionrtypes import BlockHash
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def remove_from_insert_queue(comm_inst: "OnionrCommunicatorDaemon",
b_hash: "BlockHash"):
"""Remove block hash from daemon's upload list."""
kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV")
try:
kv.get('generating_blocks').remove(b_hash)
except ValueError:
pass

12
src/communicator/onlinepeers/README.md

@ -1,12 +0,0 @@
# Online Peers
Manages a pool of peers to perform actions with. Since Onionr does not maintain socket connections, it holds a list of peers.
## Files
* \_\_init\_\_.py: exposes some functions to interact with the pool
* clearofflinepeer.py: Pop the oldest peer in the offline list
* onlinepeers.py: communicator timer to add new peers to the pool randomly
* pickonlinepeers.py: returns a random peer from the online pool
* removeonlinepeer.py: removes a specified peer from the online pool

6
src/communicator/onlinepeers/__init__.py

@ -1,6 +0,0 @@
from . import clearofflinepeer, onlinepeers, pickonlinepeers, removeonlinepeer
clear_offline_peer = clearofflinepeer.clear_offline_peer
get_online_peers = onlinepeers.get_online_peers
pick_online_peer = pickonlinepeers.pick_online_peer
remove_online_peer = removeonlinepeer.remove_online_peer

35
src/communicator/onlinepeers/clearofflinepeer.py

@ -1,35 +0,0 @@
"""Onionr - Private P2P Communication.
clear offline peer in a communicator instance
"""
from typing import TYPE_CHECKING
import logger
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def clear_offline_peer(kv: 'DeadSimpleKV'):
"""Remove the longest offline peer to retry later."""
try:
removed = kv.get('offlinePeers').pop(0)
except IndexError:
pass
else:
logger.debug('Removed ' + removed +
' from offline list, will try them again.')

63
src/communicator/onlinepeers/onlinepeers.py

@ -1,63 +0,0 @@
"""Onionr - Private P2P Communication.
get online peers in a communicator instance
"""
import time
from typing import TYPE_CHECKING
import config
from etc.humanreadabletime import human_readable_time
from communicatorutils.connectnewpeers import connect_new_peer_to_communicator
import logger
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def get_online_peers(shared_state):
"""Manage the kv.get('onlinePeers') attribute list.
Connect to more peers if we have none connected
"""
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
if config.get('general.offline_mode', False):
return
logger.info('Refreshing peer pool...')
max_peers = int(config.get('peers.max_connect', 10))
needed = max_peers - len(kv.get('onlinePeers'))
last_seen = 'never'
if not isinstance(kv.get('lastNodeSeen'), type(None)):
last_seen = human_readable_time(kv.get('lastNodeSeen'))
for _ in range(needed):
if len(kv.get('onlinePeers')) == 0:
connect_new_peer_to_communicator(shared_state, useBootstrap=True)
else:
connect_new_peer_to_communicator(shared_state)
if kv.get('shutdown'):
break
else:
if len(kv.get('onlinePeers')) == 0:
logger.debug('Couldn\'t connect to any peers.' +
f' Last node seen {last_seen} ago.')
try:
get_online_peers(shared_state)
except RecursionError:
pass
else:
kv.put('lastNodeSeen', time.time())

47
src/communicator/onlinepeers/pickonlinepeers.py

@ -1,47 +0,0 @@
"""
Onionr - Private P2P Communication.
pick online peers in a communicator instance
"""
import secrets
from typing import TYPE_CHECKING
import onionrexceptions
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def pick_online_peer(kv: 'DeadSimpleKV'):
"""Randomly picks peer from pool without bias (using secrets module)."""
ret_data = ''
peer_length = len(kv.get('onlinePeers'))
if peer_length <= 0:
raise onionrexceptions.OnlinePeerNeeded
while True:
peer_length = len(kv.get('onlinePeers'))
try:
# Get a random online peer, securely.
# May get stuck in loop if network is lost
ret_data = kv.get('onlinePeers')[secrets.randbelow(peer_length)]
except IndexError:
pass
else:
break
return ret_data

38
src/communicator/onlinepeers/removeonlinepeer.py

@ -1,38 +0,0 @@
"""Onionr - Private P2P Communication.
remove an online peer from the pool in a communicator instance
"""
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def remove_online_peer(kv, peer):
"""Remove an online peer."""
try:
del kv.get('connectTimes')[peer]
except KeyError:
pass
try:
del kv.get('dbTimestamps')[peer]
except KeyError:
pass
try:
kv.get('onlinePeers').remove(peer)
except ValueError:
pass

78
src/communicator/peeraction.py

@ -1,78 +0,0 @@
"""Onionr - Private P2P Communication.
This file implements logic for performing requests to Onionr peers
"""
from typing import TYPE_CHECKING
import streamedrequests
import logger
from onionrutils import epoch, basicrequests
from coredb import keydb
from . import onlinepeers
from onionrtypes import OnionAddressString
from onionrpeers.peerprofiles import PeerProfiles
from etc.waitforsetvar import wait_for_set_var
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def get_peer_profile(kv, address: OnionAddressString) -> 'PeerProfiles':
profile_inst_list = kv.get('peerProfiles')
for profile in profile_inst_list:
if profile.address == address:
return profile
p = PeerProfiles(address)
profile_inst_list.append(p)
return p
def peer_action(shared_state, peer, action,
returnHeaders=False, max_resp_size=5242880):
"""Perform a get request to a peer."""
penalty_score = -10
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
if len(peer) == 0:
return False
url = 'http://%s/%s' % (peer, action)
try:
ret_data = basicrequests.do_get_request(url, port=kv.get('proxyPort'),
max_size=max_resp_size)
except streamedrequests.exceptions.ResponseLimitReached:
logger.warn(
'Request failed due to max response size being overflowed',
terminal=True)
ret_data = False
penalty_score = -100
# if request failed, (error), mark peer offline
if ret_data is False:
try:
get_peer_profile(kv, peer).addScore(penalty_score)
onlinepeers.remove_online_peer(kv, peer)
keydb.transportinfo.set_address_info(
peer, 'lastConnectAttempt', epoch.get_epoch())
if action != 'ping' and not kv.get('shutdown'):
logger.warn(f'Lost connection to {peer}', terminal=True)
# Will only add a new peer to pool if needed
onlinepeers.get_online_peers(kv)
except ValueError:
pass
else:
peer_profile = get_peer_profile(kv, peer)
peer_profile.update_connect_time()
peer_profile.addScore(1)
# If returnHeaders, returns tuple of data, headers.
# If not, just data string
return ret_data

73
src/communicator/uploadqueue/__init__.py

@ -1,73 +0,0 @@
"""Onionr - Private P2P Communication.
Class to remember blocks that need to be uploaded
and not shared on startup/shutdown
"""
import atexit
import os
from typing import TYPE_CHECKING
import deadsimplekv
import filepaths
from onionrutils import localcommand
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
UPLOAD_MEMORY_FILE = filepaths.upload_list
def _add_to_hidden_blocks(cache):
for bl in cache:
localcommand.local_command('waitforshare/' + bl, post=True)
class UploadQueue:
"""Saves and loads block upload info from json file."""
def __init__(self, communicator: 'OnionrCommunicatorDaemon'):
"""Start the UploadQueue object, loading left over uploads into queue.
register save shutdown function
"""
self.communicator = communicator
cache: deadsimplekv.DeadSimpleKV = deadsimplekv.DeadSimpleKV(
UPLOAD_MEMORY_FILE)
self.kv: "DeadSimpleKV" = \
communicator.shared_state.get_by_string("DeadSimpleKV")
self.store_obj = cache
cache = cache.get('uploads')
if cache is None:
cache = []
_add_to_hidden_blocks(cache)
self.kv.get('blocksToUpload').extend(cache)
atexit.register(self.save)
def save(self):
"""Save to disk on shutdown or if called manually."""
bl: deadsimplekv.DeadSimpleKV = self.kv.get('blocksToUpload')
if len(bl) == 0:
try:
os.remove(UPLOAD_MEMORY_FILE)
except FileNotFoundError:
pass
else:
self.store_obj.put('uploads', bl)
self.store_obj.flush()

33
src/communicatorutils/README.md

@ -1,33 +0,0 @@
# communicatorutils
The files in this submodule handle various subtasks and utilities for the onionr communicator.
## Files:
announcenode.py: Uses a communicator instance to announce our transport address to connected nodes
connectnewpeers.py: takes a communicator instance and has it connect to as many peers as needed, and/or to a new specified peer.
cooldownpeer.py: randomly selects a connected peer in a communicator and disconnects them for the purpose of security and network balancing.
daemonqueuehandler.py: checks for new commands in the daemon queue and processes them accordingly.
deniableinserts.py: insert fake blocks with the communicator for plausible deniability
downloadblocks.py: iterates a communicator instance's block download queue and attempts to download the blocks from online peers
housekeeping.py: cleans old blocks and forward secrecy keys
lookupadders.py: ask connected peers to share their list of peer transport addresses
lookupblocks.py: lookup new blocks from connected peers from the communicator
netcheck.py: check if the node is online based on communicator status and onion server ping results
onionrcommunicataortimers.py: create a timer for a function to be launched on an interval. Control how many possible instances of a timer may be running a function at once and control if the timer should be ran in a thread or not.
proxypicker.py: returns a string name for the appropriate proxy to be used with a particular peer transport address.
servicecreator.py: iterate connection blocks and create new direct connection servers for them.
uploadblocks.py: iterate a communicator's upload queue and upload the blocks to connected peers

0
src/communicatorutils/__init__.py

77
src/communicatorutils/announcenode.py

@ -1,77 +0,0 @@
"""
Onionr - Private P2P Communication.
Use a communicator instance to announce
our transport address to connected nodes
"""
from typing import TYPE_CHECKING
import logger
from onionrutils import basicrequests
from utils import gettransports
from netcontroller import NetController
from communicator import onlinepeers
from coredb import keydb
import onionrexceptions
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def announce_node(shared_state):
"""Announce our node to our peers."""
ret_data = False
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
config = shared_state.get_by_string("OnionrCommunicatorDaemon").config
# Do not let announceCache get too large
if len(kv.get('announceCache')) >= 10000:
kv.get('announceCache').popitem()
if config.get('general.security_level', 0) == 0:
# Announce to random online peers
for i in kv.get('onlinePeers'):
if i not in kv.get('announceCache'):
peer = i
break
else:
try:
peer = onlinepeers.pick_online_peer(kv)
except onionrexceptions.OnlinePeerNeeded:
peer = ""
try:
ourID = gettransports.get()[0]
if not peer:
raise onionrexceptions.OnlinePeerNeeded
except (IndexError, onionrexceptions.OnlinePeerNeeded):
pass
else:
url = 'http://' + peer + '/announce'
data = {'node': ourID}
logger.info('Announcing node to ' + url)
if basicrequests.do_post_request(
url,
data,
port=shared_state.get(NetController).socksPort)\
== 'Success':
logger.info('Successfully introduced node to ' + peer,
terminal=True)
ret_data = True
keydb.transportinfo.set_address_info(peer, 'introduced', 1)
return ret_data

117
src/communicatorutils/connectnewpeers.py

@ -1,117 +0,0 @@
"""Onionr - Private P2P Communication.
Connect a new peer to our communicator instance.
Does so randomly if no peer is specified
"""
import time
import secrets
import onionrexceptions
import logger
import onionrpeers
from utils import networkmerger, gettransports
from onionrutils import stringvalidators, epoch
from communicator import peeraction, bootstrappeers
from coredb import keydb
import config
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def connect_new_peer_to_communicator(shared_state, peer='', useBootstrap=False):
retData = False
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
tried = kv.get('offlinePeers')
transports = gettransports.get()
if peer != '':
if stringvalidators.validate_transport(peer):
peerList = [peer]
else:
raise onionrexceptions.InvalidAddress(
'Will not attempt connection test to invalid address')
else:
peerList = keydb.listkeys.list_adders()
mainPeerList = keydb.listkeys.list_adders()
if not peerList:
peerList = onionrpeers.get_score_sorted_peer_list()
"""
If we don't have enough peers connected or random chance,
select new peers to try
"""
if len(peerList) < 8 or secrets.randbelow(4) == 3:
tryingNew = []
for x in kv.get('newPeers'):
if x not in peerList:
peerList.append(x)
tryingNew.append(x)
for i in tryingNew:
kv.get('newPeers').remove(i)
if len(peerList) == 0 or useBootstrap:
# Avoid duplicating bootstrap addresses in peerList
if config.get('general.use_bootstrap_list', True):
bootstrappeers.add_bootstrap_list_to_peer_list(kv, peerList)
for address in peerList:
address = address.strip()
# Don't connect to our own address
if address in transports:
continue
"""Don't connect to invalid address or
if its already been tried/connected, or if its cooled down
"""
if len(address) == 0 or address in tried \
or address in kv.get('onlinePeers') \
or address in kv.get('cooldownPeer'):
continue
if kv.get('shutdown'):
return
# Ping a peer,
ret = peeraction.peer_action(shared_state, address, 'ping')
if ret == 'pong!':
time.sleep(0.1)
if address not in mainPeerList:
# Add a peer to our list if it isn't already since it connected
networkmerger.mergeAdders(address)
if address not in kv.get('onlinePeers'):
logger.info('Connected to ' + address, terminal=True)
kv.get('onlinePeers').append(address)
kv.get('connectTimes')[address] = epoch.get_epoch()
retData = address
# add peer to profile list if they're not in it
for profile in kv.get('peerProfiles'):
if profile.address == address:
break
else:
kv.get('peerProfiles').append(
onionrpeers.PeerProfiles(address))
try:
del kv.get('plaintextDisabledPeers')[address]
except KeyError:
pass
if peeraction.peer_action(
shared_state, address, 'plaintext') == 'false':
kv.get('plaintextDisabledPeers')[address] = True
break
else:
# Mark a peer as tried if they failed to respond to ping
tried.append(address)
logger.debug('Failed to connect to %s: %s ' % (address, ret))
return retData

60
src/communicatorutils/cooldownpeer.py

@ -1,60 +0,0 @@
"""Onionr - Private P2P Communication.
Select random online peer in a communicator instance and have them "cool down"
"""
from typing import TYPE_CHECKING
from onionrutils import epoch
from communicator import onlinepeers
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def cooldown_peer(shared_state):
"""Randomly add an online peer to cooldown, so we can connect a new one."""
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
config = shared_state.get_by_string("OnionrCommunicatorDaemon").config
online_peer_amount = len(kv.get('onlinePeers'))
minTime = 300
cooldown_time = 600
to_cool = ''
tempConnectTimes = dict(kv.get('connectTimes'))
# Remove peers from cooldown that have been there long enough
tempCooldown = dict(kv.get('cooldownPeer'))
for peer in tempCooldown:
if (epoch.get_epoch() - tempCooldown[peer]) >= cooldown_time:
del kv.get('cooldownPeer')[peer]
# Cool down a peer, if we have max connections alive for long enough
if online_peer_amount >= config.get('peers.max_connect', 10, save=True):
finding = True
while finding:
try:
to_cool = min(tempConnectTimes, key=tempConnectTimes.get)
if (epoch.get_epoch() - tempConnectTimes[to_cool]) < minTime:
del tempConnectTimes[to_cool]
else:
finding = False
except ValueError:
break
else:
onlinepeers.remove_online_peer(kv, to_cool)
kv.get('cooldownPeer')[to_cool] = epoch.get_epoch()

35
src/communicatorutils/deniableinserts.py

@ -1,35 +0,0 @@
"""Onionr - Private P2P Communication.
Use the communicator to insert fake mail messages
"""
import secrets
from etc import onionrvalues
import oldblocks
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def insert_deniable_block():
"""Insert a fake block to make it more difficult to track real blocks."""
fakePeer = ''
chance = 10
if secrets.randbelow(chance) == (chance - 1):
# This assumes on the libsodium primitives to have key-privacy
fakePeer = onionrvalues.DENIABLE_PEER_ADDRESS
data = secrets.token_hex(secrets.randbelow(5120) + 1)
oldblocks.insert(data, header='pm', encryptType='asym',
asymPeer=fakePeer, disableForward=True,
meta={'subject': 'foo'})

173
src/communicatorutils/downloadblocks/__init__.py

@ -1,173 +0,0 @@
"""Onionr - Private P2P Communication.
Download blocks using the communicator instance.
"""
from typing import TYPE_CHECKING
from secrets import SystemRandom
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
from deadsimplekv import DeadSimpleKV
from gevent import spawn
import onionrexceptions
import logger
import onionrpeers
from communicator import peeraction
from communicator import onlinepeers
from oldblocks import blockmetadata
from onionrutils import validatemetadata
from coredb import blockmetadb
from onionrutils.localcommand import local_command
import onionrcrypto
import onionrstorage
from oldblocks import onionrblacklist
from oldblocks import storagecounter
from . import shoulddownload
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
storage_counter = storagecounter.StorageCounter()
def download_blocks_from_communicator(shared_state: "TooMany"):
"""Use communicator instance to download blocks in the comms's queue"""
blacklist = onionrblacklist.OnionrBlackList()
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
count: int = 0
metadata_validation_result: bool = False
# Iterate the block queue in the communicator
for blockHash in list(kv.get('blockQueue')):
count += 1
try:
blockPeers = list(kv.get('blockQueue')[blockHash])
except KeyError:
blockPeers = []
removeFromQueue = True
if not shoulddownload.should_download(shared_state, blockHash):
continue
if kv.get('shutdown') or not kv.get('isOnline') or \
storage_counter.is_full():
# Exit loop if shutting down or offline, or disk allocation reached
break
# Do not download blocks being downloaded
if blockHash in kv.get('currentDownloading'):
continue
if len(kv.get('onlinePeers')) == 0:
break
# So we can avoid concurrent downloading in other threads of same block
kv.get('currentDownloading').append(blockHash)
if len(blockPeers) == 0:
try:
peerUsed = onlinepeers.pick_online_peer(kv)
except onionrexceptions.OnlinePeerNeeded:
continue
else:
SystemRandom().shuffle(blockPeers)
peerUsed = blockPeers.pop(0)
if not kv.get('shutdown') and peerUsed.strip() != '':
logger.info(
f"Attempting to download %s from {peerUsed}..." % (blockHash[:12],))
content = peeraction.peer_action(
shared_state, peerUsed,
'getdata/' + blockHash,
max_resp_size=3000000) # block content from random peer
if content is not False and len(content) > 0:
try:
content = content.encode()
except AttributeError:
pass
realHash = onionrcrypto.hashers.sha3_hash(content)
try:
realHash = realHash.decode() # bytes on some versions for some reason
except AttributeError:
pass
if realHash == blockHash:
#content = content.decode() # decode here because sha3Hash needs bytes above
metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
metadata = metas[0]
try:
metadata_validation_result = \
validatemetadata.validate_metadata(metadata, metas[2])
except onionrexceptions.PlaintextNotSupported:
logger.debug(f"Not saving {blockHash} due to plaintext not enabled")
removeFromQueue = True
except onionrexceptions.DataExists:
metadata_validation_result = False
if metadata_validation_result: # check if metadata is valid, and verify nonce
if onionrcrypto.cryptoutils.verify_POW(content): # check if POW is enough/correct
logger.info('Attempting to save block %s...' % blockHash[:12])
try:
onionrstorage.set_data(content)
except onionrexceptions.DataExists:
logger.warn('Data is already set for %s ' % (blockHash,))
except onionrexceptions.DiskAllocationReached:
logger.error('Reached disk allocation allowance, cannot save block %s.' % (blockHash,))
removeFromQueue = False
else:
blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db
blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
spawn(
local_command,
f'/daemon-event/upload_event',
post=True,
is_json=True,
post_data={'block': blockHash}
)
else:
logger.warn('POW failed for block %s.' % (blockHash,))
else:
if blacklist.inBlacklist(realHash):
logger.warn('Block %s is blacklisted.' % (realHash,))
else:
logger.warn('Metadata for block %s is invalid.' % (blockHash,))
blacklist.addToDB(blockHash)
else:
# if block didn't meet expected hash
tempHash = onionrcrypto.hashers.sha3_hash(content) # lazy hack, TODO use var
try:
tempHash = tempHash.decode()
except AttributeError:
pass
# Punish peer for sharing invalid block (not always malicious, but is bad regardless)
onionrpeers.PeerProfiles(peerUsed).addScore(-50)
if tempHash != 'ed55e34cb828232d6c14da0479709bfa10a0923dca2b380496e6b2ed4f7a0253':
# Dumb hack for 404 response from peer. Don't log it if 404 since its likely not malicious or a critical error.
logger.warn(
'Block hash validation failed for ' +
blockHash + ' got ' + tempHash)
else:
removeFromQueue = False # Don't remove from queue if 404
if removeFromQueue:
try:
del kv.get('blockQueue')[blockHash] # remove from block queue both if success or false
if count == LOG_SKIP_COUNT:
logger.info('%s blocks remaining in queue' %
[len(kv.get('blockQueue'))], terminal=True)
count = 0
except KeyError:
pass