kev/Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

dont upload blocks that arent valid (anymore?), progress removing communicator timers

Browse Source
This commit is contained in:
Kevin Froman 2020-11-16 06:57:38 +00:00
parent cc703fc404
commit a775f3b26d
7 changed files with 44 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
scripts/disable-dev-config.py
View File

@ -16,6 +16,7 @@ conf['general']['insert_deniable_blocks'] = True
conf['general']['random_bind_ip'] = True
conf['general']['display_header'] = True
conf['general']['security_level'] = 0
conf['general']['use_bootstrap_list'] = True
conf['onboarding']['done'] = False
conf['general']['minimum_block_pow'] = 5
conf['general']['minimum_send_pow'] = 5

3
scripts/enable-dev-config.py
View File

@ -26,6 +26,9 @@ conf['general']['random_bind_ip'] = False
conf['onboarding']['done'] = True
conf['general']['minimum_block_pow'] = block_pow
conf['general']['minimum_send_pow'] = block_pow
conf['general']['use_bootstrap_list'] = False
if input("Use bootstrap list? y/n").lower() == 'y':
conf['general']['use_bootstrap_list'] = True
conf['log']['file']['remove_on_exit'] = False
conf['ui']['animated_background'] = False
if input('Stat reporting? y/n') == 'y':

19
src/communicator/__init__.py
View File

@ -57,6 +57,7 @@ class OnionrCommunicatorDaemon:
# configure logger and stuff
self.config = config
self.shared_state = shared_state # TooManyObjects module
shared_state.add(self)
# populate kv values
self.kv = self.shared_state.get_by_string('DeadSimpleKV')
@ -98,7 +99,7 @@ class OnionrCommunicatorDaemon:
add_onionr_thread(onlinepeers.clear_offline_peer, [self.kv], 58)
add_onionr_thread(
housekeeping.clean_old_blocks, [self.shared_state], 20, 1)
housekeeping.clean_old_blocks, [self.shared_state], 10, 1)
# Discover new peers
add_onionr_thread(
@ -108,12 +109,13 @@ class OnionrCommunicatorDaemon:
# Timer for adjusting which peers
# we actively communicate to at any given time,
# to avoid over-using peers
add_onionr_thread(cooldownpeer.cooldown_peer, [self.shared_state], 30, 60)
add_onionr_thread(
cooldownpeer.cooldown_peer, [self.shared_state], 30, 60)
# Timer to read the upload queue and upload the entries to peers
OnionrCommunicatorTimers(
self, uploadblocks.upload_blocks_from_communicator,
5, my_args=[self], requires_peer=True, max_threads=1)
add_onionr_thread(
uploadblocks.upload_blocks_from_communicator,
[self.shared_state], 5, 1)
# Setup direct connections
if config.get('general.ephemeral_tunnels', False):
@ -132,10 +134,7 @@ class OnionrCommunicatorDaemon:
# This timer creates deniable blocks,
# in an attempt to further obfuscate block insertion metadata
if config.get('general.insert_deniable_blocks', True):
deniableBlockTimer = OnionrCommunicatorTimers(
self, deniableinserts.insert_deniable_block,
180, my_args=[self], requires_peer=True, max_threads=1)
deniableBlockTimer.count = (deniableBlockTimer.frequency - 175)
add_onionr_thread(deniableinserts.insert_deniable_block, [], 180, 10)
if config.get('transports.tor', True):
# Timer to check for connectivity,
@ -168,8 +167,6 @@ class OnionrCommunicatorDaemon:
# Adjust initial timer triggers
cleanupTimer.count = (cleanupTimer.frequency - 60)
shared_state.add(self)
if config.get('general.use_bootstrap_list', True):
bootstrappeers.add_bootstrap_list_to_peer_list(
self.kv, [], db_only=True)

3
src/communicatorutils/deniableinserts.py
View File

@ -22,7 +22,7 @@ import onionrblocks
"""
def insert_deniable_block(comm_inst):
def insert_deniable_block():
"""Insert a fake block to make it more difficult to track real blocks."""
fakePeer = ''
chance = 10
@ -33,4 +33,3 @@ def insert_deniable_block(comm_inst):
onionrblocks.insert(data, header='pm', encryptType='asym',
asymPeer=fakePeer, disableForward=True,
meta={'subject': 'foo'})
comm_inst.decrementThreadCount('insert_deniable_block')

35
src/communicatorutils/uploadblocks/__init__.py
View File

@ -14,7 +14,9 @@ import logger
from communicatorutils import proxypicker
import onionrexceptions
from onionrblocks import onionrblockapi as block
from onionrblocks.blockmetadata.fromdata import get_block_metadata_from_data
from onionrutils import stringvalidators, basicrequests
from onionrutils.validatemetadata import validate_metadata
from communicator import onlinepeers
if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV
@ -35,15 +37,14 @@ if TYPE_CHECKING:
"""
def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
def upload_blocks_from_communicator(shared_state: 'OnionrCommunicatorDaemon'):
"""Accept a communicator instance + upload blocks from its upload queue."""
"""when inserting a block, we try to upload
it to a few peers to add some deniability & increase functionality"""
kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV")
TIMER_NAME = "upload_blocks_from_communicator"
kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
session_manager: sessionmanager.BlockUploadSessionManager
session_manager = comm_inst.shared_state.get(
session_manager = shared_state.get(
sessionmanager.BlockUploadSessionManager)
tried_peers: UserID = []
finishedUploads = []
@ -53,7 +54,7 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
def remove_from_hidden(bl):
sleep(60)
try:
comm_inst.shared_state.get_by_string(
shared_state.get_by_string(
'PublicAPI').hideBlocks.remove(bl)
except ValueError:
pass
@ -62,7 +63,6 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
for bl in kv.get('blocksToUpload'):
if not stringvalidators.validate_hash(bl):
logger.warn('Requested to upload invalid block', terminal=True)
comm_inst.decrementThreadCount(TIMER_NAME)
return
session = session_manager.add_session(bl)
for _ in range(min(len(kv.get('onlinePeers')), 6)):
@ -92,10 +92,22 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
data = block.Block(bl).getRaw()
if not data:
logger.warn(
f"Couldn't data for block in upload list {bl}",
f"Couldn't get data for block in upload list {bl}",
terminal=True)
raise onionrexceptions.NoDataAvailable
except onionrexceptions.NoDataAvailable:
try:
def __check_metadata():
metadata = get_block_metadata_from_data(data)[0]
if not validate_metadata(metadata, data):
logger.warn(
f"Metadata for uploading block not valid {bl}")
raise onionrexceptions.InvalidMetadata
__check_metadata()
except onionrexceptions.DataExists:
pass
except( # noqa
onionrexceptions.NoDataAvailable,
onionrexceptions.InvalidMetadata) as _:
finishedUploads.append(bl)
break
proxy_type = proxypicker.pick_proxy(peer)
@ -116,7 +128,9 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
else:
session.fail()
session.fail_peer(peer)
comm_inst.getPeerProfileInstance(peer).addScore(-5)
shared_state.get_by_string(
'OnionrCommunicatorDaemon').getPeerProfileInstance(
peer).addScore(-5)
logger.warn(
f'Failed to upload {bl[:8]}, reason: {resp}',
terminal=True)
@ -127,9 +141,8 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
try:
kv.get('blocksToUpload').remove(x)
comm_inst.shared_state.get_by_string(
shared_state.get_by_string(
'PublicAPI').hideBlocks.remove(x)
except ValueError:
pass
comm_inst.decrementThreadCount(TIMER_NAME)

2
src/httpapi/miscclientapi/endpoints.py
View File

@ -116,6 +116,8 @@ class PrivateEndpoints:
SerializedData).get_stats())
except AttributeError:
pass
except FileNotFoundError:
pass
@private_endpoints_bp.route('/getuptime')
def show_uptime():

4
src/httpapi/miscpublicapi/upload.py
View File

@ -71,6 +71,10 @@ def accept_upload(request):
except onionrexceptions.PlaintextNotSupported:
logger.debug("attempted plaintext upload to us: {b_hash}")
resp = 'failure'
except onionrexceptions.InvalidMetadata:
logger.debug(
'uploaded block {b_hash} has invalid metadata')
resp = 'failure'
if resp == 'failure':
abort(400)
elif resp == 'size':