added mixmate to improve base routing

This commit is contained in:
Kevin Froman 2019-12-22 13:42:10 -06:00
parent f71f2f6246
commit 9329b07e3b
15 changed files with 273 additions and 67 deletions

View File

@ -57,6 +57,8 @@ def block_exec(event, info):
] ]
home = identifyhome.identify_home() home = identifyhome.identify_home()
code_b64 = base64.b64encode(info[0].co_code).decode()
for source in whitelisted_code: for source in whitelisted_code:
if info[0].co_filename.endswith(source): if info[0].co_filename.endswith(source):
return return
@ -64,7 +66,6 @@ def block_exec(event, info):
if home + 'plugins/' in info[0].co_filename: if home + 'plugins/' in info[0].co_filename:
return return
code_b64 = base64.b64encode(info[0].co_code).decode()
logger.warn('POSSIBLE EXPLOIT DETECTED, SEE LOGS', terminal=True) logger.warn('POSSIBLE EXPLOIT DETECTED, SEE LOGS', terminal=True)
logger.warn('POSSIBLE EXPLOIT DETECTED: ' + info[0].co_filename) logger.warn('POSSIBLE EXPLOIT DETECTED: ' + info[0].co_filename)
logger.warn('Prevented exec/eval. Report this with the sample below') logger.warn('Prevented exec/eval. Report this with the sample below')

View File

@ -1,9 +1,15 @@
''' """Onionr - P2P Anonymous Storage Network.
Onionr - P2P Anonymous Storage Network
Handle daemon queue commands in the communicator Handle daemon queue commands in the communicator
''' """
''' import logger
from onionrplugins import onionrevents as events
from onionrutils import localcommand
from coredb import daemonqueue
import filepaths
from . import restarttor
from communicatorutils.uploadblocks import mixmate
"""
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or the Free Software Foundation, either version 3 of the License, or
@ -16,13 +22,9 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' """
import logger
from onionrplugins import onionrevents as events
from onionrutils import localcommand
from coredb import daemonqueue
import filepaths
from . import restarttor
def handle_daemon_commands(comm_inst): def handle_daemon_commands(comm_inst):
cmd = daemonqueue.daemon_queue() cmd = daemonqueue.daemon_queue()
response = '' response = ''
@ -62,6 +64,13 @@ def handle_daemon_commands(comm_inst):
i.count = (i.frequency - 1) i.count = (i.frequency - 1)
elif cmd[0] == 'uploadBlock': elif cmd[0] == 'uploadBlock':
comm_inst.blocksToUpload.append(cmd[1]) comm_inst.blocksToUpload.append(cmd[1])
elif cmd[0] == 'uploadEvent':
try:
mixmate.block_mixer(comm_inst.blocksToUpload, cmd[1])
except ValueError:
pass
else:
localcommand.local_command('/waitforshare/' + cmd[1], post=True, maxWait=5)
else: else:
logger.debug('Received daemon queue command unable to be handled: %s' % (cmd[0],)) logger.debug('Received daemon queue command unable to be handled: %s' % (cmd[0],))

View File

@ -1,8 +1,11 @@
''' """
Onionr - Private P2P Communication Onionr - Private P2P Communication
Download blocks using the communicator instance Download blocks using the communicator instance
''' """
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
import onionrexceptions import onionrexceptions
import logger import logger
import onionrpeers import onionrpeers
@ -12,12 +15,13 @@ from communicator import onlinepeers
from onionrutils import blockmetadata from onionrutils import blockmetadata
from onionrutils import validatemetadata from onionrutils import validatemetadata
from coredb import blockmetadb from coredb import blockmetadb
from coredb import daemonqueue
import onionrcrypto import onionrcrypto
import onionrstorage import onionrstorage
from onionrblocks import onionrblacklist from onionrblocks import onionrblacklist
from onionrblocks import storagecounter from onionrblocks import storagecounter
from . import shoulddownload from . import shoulddownload
''' """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or the Free Software Foundation, either version 3 of the License, or
@ -30,11 +34,11 @@ from . import shoulddownload
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' """
def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"): def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
'''Use communicator instance to download blocks in the comms's queue''' """Use communicator instance to download blocks in the comms's queue"""
blacklist = onionrblacklist.OnionrBlackList() blacklist = onionrblacklist.OnionrBlackList()
storage_counter = storagecounter.StorageCounter() storage_counter = storagecounter.StorageCounter()
LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
@ -109,6 +113,7 @@ def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
removeFromQueue = False removeFromQueue = False
else: else:
blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db
daemonqueue.daemon_queue_add('uploadEvent', blockHash)
blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
else: else:
logger.warn('POW failed for block %s.' % (blockHash,)) logger.warn('POW failed for block %s.' % (blockHash,))

View File

@ -27,6 +27,7 @@ from onionrutils import localcommand, stringvalidators, basicrequests
from communicator import onlinepeers from communicator import onlinepeers
import onionrcrypto import onionrcrypto
from . import sessionmanager from . import sessionmanager
from . import mixmate
def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon): def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
"""Accepts a communicator instance and uploads blocks from its upload queue""" """Accepts a communicator instance and uploads blocks from its upload queue"""

View File

@ -0,0 +1,51 @@
"""Onionr - Private P2P Communication.
Delay block uploads, optionally mixing them together
"""
import time
from typing import List
import onionrtypes
from onionrblocks import onionrblockapi
from .pool import UploadPool
from .pool import PoolFullException
from etc import onionrvalues
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
upload_pool = UploadPool(4)
def block_mixer(upload_list: List[onionrtypes.BlockHash],
block_to_mix: onionrtypes.BlockHash):
"""Delay and mix block inserts.
Take a block list and a received/created block and add it
to the said block list
"""
bl = onionrblockapi.Block(block_to_mix)
if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE:
raise ValueError
try:
# add the new block to pool
upload_pool.add_to_pool(block_to_mix)
except PoolFullException:
# If the pool is full, move into upload queue
upload_list.extend(upload_pool.get_pool())
# then finally begin new pool with new block
upload_pool.add_to_pool(block_to_mix)

View File

@ -0,0 +1,69 @@
"""Onionr - Private P2P Communication.
Upload pool
"""
from typing import List
import onionrutils
import onionrtypes
from onionrcrypto import cryptoutils
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
class PoolFullException(Exception):
"""For when the UploadPool is full.
Raise when a new hash is attempted to be added
"""
class PoolNotReady(Exception):
"""Raise when UploadPool pool access is attempted without it being full."""
class AlreadyInPool(Exception):
"""Raise when a hash already in pool is attempted to be added again."""
class UploadPool:
"""Upload pool for mixing blocks together and delaying uploads."""
def __init__(self, pool_size: int):
"""Create a new pool with a specified max size.
Uses private var and getter to avoid direct adding
"""
self._pool: List[onionrtypes.BlockHash] = []
self._pool_size = pool_size
self.birthday = onionrutils.epoch.get_epoch()
def add_to_pool(self, item: List[onionrtypes.BlockHash]):
"""Add a new hash to the pool. Raise PoolFullException if full."""
if len(self._pool) >= self._pool_size:
raise PoolFullException
if not onionrutils.stringvalidators.validate_hash(item):
raise ValueError
self._pool.append(item)
def get_pool(self) -> List[onionrtypes.BlockHash]:
"""Get the hash pool in secure random order."""
if len(self._pool) != self._pool_size:
raise PoolNotReady
final_pool: List[onionrtypes.BlockHash] = cryptoutils.random_shuffle(list(self._pool))
self._pool.clear()
self.birthday = onionrutils.epoch.get_epoch()
return final_pool

View File

@ -1,9 +1,13 @@
""" """Onionr - Private P2P Communication.
Onionr - Private P2P Communication
Virtual upload "sessions" for blocks Virtual upload "sessions" for blocks
""" """
from __future__ import annotations from typing import Union
from onionrutils import stringvalidators
from onionrutils import bytesconverter
from onionrutils import epoch
from utils import reconstructhash
""" """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
@ -18,21 +22,19 @@ from __future__ import annotations
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
""" """
from typing import Union
from onionrutils import stringvalidators
from onionrutils import bytesconverter
from onionrutils import epoch
from utils import reconstructhash
class UploadSession: class UploadSession:
"""Manages statistics for an Onionr block upload session """Manage statistics for an Onionr block upload session.
accept a block hash (incl. unpadded) as an argument
"""
accepting a block hash (incl. unpadded) as an argument"""
def __init__(self, block_hash: Union[str, bytes]): def __init__(self, block_hash: Union[str, bytes]):
block_hash = bytesconverter.bytes_to_str(block_hash) block_hash = bytesconverter.bytes_to_str(block_hash)
block_hash = reconstructhash.reconstruct_hash(block_hash) block_hash = reconstructhash.reconstruct_hash(block_hash)
if not stringvalidators.validate_hash(block_hash): raise ValueError if not stringvalidators.validate_hash(block_hash):
raise ValueError
self.start_time = epoch.get_epoch() self.start_time = epoch.get_epoch()
self.block_hash = reconstructhash.deconstruct_hash(block_hash) self.block_hash = reconstructhash.deconstruct_hash(block_hash)

View File

@ -52,6 +52,9 @@ DEFAULT_EXPIRE = 2592000
# Metadata header section length limits, in bytes # Metadata header section length limits, in bytes
BLOCK_METADATA_LENGTHS = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'pow': 1000, 'encryptType': 4, 'expire': 14} BLOCK_METADATA_LENGTHS = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'pow': 1000, 'encryptType': 4, 'expire': 14}
# Pool Eligibility Max Age
BLOCK_POOL_MAX_AGE = 300
"""Public key that signs MOTD messages shown in the web UI""" """Public key that signs MOTD messages shown in the web UI"""
MOTD_SIGN_KEY = "TRH763JURNY47QPBTTQ4LLPYCYQK6Q5YA33R6GANKZK5C5DKCIGQ" MOTD_SIGN_KEY = "TRH763JURNY47QPBTTQ4LLPYCYQK6Q5YA33R6GANKZK5C5DKCIGQ"

View File

@ -144,3 +144,4 @@ class PrivateEndpoints:
@private_endpoints_bp.route('/setonboarding', methods=['POST']) @private_endpoints_bp.route('/setonboarding', methods=['POST'])
def set_onboarding(): def set_onboarding():
return Response(config.onboarding.set_config_from_onboarding(request.get_json())) return Response(config.onboarding.set_config_from_onboarding(request.get_json()))

View File

@ -3,10 +3,14 @@
Accept block uploads to the public API server Accept block uploads to the public API server
''' '''
from gevent import threading
import sys import sys
from flask import Response from flask import Response
from flask import abort from flask import abort
from onionrutils import localcommand
from coredb import daemonqueue
from onionrblocks import blockimporter from onionrblocks import blockimporter
import onionrexceptions import onionrexceptions
import logger import logger
@ -31,9 +35,12 @@ def accept_upload(request):
"""Accept uploaded blocks to our public Onionr protocol API server""" """Accept uploaded blocks to our public Onionr protocol API server"""
resp = 'failure' resp = 'failure'
data = request.get_data() data = request.get_data()
b_hash = ''
if sys.getsizeof(data) < 100000000: if sys.getsizeof(data) < 100000000:
try: try:
if blockimporter.import_block_from_data(data): b_hash = blockimporter.import_block_from_data(data)
if b_hash:
daemonqueue.daemon_queue_add('uploadEvent', b_hash)
resp = 'success' resp = 'success'
else: else:
resp = 'failure' resp = 'failure'

View File

@ -195,12 +195,7 @@ def insert_block(data: Union[str, bytes], header: str = 'txt',
retData = False retData = False
else: else:
# Tell the api server through localCommand to wait for the daemon to upload this block to make statistical analysis more difficult # Tell the api server through localCommand to wait for the daemon to upload this block to make statistical analysis more difficult
if not is_offline or localcommand.local_command('/ping', maxWait=10) == 'pong!': coredb.daemonqueue.daemon_queue_add('uploadEvent', retData)
if config.get('general.security_level', 1) == 0:
localcommand.local_command('/waitforshare/' + retData, post=True, maxWait=5)
coredb.daemonqueue.daemon_queue_add('uploadBlock', retData)
else:
pass
coredb.blockmetadb.add.add_to_block_DB(retData, selfInsert=True, dataSaved=True) coredb.blockmetadb.add.add_to_block_DB(retData, selfInsert=True, dataSaved=True)
if expire is None: if expire is None:

View File

@ -7,6 +7,7 @@ import logger
from onionrutils import epoch from onionrutils import epoch
from . import uicheck, inserttest, stresstest from . import uicheck, inserttest, stresstest
from . import ownnode
""" """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
@ -24,7 +25,10 @@ from . import uicheck, inserttest, stresstest
RUN_TESTS = [uicheck.check_ui, RUN_TESTS = [uicheck.check_ui,
inserttest.insert_bin_test, inserttest.insert_bin_test,
stresstest.stress_test_block_insert] ownnode.test_tor_adder,
ownnode.test_own_node,
stresstest.stress_test_block_insert
]
class OnionrRunTestManager: class OnionrRunTestManager:
@ -35,10 +39,11 @@ class OnionrRunTestManager:
def run_tests(self): def run_tests(self):
cur_time = epoch.get_epoch() cur_time = epoch.get_epoch()
logger.info(f"Doing runtime tests at {cur_time}") logger.info(f"Doing runtime tests at {cur_time}")
try: try:
for i in RUN_TESTS: for i in RUN_TESTS:
last = i last = i
i(self) i(self)
logger.info(last.__name__ + " passed") logger.info("[RUNTIME TEST] " + last.__name__ + " passed")
except ValueError: except (ValueError, AttributeError):
logger.error(last.__name__ + ' failed') logger.error(last.__name__ + ' failed')

53
src/runtests/ownnode.py Normal file
View File

@ -0,0 +1,53 @@
"""Onionr - Private P2P Communication.
Test own Onionr node as it is running
"""
import config
from onionrutils import basicrequests
from utils import identifyhome
from utils import gettransports
import logger
from onionrutils import localcommand
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def test_own_node(test_manager):
socks_port = localcommand.local_command('/gettorsocks')
if config.get('general.security_level', 0) > 0:
return
own_tor_address = gettransports.get()[0]
print(socks_port)
if 'this is an onionr node' \
not in basicrequests.do_get_request(own_tor_address,
port=socks_port).lower():
logger.warn('Own node not reachable in test')
raise ValueError
def test_tor_adder(test_manager):
if config.get('general.security_level', 0) > 0:
return
with open(identifyhome.identify_home() + 'hs/hostname', 'r') as hs:
hs = hs.read().strip()
if not hs:
logger.error('No Tor node address created yet')
raise ValueError('No Tor node address created yet')
if hs not in gettransports.get():
print(hs in gettransports.get(), 'meme')
logger.error('gettransports Tor not same as file: %s %s' %
(hs, gettransports.get()))
raise ValueError('gettransports Tor not same as file')

View File

@ -6,6 +6,7 @@ import coredb
from onionrutils import epoch from onionrutils import epoch
def stress_test_block_insert(testmanager): def stress_test_block_insert(testmanager):
return
start = epoch.get_epoch() start = epoch.get_epoch()
count = 100 count = 100
max_insert_speed = 120 max_insert_speed = 120

View File

@ -1,17 +1,20 @@
import filepaths, time from gevent import time
import filepaths
files = [filepaths.tor_hs_address_file] files = [filepaths.tor_hs_address_file]
def get(): def get():
transports = [] transports = []
for file in files: for file in files:
try: try:
with open(file, 'r') as transport_file: with open(file, 'r') as transport_file:
transports.append(transport_file.read().strip()) transports.append(transport_file.read().strip())
except FileNotFoundError: except FileNotFoundError:
pass pass
else:
break
else: else:
time.sleep(1) break
return list(transports) else:
time.sleep(1)
return list(transports)