kev
/
Onionr
1
0
Fork 0
Code Issues 24 Pull Requests Projects Releases 2 Wiki Activity

added mixmate to improve base routing

This commit is contained in:
Kevin Froman 2019-12-22 13:42:10 -06:00
parent f71f2f6246
commit 9329b07e3b
15 changed files with 273 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/bigbrother/ministry/ofexec.py
View File

@ -57,6 +57,8 @@ def block_exec(event, info):
]
home = identifyhome.identify_home()
code_b64 = base64.b64encode(info[0].co_code).decode()
for source in whitelisted_code:
if info[0].co_filename.endswith(source):
return
@ -64,7 +66,6 @@ def block_exec(event, info):
if home + 'plugins/' in info[0].co_filename:
return
code_b64 = base64.b64encode(info[0].co_code).decode()
logger.warn('POSSIBLE EXPLOIT DETECTED, SEE LOGS', terminal=True)
logger.warn('POSSIBLE EXPLOIT DETECTED: ' + info[0].co_filename)
logger.warn('Prevented exec/eval. Report this with the sample below')

33
src/communicatorutils/daemonqueuehandler.py
View File

@ -1,9 +1,15 @@
'''
Onionr - P2P Anonymous Storage Network
"""Onionr - P2P Anonymous Storage Network.
Handle daemon queue commands in the communicator
'''
'''
Handle daemon queue commands in the communicator
"""
import logger
from onionrplugins import onionrevents as events
from onionrutils import localcommand
from coredb import daemonqueue
import filepaths
from . import restarttor
from communicatorutils.uploadblocks import mixmate
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,13 +22,9 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import logger
from onionrplugins import onionrevents as events
from onionrutils import localcommand
from coredb import daemonqueue
import filepaths
from . import restarttor
"""
def handle_daemon_commands(comm_inst):
cmd = daemonqueue.daemon_queue()
response = ''
@ -62,6 +64,13 @@ def handle_daemon_commands(comm_inst):
i.count = (i.frequency - 1)
elif cmd[0] == 'uploadBlock':
comm_inst.blocksToUpload.append(cmd[1])
elif cmd[0] == 'uploadEvent':
try:
mixmate.block_mixer(comm_inst.blocksToUpload, cmd[1])
except ValueError:
pass
else:
localcommand.local_command('/waitforshare/' + cmd[1], post=True, maxWait=5)
else:
logger.debug('Received daemon queue command unable to be handled: %s' % (cmd[0],))

15
src/communicatorutils/downloadblocks/__init__.py
View File

@ -1,8 +1,11 @@
'''
"""
Onionr - Private P2P Communication
Download blocks using the communicator instance
'''
"""
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
import onionrexceptions
import logger
import onionrpeers
@ -12,12 +15,13 @@ from communicator import onlinepeers
from onionrutils import blockmetadata
from onionrutils import validatemetadata
from coredb import blockmetadb
from coredb import daemonqueue
import onionrcrypto
import onionrstorage
from onionrblocks import onionrblacklist
from onionrblocks import storagecounter
from . import shoulddownload
'''
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -30,11 +34,11 @@ from . import shoulddownload
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
"""
def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
'''Use communicator instance to download blocks in the comms's queue'''
"""Use communicator instance to download blocks in the comms's queue"""
blacklist = onionrblacklist.OnionrBlackList()
storage_counter = storagecounter.StorageCounter()
LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
@ -109,6 +113,7 @@ def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
removeFromQueue = False
else:
blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db
daemonqueue.daemon_queue_add('uploadEvent', blockHash)
blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
else:
logger.warn('POW failed for block %s.' % (blockHash,))

1
src/communicatorutils/uploadblocks/__init__.py
View File

@ -27,6 +27,7 @@ from onionrutils import localcommand, stringvalidators, basicrequests
from communicator import onlinepeers
import onionrcrypto
from . import sessionmanager
from . import mixmate
def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
"""Accepts a communicator instance and uploads blocks from its upload queue"""

51
src/communicatorutils/uploadblocks/mixmate/__init__.py Normal file
View File

@ -0,0 +1,51 @@
"""Onionr - Private P2P Communication.
Delay block uploads, optionally mixing them together
"""
import time
from typing import List
import onionrtypes
from onionrblocks import onionrblockapi
from .pool import UploadPool
from .pool import PoolFullException
from etc import onionrvalues
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
upload_pool = UploadPool(4)
def block_mixer(upload_list: List[onionrtypes.BlockHash],
block_to_mix: onionrtypes.BlockHash):
"""Delay and mix block inserts.
Take a block list and a received/created block and add it
to the said block list
"""
bl = onionrblockapi.Block(block_to_mix)
if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE:
raise ValueError
try:
# add the new block to pool
upload_pool.add_to_pool(block_to_mix)
except PoolFullException:
# If the pool is full, move into upload queue
upload_list.extend(upload_pool.get_pool())
# then finally begin new pool with new block
upload_pool.add_to_pool(block_to_mix)

69
src/communicatorutils/uploadblocks/mixmate/pool.py Normal file
View File

@ -0,0 +1,69 @@
"""Onionr - Private P2P Communication.
Upload pool
"""
from typing import List
import onionrutils
import onionrtypes
from onionrcrypto import cryptoutils
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
class PoolFullException(Exception):
"""For when the UploadPool is full.
Raise when a new hash is attempted to be added
"""
class PoolNotReady(Exception):
"""Raise when UploadPool pool access is attempted without it being full."""
class AlreadyInPool(Exception):
"""Raise when a hash already in pool is attempted to be added again."""
class UploadPool:
"""Upload pool for mixing blocks together and delaying uploads."""
def __init__(self, pool_size: int):
"""Create a new pool with a specified max size.
Uses private var and getter to avoid direct adding
"""
self._pool: List[onionrtypes.BlockHash] = []
self._pool_size = pool_size
self.birthday = onionrutils.epoch.get_epoch()
def add_to_pool(self, item: List[onionrtypes.BlockHash]):
"""Add a new hash to the pool. Raise PoolFullException if full."""
if len(self._pool) >= self._pool_size:
raise PoolFullException
if not onionrutils.stringvalidators.validate_hash(item):
raise ValueError
self._pool.append(item)
def get_pool(self) -> List[onionrtypes.BlockHash]:
"""Get the hash pool in secure random order."""
if len(self._pool) != self._pool_size:
raise PoolNotReady
final_pool: List[onionrtypes.BlockHash] = cryptoutils.random_shuffle(list(self._pool))
self._pool.clear()
self.birthday = onionrutils.epoch.get_epoch()
return final_pool

34
src/communicatorutils/uploadblocks/session.py
View File

@ -1,9 +1,13 @@
"""
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
Virtual upload "sessions" for blocks
Virtual upload "sessions" for blocks
"""
from __future__ import annotations
from typing import Union
from onionrutils import stringvalidators
from onionrutils import bytesconverter
from onionrutils import epoch
from utils import reconstructhash
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -18,21 +22,19 @@ from __future__ import annotations
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from typing import Union
from onionrutils import stringvalidators
from onionrutils import bytesconverter
from onionrutils import epoch
from utils import reconstructhash
class UploadSession:
"""Manages statistics for an Onionr block upload session
accepting a block hash (incl. unpadded) as an argument"""
"""Manage statistics for an Onionr block upload session.
accept a block hash (incl. unpadded) as an argument
"""
def __init__(self, block_hash: Union[str, bytes]):
block_hash = bytesconverter.bytes_to_str(block_hash)
block_hash = reconstructhash.reconstruct_hash(block_hash)
if not stringvalidators.validate_hash(block_hash): raise ValueError
if not stringvalidators.validate_hash(block_hash):
raise ValueError
self.start_time = epoch.get_epoch()
self.block_hash = reconstructhash.deconstruct_hash(block_hash)
@ -40,15 +42,15 @@ class UploadSession:
self.total_success_count: int = 0
self.peer_fails = {}
self.peer_exists = {}
def fail_peer(self, peer):
try:
self.peer_fails[peer] += 1
except KeyError:
self.peer_fails[peer] = 0
def fail(self):
self.total_fail_count += 1
def success(self):
self.total_success_count += 1

3
src/etc/onionrvalues.py
View File

@ -52,6 +52,9 @@ DEFAULT_EXPIRE = 2592000
# Metadata header section length limits, in bytes
BLOCK_METADATA_LENGTHS = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'pow': 1000, 'encryptType': 4, 'expire': 14}
# Pool Eligibility Max Age
BLOCK_POOL_MAX_AGE = 300
"""Public key that signs MOTD messages shown in the web UI"""
MOTD_SIGN_KEY = "TRH763JURNY47QPBTTQ4LLPYCYQK6Q5YA33R6GANKZK5C5DKCIGQ"

17
src/httpapi/miscclientapi/endpoints.py
View File

@ -57,7 +57,7 @@ class PrivateEndpoints:
# Responses from the daemon. TODO: change to direct var access instead of http endpoint
client_api.queueResponse[name] = request.form['data']
return Response('success')
@private_endpoints_bp.route('/queueResponse/<name>')
def queueResponse(name):
# Fetch a daemon queue response
@ -72,7 +72,7 @@ class PrivateEndpoints:
return resp, 404
else:
return resp
@private_endpoints_bp.route('/ping')
def ping():
# Used to check if client api is working
@ -102,24 +102,24 @@ class PrivateEndpoints:
def restart_clean():
subprocess.Popen([SCRIPT_NAME, 'restart'])
return Response("bye")
@private_endpoints_bp.route('/gethidden')
def get_hidden_blocks():
return Response('\n'.join(client_api.publicAPI.hideBlocks))
@private_endpoints_bp.route('/getstats')
def getStats():
# returns node stats
while True:
try:
try:
return Response(client_api._too_many.get(SerializedData).get_stats())
except AttributeError as e:
pass
@private_endpoints_bp.route('/getuptime')
def showUptime():
return Response(str(client_api.getUptime()))
@private_endpoints_bp.route('/getActivePubkey')
def getActivePubkey():
return Response(pub_key)
@ -132,7 +132,7 @@ class PrivateEndpoints:
def getHumanReadable(name):
name = unpaddedbase32.repad(bytesconverter.str_to_bytes(name))
return Response(mnemonickeys.get_human_readable_ID(name))
@private_endpoints_bp.route('/getBase32FromHumanReadable/<words>')
def get_base32_from_human_readable(words):
return Response(bytesconverter.bytes_to_str(mnemonickeys.get_base32(words)))
@ -144,3 +144,4 @@ class PrivateEndpoints:
@private_endpoints_bp.route('/setonboarding', methods=['POST'])
def set_onboarding():
return Response(config.onboarding.set_config_from_onboarding(request.get_json()))

9
src/httpapi/miscpublicapi/upload.py
View File

@ -3,10 +3,14 @@
Accept block uploads to the public API server
'''
from gevent import threading
import sys
from flask import Response
from flask import abort
from onionrutils import localcommand
from coredb import daemonqueue
from onionrblocks import blockimporter
import onionrexceptions
import logger
@ -31,9 +35,12 @@ def accept_upload(request):
"""Accept uploaded blocks to our public Onionr protocol API server"""
resp = 'failure'
data = request.get_data()
b_hash = ''
if sys.getsizeof(data) < 100000000:
try:
if blockimporter.import_block_from_data(data):
b_hash = blockimporter.import_block_from_data(data)
if b_hash:
daemonqueue.daemon_queue_add('uploadEvent', b_hash)
resp = 'success'
else:
resp = 'failure'

13
src/onionrblocks/insert.py
View File

@ -82,7 +82,7 @@ def insert_block(data: Union[str, bytes], header: str = 'txt',
our_pub_key = bytesconverter.bytes_to_str(crypto.cryptoutils.get_pub_key_from_priv(our_private_key))
use_subprocess = powchoice.use_subprocess(config)
retData = False
if type(data) is None:
@ -195,20 +195,15 @@ def insert_block(data: Union[str, bytes], header: str = 'txt',
retData = False
else:
# Tell the api server through localCommand to wait for the daemon to upload this block to make statistical analysis more difficult
if not is_offline or localcommand.local_command('/ping', maxWait=10) == 'pong!':
if config.get('general.security_level', 1) == 0:
localcommand.local_command('/waitforshare/' + retData, post=True, maxWait=5)
coredb.daemonqueue.daemon_queue_add('uploadBlock', retData)
else:
pass
coredb.daemonqueue.daemon_queue_add('uploadEvent', retData)
coredb.blockmetadb.add.add_to_block_DB(retData, selfInsert=True, dataSaved=True)
if expire is None:
coredb.blockmetadb.update_block_info(retData, 'expire',
coredb.blockmetadb.update_block_info(retData, 'expire',
createTime + onionrvalues.DEFAULT_EXPIRE)
else:
coredb.blockmetadb.update_block_info(retData, 'expire', expire)
blockmetadata.process_block_metadata(retData)
if retData != False:

11
src/runtests/__init__.py
View File

@ -7,6 +7,7 @@ import logger
from onionrutils import epoch
from . import uicheck, inserttest, stresstest
from . import ownnode
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -24,7 +25,10 @@ from . import uicheck, inserttest, stresstest
RUN_TESTS = [uicheck.check_ui,
inserttest.insert_bin_test,
stresstest.stress_test_block_insert]
ownnode.test_tor_adder,
ownnode.test_own_node,
stresstest.stress_test_block_insert
]
class OnionrRunTestManager:
@ -35,10 +39,11 @@ class OnionrRunTestManager:
def run_tests(self):
cur_time = epoch.get_epoch()
logger.info(f"Doing runtime tests at {cur_time}")
try:
for i in RUN_TESTS:
last = i
i(self)
logger.info(last.__name__ + " passed")
except ValueError:
logger.info("[RUNTIME TEST] " + last.__name__ + " passed")
except (ValueError, AttributeError):
logger.error(last.__name__ + ' failed')

53
src/runtests/ownnode.py Normal file
View File

@ -0,0 +1,53 @@
"""Onionr - Private P2P Communication.
Test own Onionr node as it is running
"""
import config
from onionrutils import basicrequests
from utils import identifyhome
from utils import gettransports
import logger
from onionrutils import localcommand
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def test_own_node(test_manager):
socks_port = localcommand.local_command('/gettorsocks')
if config.get('general.security_level', 0) > 0:
return
own_tor_address = gettransports.get()[0]
print(socks_port)
if 'this is an onionr node' \
not in basicrequests.do_get_request(own_tor_address,
port=socks_port).lower():
logger.warn('Own node not reachable in test')
raise ValueError
def test_tor_adder(test_manager):
if config.get('general.security_level', 0) > 0:
return
with open(identifyhome.identify_home() + 'hs/hostname', 'r') as hs:
hs = hs.read().strip()
if not hs:
logger.error('No Tor node address created yet')
raise ValueError('No Tor node address created yet')
if hs not in gettransports.get():
print(hs in gettransports.get(), 'meme')
logger.error('gettransports Tor not same as file: %s %s' %
(hs, gettransports.get()))
raise ValueError('gettransports Tor not same as file')

1
src/runtests/stresstest.py
View File

@ -6,6 +6,7 @@ import coredb
from onionrutils import epoch
def stress_test_block_insert(testmanager):
return
start = epoch.get_epoch()
count = 100
max_insert_speed = 120

27
src/utils/gettransports.py
View File

@ -1,17 +1,20 @@
import filepaths, time
from gevent import time
import filepaths
files = [filepaths.tor_hs_address_file]
def get():
transports = []
for file in files:
try:
with open(file, 'r') as transport_file:
transports.append(transport_file.read().strip())
except FileNotFoundError:
pass
else:
break
transports = []
for file in files:
try:
with open(file, 'r') as transport_file:
transports.append(transport_file.read().strip())
except FileNotFoundError:
pass
else:
time.sleep(1)
return list(transports)
break
else:
time.sleep(1)
return list(transports)