diff --git a/src/bigbrother/ministry/ofexec.py b/src/bigbrother/ministry/ofexec.py
index 48e28210..e26e7a10 100644
--- a/src/bigbrother/ministry/ofexec.py
+++ b/src/bigbrother/ministry/ofexec.py
@@ -57,6 +57,8 @@ def block_exec(event, info):
]
home = identifyhome.identify_home()
+ code_b64 = base64.b64encode(info[0].co_code).decode()
+
for source in whitelisted_code:
if info[0].co_filename.endswith(source):
return
@@ -64,7 +66,6 @@ def block_exec(event, info):
if home + 'plugins/' in info[0].co_filename:
return
- code_b64 = base64.b64encode(info[0].co_code).decode()
logger.warn('POSSIBLE EXPLOIT DETECTED, SEE LOGS', terminal=True)
logger.warn('POSSIBLE EXPLOIT DETECTED: ' + info[0].co_filename)
logger.warn('Prevented exec/eval. Report this with the sample below')
diff --git a/src/communicatorutils/daemonqueuehandler.py b/src/communicatorutils/daemonqueuehandler.py
index 0936c0df..9cddc793 100755
--- a/src/communicatorutils/daemonqueuehandler.py
+++ b/src/communicatorutils/daemonqueuehandler.py
@@ -1,9 +1,15 @@
-'''
- Onionr - P2P Anonymous Storage Network
+"""Onionr - P2P Anonymous Storage Network.
- Handle daemon queue commands in the communicator
-'''
-'''
+Handle daemon queue commands in the communicator
+"""
+import logger
+from onionrplugins import onionrevents as events
+from onionrutils import localcommand
+from coredb import daemonqueue
+import filepaths
+from . import restarttor
+from communicatorutils.uploadblocks import mixmate
+"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@@ -16,13 +22,9 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see .
-'''
-import logger
-from onionrplugins import onionrevents as events
-from onionrutils import localcommand
-from coredb import daemonqueue
-import filepaths
-from . import restarttor
+"""
+
+
def handle_daemon_commands(comm_inst):
cmd = daemonqueue.daemon_queue()
response = ''
@@ -62,6 +64,13 @@ def handle_daemon_commands(comm_inst):
i.count = (i.frequency - 1)
elif cmd[0] == 'uploadBlock':
comm_inst.blocksToUpload.append(cmd[1])
+ elif cmd[0] == 'uploadEvent':
+ try:
+ mixmate.block_mixer(comm_inst.blocksToUpload, cmd[1])
+ except ValueError:
+ pass
+ else:
+ localcommand.local_command('/waitforshare/' + cmd[1], post=True, maxWait=5)
else:
logger.debug('Received daemon queue command unable to be handled: %s' % (cmd[0],))
diff --git a/src/communicatorutils/downloadblocks/__init__.py b/src/communicatorutils/downloadblocks/__init__.py
index 86c427d1..d532d5c9 100755
--- a/src/communicatorutils/downloadblocks/__init__.py
+++ b/src/communicatorutils/downloadblocks/__init__.py
@@ -1,8 +1,11 @@
-'''
+"""
Onionr - Private P2P Communication
Download blocks using the communicator instance
-'''
+"""
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+ from communicator import OnionrCommunicatorDaemon
import onionrexceptions
import logger
import onionrpeers
@@ -12,12 +15,13 @@ from communicator import onlinepeers
from onionrutils import blockmetadata
from onionrutils import validatemetadata
from coredb import blockmetadb
+from coredb import daemonqueue
import onionrcrypto
import onionrstorage
from onionrblocks import onionrblacklist
from onionrblocks import storagecounter
from . import shoulddownload
-'''
+"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@@ -30,11 +34,11 @@ from . import shoulddownload
You should have received a copy of the GNU General Public License
along with this program. If not, see .
-'''
+"""
def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
- '''Use communicator instance to download blocks in the comms's queue'''
+ """Use communicator instance to download blocks in the comms's queue"""
blacklist = onionrblacklist.OnionrBlackList()
storage_counter = storagecounter.StorageCounter()
LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
@@ -109,6 +113,7 @@ def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
removeFromQueue = False
else:
blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db
+ daemonqueue.daemon_queue_add('uploadEvent', blockHash)
blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
else:
logger.warn('POW failed for block %s.' % (blockHash,))
diff --git a/src/communicatorutils/uploadblocks/__init__.py b/src/communicatorutils/uploadblocks/__init__.py
index baff0ae2..b1420925 100755
--- a/src/communicatorutils/uploadblocks/__init__.py
+++ b/src/communicatorutils/uploadblocks/__init__.py
@@ -27,6 +27,7 @@ from onionrutils import localcommand, stringvalidators, basicrequests
from communicator import onlinepeers
import onionrcrypto
from . import sessionmanager
+from . import mixmate
def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
"""Accepts a communicator instance and uploads blocks from its upload queue"""
diff --git a/src/communicatorutils/uploadblocks/mixmate/__init__.py b/src/communicatorutils/uploadblocks/mixmate/__init__.py
new file mode 100644
index 00000000..ce50be7e
--- /dev/null
+++ b/src/communicatorutils/uploadblocks/mixmate/__init__.py
@@ -0,0 +1,51 @@
+"""Onionr - Private P2P Communication.
+
+Delay block uploads, optionally mixing them together
+"""
+import time
+from typing import List
+
+import onionrtypes
+from onionrblocks import onionrblockapi
+
+from .pool import UploadPool
+from .pool import PoolFullException
+
+from etc import onionrvalues
+
+"""
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see .
+"""
+upload_pool = UploadPool(4)
+
+
+def block_mixer(upload_list: List[onionrtypes.BlockHash],
+ block_to_mix: onionrtypes.BlockHash):
+ """Delay and mix block inserts.
+
+ Take a block list and a received/created block and add it
+ to the said block list
+ """
+ bl = onionrblockapi.Block(block_to_mix)
+ if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE:
+ raise ValueError
+
+ try:
+ # add the new block to pool
+ upload_pool.add_to_pool(block_to_mix)
+ except PoolFullException:
+ # If the pool is full, move into upload queue
+ upload_list.extend(upload_pool.get_pool())
+ # then finally begin new pool with new block
+ upload_pool.add_to_pool(block_to_mix)
diff --git a/src/communicatorutils/uploadblocks/mixmate/pool.py b/src/communicatorutils/uploadblocks/mixmate/pool.py
new file mode 100644
index 00000000..8cc944c1
--- /dev/null
+++ b/src/communicatorutils/uploadblocks/mixmate/pool.py
@@ -0,0 +1,69 @@
+"""Onionr - Private P2P Communication.
+
+Upload pool
+"""
+from typing import List
+
+import onionrutils
+import onionrtypes
+from onionrcrypto import cryptoutils
+"""
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see .
+"""
+
+
+class PoolFullException(Exception):
+ """For when the UploadPool is full.
+
+ Raise when a new hash is attempted to be added
+ """
+
+
+class PoolNotReady(Exception):
+ """Raise when UploadPool pool access is attempted without it being full."""
+
+
+class AlreadyInPool(Exception):
+ """Raise when a hash already in pool is attempted to be added again."""
+
+
+class UploadPool:
+ """Upload pool for mixing blocks together and delaying uploads."""
+
+ def __init__(self, pool_size: int):
+ """Create a new pool with a specified max size.
+
+ Uses private var and getter to avoid direct adding
+ """
+ self._pool: List[onionrtypes.BlockHash] = []
+ self._pool_size = pool_size
+ self.birthday = onionrutils.epoch.get_epoch()
+
+ def add_to_pool(self, item: List[onionrtypes.BlockHash]):
+ """Add a new hash to the pool. Raise PoolFullException if full."""
+ if len(self._pool) >= self._pool_size:
+ raise PoolFullException
+ if not onionrutils.stringvalidators.validate_hash(item):
+ raise ValueError
+ self._pool.append(item)
+
+ def get_pool(self) -> List[onionrtypes.BlockHash]:
+ """Get the hash pool in secure random order."""
+ if len(self._pool) != self._pool_size:
+ raise PoolNotReady
+ final_pool: List[onionrtypes.BlockHash] = cryptoutils.random_shuffle(list(self._pool))
+
+ self._pool.clear()
+ self.birthday = onionrutils.epoch.get_epoch()
+ return final_pool
diff --git a/src/communicatorutils/uploadblocks/session.py b/src/communicatorutils/uploadblocks/session.py
index 6c0b3472..c3fdc2a9 100644
--- a/src/communicatorutils/uploadblocks/session.py
+++ b/src/communicatorutils/uploadblocks/session.py
@@ -1,9 +1,13 @@
-"""
- Onionr - Private P2P Communication
+"""Onionr - Private P2P Communication.
- Virtual upload "sessions" for blocks
+Virtual upload "sessions" for blocks
"""
-from __future__ import annotations
+from typing import Union
+
+from onionrutils import stringvalidators
+from onionrutils import bytesconverter
+from onionrutils import epoch
+from utils import reconstructhash
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -18,21 +22,19 @@ from __future__ import annotations
You should have received a copy of the GNU General Public License
along with this program. If not, see .
"""
-from typing import Union
-from onionrutils import stringvalidators
-from onionrutils import bytesconverter
-from onionrutils import epoch
-from utils import reconstructhash
class UploadSession:
- """Manages statistics for an Onionr block upload session
-
- accepting a block hash (incl. unpadded) as an argument"""
+ """Manage statistics for an Onionr block upload session.
+
+ accept a block hash (incl. unpadded) as an argument
+ """
+
def __init__(self, block_hash: Union[str, bytes]):
block_hash = bytesconverter.bytes_to_str(block_hash)
block_hash = reconstructhash.reconstruct_hash(block_hash)
- if not stringvalidators.validate_hash(block_hash): raise ValueError
+ if not stringvalidators.validate_hash(block_hash):
+ raise ValueError
self.start_time = epoch.get_epoch()
self.block_hash = reconstructhash.deconstruct_hash(block_hash)
@@ -40,15 +42,15 @@ class UploadSession:
self.total_success_count: int = 0
self.peer_fails = {}
self.peer_exists = {}
-
+
def fail_peer(self, peer):
try:
self.peer_fails[peer] += 1
except KeyError:
self.peer_fails[peer] = 0
-
+
def fail(self):
self.total_fail_count += 1
-
+
def success(self):
self.total_success_count += 1
diff --git a/src/etc/onionrvalues.py b/src/etc/onionrvalues.py
index b5cc887d..c4c30b65 100755
--- a/src/etc/onionrvalues.py
+++ b/src/etc/onionrvalues.py
@@ -52,6 +52,9 @@ DEFAULT_EXPIRE = 2592000
# Metadata header section length limits, in bytes
BLOCK_METADATA_LENGTHS = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'pow': 1000, 'encryptType': 4, 'expire': 14}
+# Pool Eligibility Max Age
+BLOCK_POOL_MAX_AGE = 300
+
"""Public key that signs MOTD messages shown in the web UI"""
MOTD_SIGN_KEY = "TRH763JURNY47QPBTTQ4LLPYCYQK6Q5YA33R6GANKZK5C5DKCIGQ"
diff --git a/src/httpapi/miscclientapi/endpoints.py b/src/httpapi/miscclientapi/endpoints.py
index 46459369..e92bfe05 100644
--- a/src/httpapi/miscclientapi/endpoints.py
+++ b/src/httpapi/miscclientapi/endpoints.py
@@ -57,7 +57,7 @@ class PrivateEndpoints:
# Responses from the daemon. TODO: change to direct var access instead of http endpoint
client_api.queueResponse[name] = request.form['data']
return Response('success')
-
+
@private_endpoints_bp.route('/queueResponse/')
def queueResponse(name):
# Fetch a daemon queue response
@@ -72,7 +72,7 @@ class PrivateEndpoints:
return resp, 404
else:
return resp
-
+
@private_endpoints_bp.route('/ping')
def ping():
# Used to check if client api is working
@@ -102,24 +102,24 @@ class PrivateEndpoints:
def restart_clean():
subprocess.Popen([SCRIPT_NAME, 'restart'])
return Response("bye")
-
+
@private_endpoints_bp.route('/gethidden')
def get_hidden_blocks():
return Response('\n'.join(client_api.publicAPI.hideBlocks))
-
+
@private_endpoints_bp.route('/getstats')
def getStats():
# returns node stats
while True:
- try:
+ try:
return Response(client_api._too_many.get(SerializedData).get_stats())
except AttributeError as e:
pass
-
+
@private_endpoints_bp.route('/getuptime')
def showUptime():
return Response(str(client_api.getUptime()))
-
+
@private_endpoints_bp.route('/getActivePubkey')
def getActivePubkey():
return Response(pub_key)
@@ -132,7 +132,7 @@ class PrivateEndpoints:
def getHumanReadable(name):
name = unpaddedbase32.repad(bytesconverter.str_to_bytes(name))
return Response(mnemonickeys.get_human_readable_ID(name))
-
+
@private_endpoints_bp.route('/getBase32FromHumanReadable/')
def get_base32_from_human_readable(words):
return Response(bytesconverter.bytes_to_str(mnemonickeys.get_base32(words)))
@@ -144,3 +144,4 @@ class PrivateEndpoints:
@private_endpoints_bp.route('/setonboarding', methods=['POST'])
def set_onboarding():
return Response(config.onboarding.set_config_from_onboarding(request.get_json()))
+
diff --git a/src/httpapi/miscpublicapi/upload.py b/src/httpapi/miscpublicapi/upload.py
index 9977f7bb..e2c3fe47 100755
--- a/src/httpapi/miscpublicapi/upload.py
+++ b/src/httpapi/miscpublicapi/upload.py
@@ -3,10 +3,14 @@
Accept block uploads to the public API server
'''
+from gevent import threading
+
import sys
from flask import Response
from flask import abort
+from onionrutils import localcommand
+from coredb import daemonqueue
from onionrblocks import blockimporter
import onionrexceptions
import logger
@@ -31,9 +35,12 @@ def accept_upload(request):
"""Accept uploaded blocks to our public Onionr protocol API server"""
resp = 'failure'
data = request.get_data()
+ b_hash = ''
if sys.getsizeof(data) < 100000000:
try:
- if blockimporter.import_block_from_data(data):
+ b_hash = blockimporter.import_block_from_data(data)
+ if b_hash:
+ daemonqueue.daemon_queue_add('uploadEvent', b_hash)
resp = 'success'
else:
resp = 'failure'
diff --git a/src/onionrblocks/insert.py b/src/onionrblocks/insert.py
index 791bdc51..29300b03 100644
--- a/src/onionrblocks/insert.py
+++ b/src/onionrblocks/insert.py
@@ -82,7 +82,7 @@ def insert_block(data: Union[str, bytes], header: str = 'txt',
our_pub_key = bytesconverter.bytes_to_str(crypto.cryptoutils.get_pub_key_from_priv(our_private_key))
use_subprocess = powchoice.use_subprocess(config)
-
+
retData = False
if type(data) is None:
@@ -195,20 +195,15 @@ def insert_block(data: Union[str, bytes], header: str = 'txt',
retData = False
else:
# Tell the api server through localCommand to wait for the daemon to upload this block to make statistical analysis more difficult
- if not is_offline or localcommand.local_command('/ping', maxWait=10) == 'pong!':
- if config.get('general.security_level', 1) == 0:
- localcommand.local_command('/waitforshare/' + retData, post=True, maxWait=5)
- coredb.daemonqueue.daemon_queue_add('uploadBlock', retData)
- else:
- pass
+ coredb.daemonqueue.daemon_queue_add('uploadEvent', retData)
coredb.blockmetadb.add.add_to_block_DB(retData, selfInsert=True, dataSaved=True)
if expire is None:
- coredb.blockmetadb.update_block_info(retData, 'expire',
+ coredb.blockmetadb.update_block_info(retData, 'expire',
createTime + onionrvalues.DEFAULT_EXPIRE)
else:
coredb.blockmetadb.update_block_info(retData, 'expire', expire)
-
+
blockmetadata.process_block_metadata(retData)
if retData != False:
diff --git a/src/runtests/__init__.py b/src/runtests/__init__.py
index 5e66d647..51b29eb6 100644
--- a/src/runtests/__init__.py
+++ b/src/runtests/__init__.py
@@ -7,6 +7,7 @@ import logger
from onionrutils import epoch
from . import uicheck, inserttest, stresstest
+from . import ownnode
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -24,7 +25,10 @@ from . import uicheck, inserttest, stresstest
RUN_TESTS = [uicheck.check_ui,
inserttest.insert_bin_test,
- stresstest.stress_test_block_insert]
+ ownnode.test_tor_adder,
+ ownnode.test_own_node,
+ stresstest.stress_test_block_insert
+ ]
class OnionrRunTestManager:
@@ -35,10 +39,11 @@ class OnionrRunTestManager:
def run_tests(self):
cur_time = epoch.get_epoch()
logger.info(f"Doing runtime tests at {cur_time}")
+
try:
for i in RUN_TESTS:
last = i
i(self)
- logger.info(last.__name__ + " passed")
- except ValueError:
+ logger.info("[RUNTIME TEST] " + last.__name__ + " passed")
+ except (ValueError, AttributeError):
logger.error(last.__name__ + ' failed')
diff --git a/src/runtests/ownnode.py b/src/runtests/ownnode.py
new file mode 100644
index 00000000..ec78b51e
--- /dev/null
+++ b/src/runtests/ownnode.py
@@ -0,0 +1,53 @@
+"""Onionr - Private P2P Communication.
+
+Test own Onionr node as it is running
+"""
+import config
+from onionrutils import basicrequests
+from utils import identifyhome
+from utils import gettransports
+import logger
+from onionrutils import localcommand
+"""
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see .
+"""
+
+
+def test_own_node(test_manager):
+ socks_port = localcommand.local_command('/gettorsocks')
+ if config.get('general.security_level', 0) > 0:
+ return
+ own_tor_address = gettransports.get()[0]
+ print(socks_port)
+ if 'this is an onionr node' \
+ not in basicrequests.do_get_request(own_tor_address,
+ port=socks_port).lower():
+ logger.warn('Own node not reachable in test')
+ raise ValueError
+
+
+def test_tor_adder(test_manager):
+ if config.get('general.security_level', 0) > 0:
+ return
+ with open(identifyhome.identify_home() + 'hs/hostname', 'r') as hs:
+ hs = hs.read().strip()
+ if not hs:
+ logger.error('No Tor node address created yet')
+ raise ValueError('No Tor node address created yet')
+
+ if hs not in gettransports.get():
+ print(hs in gettransports.get(), 'meme')
+ logger.error('gettransports Tor not same as file: %s %s' %
+ (hs, gettransports.get()))
+ raise ValueError('gettransports Tor not same as file')
diff --git a/src/runtests/stresstest.py b/src/runtests/stresstest.py
index a145fd76..35a9967c 100644
--- a/src/runtests/stresstest.py
+++ b/src/runtests/stresstest.py
@@ -6,6 +6,7 @@ import coredb
from onionrutils import epoch
def stress_test_block_insert(testmanager):
+ return
start = epoch.get_epoch()
count = 100
max_insert_speed = 120
diff --git a/src/utils/gettransports.py b/src/utils/gettransports.py
index f5fd823e..5da882fd 100644
--- a/src/utils/gettransports.py
+++ b/src/utils/gettransports.py
@@ -1,17 +1,20 @@
-import filepaths, time
+from gevent import time
+
+import filepaths
files = [filepaths.tor_hs_address_file]
+
def get():
- transports = []
- for file in files:
- try:
- with open(file, 'r') as transport_file:
- transports.append(transport_file.read().strip())
- except FileNotFoundError:
- pass
- else:
- break
+ transports = []
+ for file in files:
+ try:
+ with open(file, 'r') as transport_file:
+ transports.append(transport_file.read().strip())
+ except FileNotFoundError:
+ pass
else:
- time.sleep(1)
- return list(transports)
\ No newline at end of file
+ break
+ else:
+ time.sleep(1)
+ return list(transports)