bumped network version and main version, lots of test work and some stability improvements

This commit is contained in:
Kevin Froman 2020-02-08 03:07:07 -06:00
vanhempi e77d422fc2
commit 572e29f5d5
24 muutettua tiedostoa jossa 243 lisäystä ja 33 poistoa

Näytä tiedosto

@ -3,14 +3,22 @@ rm -rf testdata;
mkdir testdata;
ran=0
SECONDS=0 ;
SECONDS=0 ;
close () {
rm -rf testdata;
exit 10;
}
for f in tests/*.py; do
python3 "$f" || close # if needed
python3 "$f" || close # if needed
let "ran++"
done
echo "ran $ran test files successfully in $SECONDS seconds"
echo "ran $ran unittests. Unittest Time: $SECONDS"
ran=0;
for f in tests/integration-tests/*.py; do
python3 "$f" || close # if needed
let "ran++"
done
echo "ran $ran integration test tests."
echo "total test time $SECONDS"

Näytä tiedosto

@ -17,6 +17,7 @@ conf['general']['random_bind_ip'] = True
conf['onboarding']['done'] = False
conf['general']['minimum_block_pow'] = 5
conf['general']['minimum_send_pow'] = 5
conf['log']['file']['remove_on_exit'] = True
json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4)

Näytä tiedosto

@ -18,6 +18,7 @@ conf['general']['random_bind_ip'] = False
conf['onboarding']['done'] = True
conf['general']['minimum_block_pow'] = 4
conf['general']['minimum_send_pow'] = 4
conf['log']['file']['remove_on_exit'] = False
json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4)

Näytä tiedosto

@ -68,6 +68,7 @@ setup.setup_config()
import config # noqa
from utils import identifyhome # noqa
import filepaths # noqa
if config.get('advanced.security_auditing', True):
try:
@ -93,7 +94,10 @@ if ran_as_script:
# If the setting is there, shred log file on exit
if config.get('log.file.remove_on_exit', True):
nuke.clean(config.get_config_file())
try:
nuke.clean(filepaths.log_file)
except FileNotFoundError:
pass
# Cleanup standard out/err because Python refuses to do it itsself
try:

Näytä tiedosto

@ -72,7 +72,7 @@ def block_exec(event, info):
if info[0].co_filename.endswith(source):
return
if home + 'plugins/' in info[0].co_filename:
if 'plugins/' in info[0].co_filename:
return
logger.warn('POSSIBLE EXPLOIT DETECTED, SEE LOGS', terminal=True)

Näytä tiedosto

@ -23,10 +23,10 @@ import filepaths
DENIABLE_PEER_ADDRESS = "OVPCZLOXD6DC5JHX4EQ3PSOGAZ3T24F75HQLIUZSDSMYPEOXCPFA"
PASSWORD_LENGTH = 25
ONIONR_TAGLINE = 'Private P2P Communication - GPLv3 - https://Onionr.net'
ONIONR_VERSION = '2.0.0'
ONIONR_VERSION = '3.0.0'
ONIONR_VERSION_CODENAME = 'Genesis'
ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
API_VERSION = '0' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
API_VERSION = '1' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
MIN_PY_VERSION = 7 # min version of 7 so we can take advantage of non-cyclic type hints
DEVELOPMENT_MODE = False
"""limit type length for a block (soft enforced, ignored if invalid but block still stored)."""
@ -48,6 +48,8 @@ WSGI_SERVER_REQUEST_TIMEOUT_SECS = 120
MAX_NEW_PEER_QUEUE = 1000
BLOCK_EXPORT_FILE_EXT = '.dat'
# Begin OnionrValues migrated values
"""30 days is plenty of time for someone to decide to renew a block"""

Näytä tiedosto

@ -32,3 +32,5 @@ data_nonce_file = home + 'block-nonces.dat'
keys_file = home + 'keys.txt'
onboarding_mark_file = home + 'onboarding-completed'
log_file = home + 'onionr.log'

Näytä tiedosto

@ -19,6 +19,7 @@
'''
import os
from utils import identifyhome
import filepaths
data_home = os.environ.get('ONIONR_LOG_DIR', identifyhome.identify_home())
# Use the bitwise operators to merge these settings
@ -39,7 +40,8 @@ MAX_LOG_FILE_LINES = 10000
_type = OUTPUT_TO_CONSOLE | USE_ANSI # the default settings for logging
_level = LEVEL_DEBUG # the lowest level to log
_outputfile = '%s/onionr.log' % (data_home,) # the file to log to
# the file to log to
_outputfile = filepaths.log_file
def set_settings(type):
'''

Näytä tiedosto

@ -9,6 +9,9 @@ import onionrstorage
from utils import createdirs
from onionrutils import stringvalidators
import filepaths
import os
from coredb import blockmetadb
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by

Näytä tiedosto

@ -1,9 +1,8 @@
'''
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
DBCreator, creates sqlite3 databases used by Onionr
'''
'''
DBCreator, creates sqlite3 databases used by Onionr
"""
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,7 +15,7 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
"""
import sqlite3, os
from coredb import dbfiles
import filepaths

Näytä tiedosto

@ -22,13 +22,19 @@ import os, shutil
import onionrplugins as plugins
import logger
import filepaths
from utils.readstatic import get_static_dir
def setup_default_plugins():
# Copy default plugins into plugins folder
if not os.path.exists(plugins.get_plugins_folder()):
if os.path.exists('../static-data/default-plugins/'):
names = [f for f in os.listdir("../static-data/default-plugins/")]
shutil.copytree('../static-data/default-plugins/', plugins.get_plugins_folder())
if os.path.exists(get_static_dir() + '/default-plugins/'):
names = [f for f in os.listdir(get_static_dir() + '/default-plugins/')]
try:
shutil.copytree(
get_static_dir() + '/default-plugins/',
plugins.get_plugins_folder())
except FileExistsError:
pass
# Enable plugins
for name in names:
@ -39,6 +45,8 @@ def setup_default_plugins():
if not os.path.exists(plugins.get_plugin_data_folder(name)):
try:
os.mkdir(plugins.get_plugin_data_folder(name))
except FileExistsError:
pass
except Exception as e:
#logger.warn('Error enabling plugin: ' + str(e), terminal=True)
plugins.disable(name, stop_event = False)

Näytä tiedosto

@ -77,7 +77,7 @@ def store(data, blockHash=''):
raise ValueError('Hash specified does not meet internal hash check')
else:
blockHash = ourHash
if DB_ENTRY_SIZE_LIMIT >= sys.getsizeof(data):
_dbInsert(blockHash, data)
else:
@ -86,21 +86,23 @@ def store(data, blockHash=''):
def getData(bHash):
if not stringvalidators.validate_hash(bHash): raise ValueError
bHash = bytesconverter.bytes_to_str(bHash)
bHash = bHash.strip()
# First check DB for data entry by hash
# if no entry, check disk
# If no entry in either, raise an exception
retData = None
fileLocation = '%s/%s.dat' % (filepaths.block_data_location, bHash)
not_found_msg = "Flock data not found for: "
not_found_msg = "Block data not found for: "
if os.path.exists(fileLocation):
with open(fileLocation, 'rb') as block:
retData = block.read()
else:
retData = _dbFetch(bHash)
if retData is None:
raise onionrexceptions.NoDataAvailable(not_found_msg + str(bHash))
return retData

Näytä tiedosto

@ -1,13 +1,31 @@
import sys, sqlite3
"""Onionr - Private P2P Communication.
Test Onionr as it is running
"""
import sys
import sqlite3
import onionrstorage, onionrexceptions, onionrcrypto as crypto
import filepaths
from onionrblocks import storagecounter
from coredb import dbfiles
from onionrutils import blockmetadata, bytesconverter
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def set_data(data)->str:
'''
Set the data assciated with a hash
'''
"""Set the data assciated with a hash."""
storage_counter = storagecounter.StorageCounter()
data = data
dataSize = sys.getsizeof(data)

Näytä tiedosto

@ -11,6 +11,7 @@ from . import uicheck, inserttest, stresstest
from . import ownnode
from .webpasstest import webpass_test
from .osver import test_os_ver_endpoint
from .clearnettor import test_clearnet_tor_request
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -32,7 +33,8 @@ RUN_TESTS = [uicheck.check_ui,
ownnode.test_own_node,
stresstest.stress_test_block_insert,
webpass_test,
test_os_ver_endpoint
test_os_ver_endpoint,
test_clearnet_tor_request
]
SUCCESS_FILE = os.path.dirname(os.path.realpath(__file__)) + '/../../tests/runtime-result.txt'

Näytä tiedosto

@ -0,0 +1,61 @@
"""Onionr - Private P2P Communication.
Ensure that clearnet cannot be reached
"""
from threading import Thread
from onionrutils.basicrequests import do_get_request
from onionrutils import localcommand
import logger
import config
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def test_clearnet_tor_request(testmanager):
"""Ensure that Tor cannot request clearnet address.
Does not run if Tor is being reused
"""
config.reload()
leak_result = ""
if config.get('tor.use_existing_tor', False):
logger.warn(
"Can't ensure Tor reqs to clearnet won't happen when reusing Tor")
return
socks_port = localcommand.local_command('/gettorsocks')
# Don't worry, this request isn't meant to go through,
# but if it did it would be through Tor
try:
leak_result: str = do_get_request(
'https://onionr.net/404',
port=socks_port, ignoreAPI=True).lower()
except AttributeError:
leak_result = ""
except Exception as e:
logger.warn(str(e))
try:
if 'not found' in leak_result:
logger.error('Tor was able to request a clearnet site')
raise ValueError('Tor was able to request a clearnet site')
except TypeError:
pass

Näytä tiedosto

@ -11,7 +11,7 @@ def _check_remote_node(testmanager):
def insert_bin_test(testmanager):
data = os.urandom(32)
b_hash = onionrblocks.insert(data, )
b_hash = onionrblocks.insert(data)
if not b_hash in coredb.blockmetadb.get_block_list():
logger.error(str(b_hash) + 'is not in bl')

Näytä tiedosto

@ -25,14 +25,16 @@ from onionrutils import localcommand
def test_own_node(test_manager):
return
socks_port = localcommand.local_command('/gettorsocks')
if config.get('general.security_level', 0) > 0:
return
own_tor_address = gettransports.get()[0]
if 'this is an onionr node' \
not in basicrequests.do_get_request('http://' + own_tor_address,
port=socks_port, ignoreAPI=True).lower():
logger.warn('Own node not reachable in test')
port=socks_port,
ignoreAPI=True).lower():
logger.warn(f'Own node not reachable in test {own_tor_address}')
raise ValueError

Näytä tiedosto

@ -1,4 +1,4 @@
{ "name": "flow",
"version": "0.0.1",
"version": "0.1.0",
"author": "onionr"
}

Näytä tiedosto

@ -39,8 +39,9 @@ flask_blueprint = flowapi.flask_blueprint
security_whitelist = ['staticfiles.boardContent', 'staticfiles.board']
plugin_name = 'flow'
PLUGIN_VERSION = '0.0.1'
PLUGIN_VERSION = '0.1.0'
EXPIRE_TIME = 43200
class OnionrFlow:
def __init__(self):
@ -75,7 +76,7 @@ class OnionrFlow:
else:
if message == "q":
self.flowRunning = False
expireTime = epoch.get_epoch() + 43200
expireTime = epoch.get_epoch() + EXPIRE_TIME
if len(message) > 0:
logger.info('Inserting message as block...', terminal=True)
onionrblocks.insert(message, header='brd',
@ -118,6 +119,24 @@ def on_flow_cmd(api, data=None):
OnionrFlow().start()
def on_flowsend_cmd(api, data=None):
err_msg = "Second arg is board name, third is quoted message"
try:
sys.argv[2]
except IndexError:
logger.error(err_msg, terminal=True)
try:
sys.argv[3]
except IndexError:
logger.error(err_msg, terminal=True)
bl = onionrblocks.insert(sys.argv[3], header='brd',
expire=(EXPIRE_TIME + epoch.get_epoch()),
meta={'ch': sys.argv[2]})
print(bl)
def on_softreset(api, data=None):
try:
os.remove(identifyhome.identify_home() + '/board-index.cache.json')

Näytä tiedosto

@ -46,7 +46,9 @@
"minimum_score": -100
},
"plugins": {
"disabled": [],
"disabled": [
"chat"
],
"enabled": []
},
"timers": {

Näytä tiedosto

@ -0,0 +1,18 @@
import sys
import os
from subprocess import Popen, PIPE
import uuid
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
os.environ["ONIONR_HOME"] = TEST_DIR
print(f'running integration test for {__file__}')
with Popen(['./onionr.sh', 'details'], stdout=PIPE) as onionr_proc:
output = onionr_proc.stdout.read().decode()
if onionr_proc.returncode != 0:
raise ValueError('Raised non zero exit ' + str(onionr_proc.returncode))
for word in ['Node', 'Human-readable']:
if word not in output:
raise ValueError(word + " not in " + output)

Näytä tiedosto

@ -0,0 +1,40 @@
from unittest.mock import patch
import sys, os
sys.path.append(".")
sys.path.append("src/")
import unittest, uuid
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
from utils import createdirs
from onionrcommands import parser
import onionrsetup as setup
from netcontroller.torcontrol import customtorrc
from utils import createdirs
from onionrsetup import setup_config, setup_default_plugins
from coredb import blockmetadb
from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
createdirs.create_dirs()
setup_config()
setup_default_plugins()
import config
from filepaths import export_location
class OnionrTests(unittest.TestCase):
def test_export(self):
testargs = ["onionr.py", "flowsend", "tests", "hello"]
with patch.object(sys, 'argv', testargs):
parser.register()
bl = blockmetadb.get_block_list()[0]
testargs = ["onionr.py", "export-block", bl]
with patch.object(sys, 'argv', testargs):
parser.register()
with open(export_location + '/' + bl + BLOCK_EXPORT_FILE_EXT, 'rb') as f:
if b'hello' not in f.read():
raise ValueError('No exported block')
unittest.main()

Näytä tiedosto

@ -0,0 +1,16 @@
import sys
import os
from subprocess import Popen, PIPE
import uuid
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
os.environ["ONIONR_HOME"] = TEST_DIR
print(f'running integration test for {__file__}')
with Popen(['./onionr.sh'], stdout=PIPE) as onionr_proc:
output = onionr_proc.stdout.read().decode()
if onionr_proc.returncode != 0:
raise ValueError('Raised non zero exit ' + str(onionr_proc.returncode))
if output != '':
raise ValueError('No command run returned non-blank output')

Näytä tiedosto

@ -1 +1 @@
1580971981
1581152327