Pārlūkot izejas kodu

bumped network version and main version, lots of test work and some stability improvements

tags/4.0.0
Kevin Froman pirms 9 mēnešiem
vecāks
revīzija
572e29f5d5
24 mainītis faili ar 243 papildinājumiem un 33 dzēšanām
  1. +11
    -3
      run_tests.sh
  2. +1
    -0
      scripts/disable-dev-config.py
  3. +1
    -0
      scripts/enable-dev-config.py
  4. +5
    -1
      src/__init__.py
  5. +1
    -1
      src/bigbrother/ministry/ofexec.py
  6. +4
    -2
      src/etc/onionrvalues.py
  7. +2
    -0
      src/filepaths/__init__.py
  8. +3
    -1
      src/logger/settings.py
  9. +3
    -0
      src/onionrcommands/exportblocks.py
  10. +5
    -6
      src/onionrsetup/dbcreator.py
  11. +11
    -3
      src/onionrsetup/defaultpluginsetup.py
  12. +5
    -3
      src/onionrstorage/__init__.py
  13. +22
    -4
      src/onionrstorage/setdata.py
  14. +3
    -1
      src/runtests/__init__.py
  15. +61
    -0
      src/runtests/clearnettor.py
  16. +1
    -1
      src/runtests/inserttest.py
  17. +4
    -2
      src/runtests/ownnode.py
  18. +1
    -1
      static-data/default-plugins/flow/info.json
  19. +21
    -2
      static-data/default-plugins/flow/main.py
  20. +3
    -1
      static-data/default_config.json
  21. +18
    -0
      tests/integration-tests/details-test.py
  22. +40
    -0
      tests/integration-tests/export-test.py
  23. +16
    -0
      tests/integration-tests/no-command-run.py
  24. +1
    -1
      tests/runtime-result.txt

+ 11
- 3
run_tests.sh Parādīt failu

@@ -3,14 +3,22 @@ rm -rf testdata;
mkdir testdata;
ran=0

SECONDS=0 ;
SECONDS=0 ;
close () {
rm -rf testdata;
exit 10;
}

for f in tests/*.py; do
python3 "$f" || close # if needed
python3 "$f" || close # if needed
let "ran++"
done
echo "ran $ran test files successfully in $SECONDS seconds"
echo "ran $ran unittests. Unittest Time: $SECONDS"
ran=0;

for f in tests/integration-tests/*.py; do
python3 "$f" || close # if needed
let "ran++"
done
echo "ran $ran integration test tests."
echo "total test time $SECONDS"

+ 1
- 0
scripts/disable-dev-config.py Parādīt failu

@@ -17,6 +17,7 @@ conf['general']['random_bind_ip'] = True
conf['onboarding']['done'] = False
conf['general']['minimum_block_pow'] = 5
conf['general']['minimum_send_pow'] = 5
conf['log']['file']['remove_on_exit'] = True

json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4)


+ 1
- 0
scripts/enable-dev-config.py Parādīt failu

@@ -18,6 +18,7 @@ conf['general']['random_bind_ip'] = False
conf['onboarding']['done'] = True
conf['general']['minimum_block_pow'] = 4
conf['general']['minimum_send_pow'] = 4
conf['log']['file']['remove_on_exit'] = False

json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4)


+ 5
- 1
src/__init__.py Parādīt failu

@@ -68,6 +68,7 @@ setup.setup_config()

import config # noqa
from utils import identifyhome # noqa
import filepaths # noqa

if config.get('advanced.security_auditing', True):
try:
@@ -93,7 +94,10 @@ if ran_as_script:

# If the setting is there, shred log file on exit
if config.get('log.file.remove_on_exit', True):
nuke.clean(config.get_config_file())
try:
nuke.clean(filepaths.log_file)
except FileNotFoundError:
pass

# Cleanup standard out/err because Python refuses to do it itsself
try:


+ 1
- 1
src/bigbrother/ministry/ofexec.py Parādīt failu

@@ -72,7 +72,7 @@ def block_exec(event, info):
if info[0].co_filename.endswith(source):
return

if home + 'plugins/' in info[0].co_filename:
if 'plugins/' in info[0].co_filename:
return

logger.warn('POSSIBLE EXPLOIT DETECTED, SEE LOGS', terminal=True)


+ 4
- 2
src/etc/onionrvalues.py Parādīt failu

@@ -23,10 +23,10 @@ import filepaths
DENIABLE_PEER_ADDRESS = "OVPCZLOXD6DC5JHX4EQ3PSOGAZ3T24F75HQLIUZSDSMYPEOXCPFA"
PASSWORD_LENGTH = 25
ONIONR_TAGLINE = 'Private P2P Communication - GPLv3 - https://Onionr.net'
ONIONR_VERSION = '2.0.0'
ONIONR_VERSION = '3.0.0'
ONIONR_VERSION_CODENAME = 'Genesis'
ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
API_VERSION = '0' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
API_VERSION = '1' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
MIN_PY_VERSION = 7 # min version of 7 so we can take advantage of non-cyclic type hints
DEVELOPMENT_MODE = False
"""limit type length for a block (soft enforced, ignored if invalid but block still stored)."""
@@ -48,6 +48,8 @@ WSGI_SERVER_REQUEST_TIMEOUT_SECS = 120

MAX_NEW_PEER_QUEUE = 1000

BLOCK_EXPORT_FILE_EXT = '.dat'

# Begin OnionrValues migrated values

"""30 days is plenty of time for someone to decide to renew a block"""


+ 2
- 0
src/filepaths/__init__.py Parādīt failu

@@ -32,3 +32,5 @@ data_nonce_file = home + 'block-nonces.dat'
keys_file = home + 'keys.txt'

onboarding_mark_file = home + 'onboarding-completed'

log_file = home + 'onionr.log'

+ 3
- 1
src/logger/settings.py Parādīt failu

@@ -19,6 +19,7 @@
'''
import os
from utils import identifyhome
import filepaths

data_home = os.environ.get('ONIONR_LOG_DIR', identifyhome.identify_home())
# Use the bitwise operators to merge these settings
@@ -39,7 +40,8 @@ MAX_LOG_FILE_LINES = 10000

_type = OUTPUT_TO_CONSOLE | USE_ANSI # the default settings for logging
_level = LEVEL_DEBUG # the lowest level to log
_outputfile = '%s/onionr.log' % (data_home,) # the file to log to
# the file to log to
_outputfile = filepaths.log_file

def set_settings(type):
'''


+ 3
- 0
src/onionrcommands/exportblocks.py Parādīt failu

@@ -9,6 +9,9 @@ import onionrstorage
from utils import createdirs
from onionrutils import stringvalidators
import filepaths

import os
from coredb import blockmetadb
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by


+ 5
- 6
src/onionrsetup/dbcreator.py Parādīt failu

@@ -1,9 +1,8 @@
'''
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.

DBCreator, creates sqlite3 databases used by Onionr
'''
'''
DBCreator, creates sqlite3 databases used by Onionr
"""
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@@ -16,7 +15,7 @@

You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
"""
import sqlite3, os
from coredb import dbfiles
import filepaths


+ 11
- 3
src/onionrsetup/defaultpluginsetup.py Parādīt failu

@@ -22,13 +22,19 @@ import os, shutil
import onionrplugins as plugins
import logger
import filepaths
from utils.readstatic import get_static_dir

def setup_default_plugins():
# Copy default plugins into plugins folder
if not os.path.exists(plugins.get_plugins_folder()):
if os.path.exists('../static-data/default-plugins/'):
names = [f for f in os.listdir("../static-data/default-plugins/")]
shutil.copytree('../static-data/default-plugins/', plugins.get_plugins_folder())
if os.path.exists(get_static_dir() + '/default-plugins/'):
names = [f for f in os.listdir(get_static_dir() + '/default-plugins/')]
try:
shutil.copytree(
get_static_dir() + '/default-plugins/',
plugins.get_plugins_folder())
except FileExistsError:
pass

# Enable plugins
for name in names:
@@ -39,6 +45,8 @@ def setup_default_plugins():
if not os.path.exists(plugins.get_plugin_data_folder(name)):
try:
os.mkdir(plugins.get_plugin_data_folder(name))
except FileExistsError:
pass
except Exception as e:
#logger.warn('Error enabling plugin: ' + str(e), terminal=True)
plugins.disable(name, stop_event = False)

+ 5
- 3
src/onionrstorage/__init__.py Parādīt failu

@@ -77,7 +77,7 @@ def store(data, blockHash=''):
raise ValueError('Hash specified does not meet internal hash check')
else:
blockHash = ourHash
if DB_ENTRY_SIZE_LIMIT >= sys.getsizeof(data):
_dbInsert(blockHash, data)
else:
@@ -86,21 +86,23 @@ def store(data, blockHash=''):


def getData(bHash):

if not stringvalidators.validate_hash(bHash): raise ValueError

bHash = bytesconverter.bytes_to_str(bHash)
bHash = bHash.strip()
# First check DB for data entry by hash
# if no entry, check disk
# If no entry in either, raise an exception
retData = None
fileLocation = '%s/%s.dat' % (filepaths.block_data_location, bHash)
not_found_msg = "Flock data not found for: "
not_found_msg = "Block data not found for: "
if os.path.exists(fileLocation):
with open(fileLocation, 'rb') as block:
retData = block.read()
else:
retData = _dbFetch(bHash)

if retData is None:
raise onionrexceptions.NoDataAvailable(not_found_msg + str(bHash))
return retData

+ 22
- 4
src/onionrstorage/setdata.py Parādīt failu

@@ -1,13 +1,31 @@
import sys, sqlite3
"""Onionr - Private P2P Communication.

Test Onionr as it is running
"""
import sys
import sqlite3

import onionrstorage, onionrexceptions, onionrcrypto as crypto
import filepaths
from onionrblocks import storagecounter
from coredb import dbfiles
from onionrutils import blockmetadata, bytesconverter
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def set_data(data)->str:
'''
Set the data assciated with a hash
'''
"""Set the data assciated with a hash."""
storage_counter = storagecounter.StorageCounter()
data = data
dataSize = sys.getsizeof(data)


+ 3
- 1
src/runtests/__init__.py Parādīt failu

@@ -11,6 +11,7 @@ from . import uicheck, inserttest, stresstest
from . import ownnode
from .webpasstest import webpass_test
from .osver import test_os_ver_endpoint
from .clearnettor import test_clearnet_tor_request
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -32,7 +33,8 @@ RUN_TESTS = [uicheck.check_ui,
ownnode.test_own_node,
stresstest.stress_test_block_insert,
webpass_test,
test_os_ver_endpoint
test_os_ver_endpoint,
test_clearnet_tor_request
]

SUCCESS_FILE = os.path.dirname(os.path.realpath(__file__)) + '/../../tests/runtime-result.txt'


+ 61
- 0
src/runtests/clearnettor.py Parādīt failu

@@ -0,0 +1,61 @@
"""Onionr - Private P2P Communication.

Ensure that clearnet cannot be reached
"""
from threading import Thread

from onionrutils.basicrequests import do_get_request
from onionrutils import localcommand
import logger
import config

"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""


def test_clearnet_tor_request(testmanager):
"""Ensure that Tor cannot request clearnet address.

Does not run if Tor is being reused
"""
config.reload()
leak_result = ""

if config.get('tor.use_existing_tor', False):
logger.warn(
"Can't ensure Tor reqs to clearnet won't happen when reusing Tor")
return


socks_port = localcommand.local_command('/gettorsocks')

# Don't worry, this request isn't meant to go through,
# but if it did it would be through Tor

try:
leak_result: str = do_get_request(
'https://onionr.net/404',
port=socks_port, ignoreAPI=True).lower()
except AttributeError:
leak_result = ""
except Exception as e:
logger.warn(str(e))
try:
if 'not found' in leak_result:
logger.error('Tor was able to request a clearnet site')
raise ValueError('Tor was able to request a clearnet site')
except TypeError:
pass

+ 1
- 1
src/runtests/inserttest.py Parādīt failu

@@ -11,7 +11,7 @@ def _check_remote_node(testmanager):

def insert_bin_test(testmanager):
data = os.urandom(32)
b_hash = onionrblocks.insert(data, )
b_hash = onionrblocks.insert(data)

if not b_hash in coredb.blockmetadb.get_block_list():
logger.error(str(b_hash) + 'is not in bl')


+ 4
- 2
src/runtests/ownnode.py Parādīt failu

@@ -25,14 +25,16 @@ from onionrutils import localcommand


def test_own_node(test_manager):
return
socks_port = localcommand.local_command('/gettorsocks')
if config.get('general.security_level', 0) > 0:
return
own_tor_address = gettransports.get()[0]
if 'this is an onionr node' \
not in basicrequests.do_get_request('http://' + own_tor_address,
port=socks_port, ignoreAPI=True).lower():
logger.warn('Own node not reachable in test')
port=socks_port,
ignoreAPI=True).lower():
logger.warn(f'Own node not reachable in test {own_tor_address}')
raise ValueError




+ 1
- 1
static-data/default-plugins/flow/info.json Parādīt failu

@@ -1,4 +1,4 @@
{ "name": "flow",
"version": "0.0.1",
"version": "0.1.0",
"author": "onionr"
}

+ 21
- 2
static-data/default-plugins/flow/main.py Parādīt failu

@@ -39,8 +39,9 @@ flask_blueprint = flowapi.flask_blueprint
security_whitelist = ['staticfiles.boardContent', 'staticfiles.board']

plugin_name = 'flow'
PLUGIN_VERSION = '0.0.1'
PLUGIN_VERSION = '0.1.0'

EXPIRE_TIME = 43200

class OnionrFlow:
def __init__(self):
@@ -75,7 +76,7 @@ class OnionrFlow:
else:
if message == "q":
self.flowRunning = False
expireTime = epoch.get_epoch() + 43200
expireTime = epoch.get_epoch() + EXPIRE_TIME
if len(message) > 0:
logger.info('Inserting message as block...', terminal=True)
onionrblocks.insert(message, header='brd',
@@ -118,6 +119,24 @@ def on_flow_cmd(api, data=None):
OnionrFlow().start()


def on_flowsend_cmd(api, data=None):
err_msg = "Second arg is board name, third is quoted message"
try:
sys.argv[2]
except IndexError:
logger.error(err_msg, terminal=True)
try:
sys.argv[3]
except IndexError:
logger.error(err_msg, terminal=True)

bl = onionrblocks.insert(sys.argv[3], header='brd',
expire=(EXPIRE_TIME + epoch.get_epoch()),
meta={'ch': sys.argv[2]})
print(bl)



def on_softreset(api, data=None):
try:
os.remove(identifyhome.identify_home() + '/board-index.cache.json')


+ 3
- 1
static-data/default_config.json Parādīt failu

@@ -46,7 +46,9 @@
"minimum_score": -100
},
"plugins": {
"disabled": [],
"disabled": [
"chat"
],
"enabled": []
},
"timers": {


+ 18
- 0
tests/integration-tests/details-test.py Parādīt failu

@@ -0,0 +1,18 @@
import sys
import os
from subprocess import Popen, PIPE
import uuid

TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
os.environ["ONIONR_HOME"] = TEST_DIR

print(f'running integration test for {__file__}')

with Popen(['./onionr.sh', 'details'], stdout=PIPE) as onionr_proc:
output = onionr_proc.stdout.read().decode()
if onionr_proc.returncode != 0:
raise ValueError('Raised non zero exit ' + str(onionr_proc.returncode))

for word in ['Node', 'Human-readable']:
if word not in output:
raise ValueError(word + " not in " + output)

+ 40
- 0
tests/integration-tests/export-test.py Parādīt failu

@@ -0,0 +1,40 @@
from unittest.mock import patch
import sys, os
sys.path.append(".")
sys.path.append("src/")
import unittest, uuid
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
from utils import createdirs
from onionrcommands import parser
import onionrsetup as setup
from netcontroller.torcontrol import customtorrc
from utils import createdirs
from onionrsetup import setup_config, setup_default_plugins
from coredb import blockmetadb
from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT

createdirs.create_dirs()
setup_config()
setup_default_plugins()
import config
from filepaths import export_location

class OnionrTests(unittest.TestCase):
def test_export(self):
testargs = ["onionr.py", "flowsend", "tests", "hello"]
with patch.object(sys, 'argv', testargs):
parser.register()
bl = blockmetadb.get_block_list()[0]
testargs = ["onionr.py", "export-block", bl]
with patch.object(sys, 'argv', testargs):
parser.register()

with open(export_location + '/' + bl + BLOCK_EXPORT_FILE_EXT, 'rb') as f:

if b'hello' not in f.read():
raise ValueError('No exported block')


unittest.main()

+ 16
- 0
tests/integration-tests/no-command-run.py Parādīt failu

@@ -0,0 +1,16 @@
import sys
import os
from subprocess import Popen, PIPE
import uuid

TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
os.environ["ONIONR_HOME"] = TEST_DIR

print(f'running integration test for {__file__}')
with Popen(['./onionr.sh'], stdout=PIPE) as onionr_proc:
output = onionr_proc.stdout.read().decode()
if onionr_proc.returncode != 0:
raise ValueError('Raised non zero exit ' + str(onionr_proc.returncode))

if output != '':
raise ValueError('No command run returned non-blank output')

+ 1
- 1
tests/runtime-result.txt Parādīt failu

@@ -1 +1 @@
1580971981
1581152327

Notiek ielāde…
Atcelt
Saglabāt