From 70a26598fb851400016366c483deab98697b05e1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 28 Mar 2020 05:41:11 +0000 Subject: [PATCH 001/105] Bump deadsimplekv from 0.3.0 to 0.3.1 Bumps [deadsimplekv](https://github.com/beardog108/deadsimplekv) from 0.3.0 to 0.3.1. - [Release notes](https://github.com/beardog108/deadsimplekv/releases) - [Changelog](https://github.com/beardog108/DeadSimpleKV/blob/master/CHANGELOG.md) - [Commits](https://github.com/beardog108/deadsimplekv/commits) Signed-off-by: dependabot-preview[bot] --- requirements.in | 2 +- requirements.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.in b/requirements.in index d7143af7..dcbdfff6 100644 --- a/requirements.in +++ b/requirements.in @@ -5,7 +5,7 @@ gevent==1.4.0 Flask==1.1.1 PySocks==1.7.1 stem==1.8.0 -deadsimplekv==0.3.0 +deadsimplekv==0.3.1 unpaddedbase32==0.2.0 streamedrequests==1.0.0 jinja2==2.11.1 diff --git a/requirements.txt b/requirements.txt index c062e6a2..8dcaaf28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,9 +46,9 @@ click==7.0 \ --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \ --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \ # via flask -deadsimplekv==0.3.0 \ - --hash=sha256:66a96e648f1306433628d1b7e2cc2d828d9aa488ad95f2b641a15cd5f89814fa \ - --hash=sha256:ac9cf72f8a8e26933d572b7fb607cee84604aab5d9f3a4fd6bf6046818c5d4fc \ +deadsimplekv==0.3.1 \ + --hash=sha256:85aaacba793178018210728d104c95ddec7755b419b41621310e109fc24cff46 \ + --hash=sha256:a61e3f783f5698543d1011ec0ce72b252e950ca2af15e565b43ccc7886213311 \ # via -r requirements.in filenuke==0.0.0 \ --hash=sha256:147011c0125121469cae0a8a7f4df399f470e54aa29a08f2d2c099bf0118dcee \ From 8804c1322586c9a9a4b240844450b602e49147b6 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 3 Apr 2020 03:28:48 -0500 Subject: [PATCH 002/105] * refactored subprocess and made it 20%+ faster by using ujson + added watchdog dep for sneakernet + added ujson dep for faster json esp in pow --- requirements.in | 3 ++- requirements.txt | 9 +++++++++ src/onionrproofs/subprocesspow.py | 15 +++++++-------- tests/runtime-result.txt | 2 +- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/requirements.in b/requirements.in index 899df4a8..8d04fa62 100644 --- a/requirements.in +++ b/requirements.in @@ -13,4 +13,5 @@ niceware==0.2.1 psutil==5.7.0 filenuke==0.0.0 mimcvdf==1.0.0 -watchdog==0.10.2 \ No newline at end of file +watchdog==0.10.2 +ujson==2.0.3 diff --git a/requirements.txt b/requirements.txt index 897b0881..31c9b8ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -216,6 +216,15 @@ streamedrequests==1.0.0 \ toomanyobjs==1.1.0 \ --hash=sha256:99e27468f9dad19127be9e2fb086b42acd69aed9ad7e63cef74d6e4389be0534 \ # via -r requirements.in +ujson==2.0.3 \ + --hash=sha256:0c23f21e8d2b60efab57bc6ce9d1fb7c4e96f4bfefbf5a6043a3f3309e2a738a \ + --hash=sha256:2ab88e330405315512afe9276f29a60e9b3439187b273665630a57ed7fe1d936 \ + --hash=sha256:3d1f4705a4ec1e48ff383a4d92299d8ec25e9a8158bcea619912440948117634 \ + --hash=sha256:6217c63a36e9b26e9271e686d212397ce7fb04c07d85509dd4e2ed73493320f8 \ + --hash=sha256:7ae13733d9467d16ccac2f38212cdee841b49ae927085c533425be9076b0bc9d \ + --hash=sha256:bd2deffc983827510e5145fb66e4cc0f577480c62fe0b4882139f8f7d27ae9a3 \ + --hash=sha256:c8369ef49169804944e920c427e350182e33756422b69989c55608fc28bebf98 \ + # via -r requirements.in unpaddedbase32==0.2.0 \ --hash=sha256:4aacee75f8fd6c8cf129842ecba45ca59c11bfb13dae19d86f32b48fa3715403 \ --hash=sha256:b7b780c31d27d55e66abf6c221216a35690ee8892c2daacff7f2528e229bd9c3 \ diff --git a/src/onionrproofs/subprocesspow.py b/src/onionrproofs/subprocesspow.py index f3470699..e4852e8b 100755 --- a/src/onionrproofs/subprocesspow.py +++ b/src/onionrproofs/subprocesspow.py @@ -1,22 +1,20 @@ #!/usr/bin/env python3 -""" - Onionr - Private P2P Communication +"""Onionr - Private P2P Communication. - Multiprocess proof of work +Multiprocess proof of work """ import os from multiprocessing import Pipe, Process import threading import time -import json -import secrets +import onionrproofs + +import ujson as json import logger -import onionrproofs import onionrcrypto as crypto from onionrutils import bytesconverter -from .blocknoncestart import BLOCK_NONCE_START_INT """ This program is free software: you can redistribute it and/or modify @@ -110,11 +108,12 @@ class SubprocessPOW: def do_pow(self, pipe): """find partial hash colision generating nonce for a block""" - nonce = -secrets.randbelow(10**10) + nonce = 0 data = self.data metadata = self.metadata puzzle = self.puzzle difficulty = self.difficulty + while True: # Break if shutdown received if pipe.poll() and pipe.recv() == 'shutdown': diff --git a/tests/runtime-result.txt b/tests/runtime-result.txt index 52b64ce6..8dc1d799 100644 --- a/tests/runtime-result.txt +++ b/tests/runtime-result.txt @@ -1 +1 @@ -1585619396 \ No newline at end of file +1585902463 \ No newline at end of file From 46ce27cbe8e5131725c34084e3b7f6c9729f3581 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 3 Apr 2020 03:33:05 -0500 Subject: [PATCH 003/105] fixed get_lan_ips test --- tests/test_get_lan_ips.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_get_lan_ips.py b/tests/test_get_lan_ips.py index 1d514bd4..cf6c0dbf 100644 --- a/tests/test_get_lan_ips.py +++ b/tests/test_get_lan_ips.py @@ -8,6 +8,12 @@ TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' print("Test directory:", TEST_DIR) os.environ["ONIONR_HOME"] = TEST_DIR import unittest +from utils import identifyhome, createdirs +from onionrsetup import setup_config + +createdirs.create_dirs() +setup_config() + from lan.getip import lan_ips, best_ip From b23343e7cbdbe053880e7dcb831b59dc94b0c338 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 3 Apr 2020 03:51:31 -0500 Subject: [PATCH 004/105] use ujson in serializeddata --- src/onionrstatistics/serializeddata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/onionrstatistics/serializeddata.py b/src/onionrstatistics/serializeddata.py index 37fba2e4..359d35ea 100755 --- a/src/onionrstatistics/serializeddata.py +++ b/src/onionrstatistics/serializeddata.py @@ -2,10 +2,10 @@ Serialize various node information """ -import json from gevent import sleep from psutil import Process, WINDOWS +import ujson as json from coredb import blockmetadb from utils.sizeutils import size, human_size From 5be4df1276fb52e4da5d5b6619739b0e4df7711f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 3 Apr 2020 08:40:21 +0000 Subject: [PATCH 005/105] Bump helium from 3.0.1 to 3.0.2 Bumps [helium](https://github.com/mherrmann/helium) from 3.0.1 to 3.0.2. - [Release notes](https://github.com/mherrmann/helium/releases) - [Commits](https://github.com/mherrmann/helium/compare/v3.0.1...v3.0.2) Signed-off-by: dependabot-preview[bot] --- requirements-dev.in | 2 +- requirements-dev.txt | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/requirements-dev.in b/requirements-dev.in index 52978779..2b526326 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,3 +1,3 @@ pdoc3==0.7.5 pip-tools==4.5.1 -helium==3.0.1 +helium==3.0.2 diff --git a/requirements-dev.txt b/requirements-dev.txt index cbbad0e6..791cfc96 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,8 +8,9 @@ click==7.0 \ --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \ --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \ # via pip-tools -helium==3.0.1 \ - --hash=sha256:a7be1ad48702a38e11e9ee9b262459cde3c986d2dcbd79fcf8d6f5b54c5f5d9e +helium==3.0.2 \ + --hash=sha256:0413dffb50a5e8e6c05bb3fc6bc458992a72d310b2132451968caee93b022048 \ + # via -r requirements-dev.in mako==1.1.1 \ --hash=sha256:2984a6733e1d472796ceef37ad48c26f4a984bb18119bb2dbc37a44d8f6e75a4 \ # via pdoc3 @@ -52,10 +53,6 @@ markupsafe==1.1.1 \ --hash=sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7 \ --hash=sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be \ # via mako -selenium==3.141.0 \ - --hash=sha256:2d7131d7bc5a5b99a2d9b04aaf2612c411b03b8ca1b1ee8d3de5845a9be2cb3c \ - --hash=sha256:deaf32b60ad91a4611b98d8002757f29e6f2c2d5fcaf202e1c9ad06d6772300d \ - # via helium pdoc3==0.7.5 \ --hash=sha256:ebca75b7fcf23f3b4320abe23339834d3f08c28517718e9d29e555fc38eeb33c \ # via -r requirements-dev.in @@ -63,6 +60,10 @@ pip-tools==4.5.1 \ --hash=sha256:693f30e451875796b1b25203247f0b4cf48a4c4a5ab7341f4f33ffd498cdcc98 \ --hash=sha256:be9c796aa88b2eec5cabf1323ba1cb60a08212b84bfb75b8b4037a8ef8cb8cb6 \ # via -r requirements-dev.in +selenium==3.141.0 \ + --hash=sha256:2d7131d7bc5a5b99a2d9b04aaf2612c411b03b8ca1b1ee8d3de5845a9be2cb3c \ + --hash=sha256:deaf32b60ad91a4611b98d8002757f29e6f2c2d5fcaf202e1c9ad06d6772300d \ + # via helium six==1.14.0 \ --hash=sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a \ --hash=sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c \ From dee5d8ceedfe4e3aa6109e7ed08ea3a2976c7035 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 3 Apr 2020 04:02:36 -0500 Subject: [PATCH 006/105] make config use ujson made httpapi use ujson made onionrblocks mod use ujson made setup config use ujson made devreporting use ujson made transports/tor use ujson made contactmanager use ujson --- src/config/__init__.py | 5 +- src/httpapi/apiutils/getblockdata.py | 3 +- src/httpapi/configapi/__init__.py | 3 +- src/httpapi/friendsapi/__init__.py | 21 ++- src/httpapi/insertblock.py | 2 +- src/onionrblocks/insert/main.py | 2 +- src/onionrblocks/onionrblockapi.py | 134 ++++++++++-------- src/onionrsetup/setupconfig.py | 29 ++-- src/onionrstatistics/devreporting/__init__.py | 21 ++- .../transports/tor/__init__.py | 3 +- src/onionrusers/contactmanager.py | 37 ++--- 11 files changed, 149 insertions(+), 111 deletions(-) diff --git a/src/config/__init__.py b/src/config/__init__.py index 8f2ede60..2762a522 100755 --- a/src/config/__init__.py +++ b/src/config/__init__.py @@ -2,7 +2,10 @@ This file deals with configuration management. """ -import os, json, logger +import os + +import ujson as json +import logger import filepaths from . import onboarding diff --git a/src/httpapi/apiutils/getblockdata.py b/src/httpapi/apiutils/getblockdata.py index ad420ae1..fc9a615d 100644 --- a/src/httpapi/apiutils/getblockdata.py +++ b/src/httpapi/apiutils/getblockdata.py @@ -1,4 +1,5 @@ -import json +import ujson as json + from onionrblocks import onionrblockapi from onionrutils import bytesconverter, stringvalidators import onionrexceptions diff --git a/src/httpapi/configapi/__init__.py b/src/httpapi/configapi/__init__.py index e1226a3a..ea8fbc85 100755 --- a/src/httpapi/configapi/__init__.py +++ b/src/httpapi/configapi/__init__.py @@ -17,8 +17,9 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . """ -import json +import ujson as json from flask import Blueprint, request, Response, abort + import config, onionrutils from onionrutils.bytesconverter import bytes_to_str diff --git a/src/httpapi/friendsapi/__init__.py b/src/httpapi/friendsapi/__init__.py index 4447cc9c..03bc552c 100755 --- a/src/httpapi/friendsapi/__init__.py +++ b/src/httpapi/friendsapi/__init__.py @@ -1,9 +1,13 @@ -''' - Onionr - Private P2P Communication +"""Onionr - Private P2P Communication. - This file creates http endpoints for friend management -''' -''' +This file creates http endpoints for friend management +""" +import ujson as json + +from onionrusers import contactmanager +from flask import Blueprint, Response, request, abort, redirect +from coredb import keydb +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -16,12 +20,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' -import json -from onionrusers import contactmanager -from flask import Blueprint, Response, request, abort, redirect -from coredb import keydb - +""" friends = Blueprint('friends', __name__) @friends.route('/friends/list') diff --git a/src/httpapi/insertblock.py b/src/httpapi/insertblock.py index c245ea87..ce25b968 100644 --- a/src/httpapi/insertblock.py +++ b/src/httpapi/insertblock.py @@ -2,7 +2,7 @@ Create blocks with the client api server """ -import json +import ujson as json import threading from flask import Blueprint, Response, request, g diff --git a/src/onionrblocks/insert/main.py b/src/onionrblocks/insert/main.py index 10906d5e..948731af 100644 --- a/src/onionrblocks/insert/main.py +++ b/src/onionrblocks/insert/main.py @@ -18,7 +18,7 @@ along with this program. If not, see . """ from typing import Union -import json +import ujson as json from gevent import spawn diff --git a/src/onionrblocks/onionrblockapi.py b/src/onionrblocks/onionrblockapi.py index a03b5788..009a65ef 100755 --- a/src/onionrblocks/onionrblockapi.py +++ b/src/onionrblocks/onionrblockapi.py @@ -1,9 +1,26 @@ -''' - Onionr - P2P Anonymous Storage Network +"""Onionr - P2P Anonymous Storage Network. - This file contains the OnionrBlocks class which is a class for working with Onionr blocks -''' -''' +OnionrBlocks class for abstraction of blocks +""" +import binascii +import os +import datetime +import onionrstorage + +import unpaddedbase32 +import ujson as json +import nacl.exceptions + +import logger +import onionrexceptions +from onionrusers import onionrusers +from onionrutils import stringvalidators, epoch +from coredb import blockmetadb +from onionrutils import bytesconverter +from onionrstorage import removeblock +import onionrblocks +from onionrcrypto import encryption, cryptoutils as cryptoutils, signing +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -16,18 +33,9 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' -import unpaddedbase32 -import binascii -import logger, config, onionrexceptions, nacl.exceptions -import json, os, sys, datetime, base64, onionrstorage -from onionrusers import onionrusers -from onionrutils import stringvalidators, epoch -from coredb import blockmetadb -from onionrutils import bytesconverter -from onionrstorage import removeblock -import onionrblocks -from onionrcrypto import encryption, cryptoutils as cryptoutils, signing +""" + + class Block: blockCacheOrder = list() # NEVER write your own code that writes to this! blockCache = dict() # should never be accessed directly, look at Block.getCache() @@ -62,9 +70,9 @@ class Block: self.update() def decrypt(self, encodedData = True): - ''' + """ Decrypt a block, loading decrypted data into their vars - ''' + """ if self.decrypted: return True @@ -126,9 +134,9 @@ class Block: return retData def verifySig(self): - ''' + """ Verify if a block's signature is signed by its claimed signer - ''' + """ if self.signer is None: return False if signing.ed_verify(data=self.signedData, key=self.signer, sig=self.signature, encodedData=True): @@ -138,7 +146,7 @@ class Block: return self.validSig def update(self, data = None, file = None): - ''' + """ Loads data from a block in to the current object. Inputs: @@ -151,7 +159,7 @@ class Block: Outputs: - (bool): indicates whether or not the operation was successful - ''' + """ try: # import from string blockdata = data @@ -205,12 +213,12 @@ class Block: return False def delete(self): - ''' + """ Deletes the block's file and records, if they exist Outputs: - (bool): whether or not the operation was successful - ''' + """ if self.exists(): try: @@ -224,7 +232,7 @@ class Block: return False def save(self, sign = False, recreate = True): - ''' + """ Saves a block to file and imports it into Onionr Inputs: @@ -233,7 +241,7 @@ class Block: Outputs: - (bool): whether or not the operation was successful - ''' + """ try: if self.isValid() is True: @@ -253,45 +261,45 @@ class Block: # getters def getExpire(self): - ''' + """ Returns the expire time for a block Outputs: - (int): the expire time for a block, or None - ''' + """ return self.expire def getHash(self): - ''' + """ Returns the hash of the block if saved to file Outputs: - (str): the hash of the block, or None - ''' + """ return self.hash def getType(self): - ''' + """ Returns the type of the block Outputs: - (str): the type of the block - ''' + """ return self.btype def getRaw(self): - ''' + """ Returns the raw contents of the block, if saved to file Outputs: - (bytes): the raw contents of the block, or None - ''' + """ return self.raw def getHeader(self, key = None, default = None): - ''' + """ Returns the header information Inputs: @@ -299,7 +307,7 @@ class Block: Outputs: - (dict/str): either the whole header as a dict, or one value - ''' + """ if not key is None: if key in self.getHeader(): @@ -308,7 +316,7 @@ class Block: return self.bheader def getMetadata(self, key = None, default = None): - ''' + """ Returns the metadata information Inputs: @@ -316,7 +324,7 @@ class Block: Outputs: - (dict/str): either the whole metadata as a dict, or one value - ''' + """ if not key is None: if key in self.getMetadata(): @@ -325,77 +333,77 @@ class Block: return self.bmetadata def getContent(self): - ''' + """ Returns the contents of the block Outputs: - (str): the contents of the block - ''' + """ return self.bcontent def getDate(self): - ''' + """ Returns the date that the block was received, if loaded from file Outputs: - (datetime): the date that the block was received - ''' + """ return self.date def getBlockFile(self): - ''' + """ Returns the location of the block file if it is saved Outputs: - (str): the location of the block file, or None - ''' + """ return self.blockFile def isValid(self): - ''' + """ Checks if the block is valid Outputs: - (bool): whether or not the block is valid - ''' + """ return self.valid def isSigned(self): - ''' + """ Checks if the block was signed Outputs: - (bool): whether or not the block is signed - ''' + """ return self.signed def getSignature(self): - ''' + """ Returns the base64-encoded signature Outputs: - (str): the signature, or None - ''' + """ return self.signature def getSignedData(self): - ''' + """ Returns the data that was signed Outputs: - (str): the data that was signed, or None - ''' + """ return self.signedData def isSigner(self, signer, encodedData = True): - ''' + """ Checks if the block was signed by the signer inputted Inputs: @@ -404,7 +412,7 @@ class Block: Outputs: - (bool): whether or not the signer of the block is the signer inputted - ''' + """ signer = unpaddedbase32.repad(bytesconverter.str_to_bytes(signer)) try: if (not self.isSigned()) or (not stringvalidators.validate_pub_key(signer)): @@ -417,7 +425,7 @@ class Block: # setters def setType(self, btype): - ''' + """ Sets the type of the block Inputs: @@ -425,13 +433,13 @@ class Block: Outputs: - (Block): the Block instance - ''' + """ self.btype = btype return self def setMetadata(self, key, val): - ''' + """ Sets a custom metadata value Metadata should not store block-specific data structures. @@ -442,13 +450,13 @@ class Block: Outputs: - (Block): the Block instance - ''' + """ self.bmetadata[key] = val return self def setContent(self, bcontent): - ''' + """ Sets the contents of the block Inputs: @@ -456,14 +464,14 @@ class Block: Outputs: - (Block): the Block instance - ''' + """ self.bcontent = str(bcontent) return self # static functions def exists(bHash): - ''' + """ Checks if a block is saved to file or not Inputs: @@ -473,7 +481,7 @@ class Block: Outputs: - (bool): whether or not the block file exists - ''' + """ # no input data? scrap it. if bHash is None: diff --git a/src/onionrsetup/setupconfig.py b/src/onionrsetup/setupconfig.py index 12024459..703e1853 100755 --- a/src/onionrsetup/setupconfig.py +++ b/src/onionrsetup/setupconfig.py @@ -1,9 +1,19 @@ -''' - Onionr - Private P2P Communication +"""Onionr - Private P2P Communication. - Initialize Onionr configuration -''' -''' +Initialize Onionr configuration +""" +import os +import base64 + +import ujson as json + +import config +import logger +import netcontroller +from etc import onionrvalues +from logger.settings import * +from utils import readstatic +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -16,15 +26,10 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' -import os, json, base64 -import config, logger, netcontroller -from etc import onionrvalues -from logger.settings import * -from utils import readstatic +""" + def setup_config(): - if not os.path.exists(config._configfile): # this is the default config, it will be overwritten if a config file already exists. Else, it saves it conf_data = readstatic.read_static('default_config.json', ret_bin=False) diff --git a/src/onionrstatistics/devreporting/__init__.py b/src/onionrstatistics/devreporting/__init__.py index e9047832..6a9bb422 100644 --- a/src/onionrstatistics/devreporting/__init__.py +++ b/src/onionrstatistics/devreporting/__init__.py @@ -1,10 +1,29 @@ +"""Onionr - Private P2P Communication. + +Dev utility to profile an Onionr subnetwork. +""" +import ujson as json + import config from utils.bettersleep import better_sleep from utils.gettransports import get as get_transports from onionrutils import basicrequests from onionrutils import epoch +""" + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" -import json def statistics_reporter(shared_state): server = config.get('statistics.server', '') diff --git a/src/onionrstatistics/transports/tor/__init__.py b/src/onionrstatistics/transports/tor/__init__.py index 577d1620..cb98f91a 100644 --- a/src/onionrstatistics/transports/tor/__init__.py +++ b/src/onionrstatistics/transports/tor/__init__.py @@ -2,8 +2,7 @@ """ -import json - +import ujson as json from stem import CircStatus from netcontroller.torcontrol.torcontroller import get_controller diff --git a/src/onionrusers/contactmanager.py b/src/onionrusers/contactmanager.py index d5910b51..c4630f03 100755 --- a/src/onionrusers/contactmanager.py +++ b/src/onionrusers/contactmanager.py @@ -1,9 +1,19 @@ -''' - Onionr - Private P2P Communication +"""Onionr - Private P2P Communication. - Sets more abstract information related to a peer. Can be thought of as traditional 'contact' system -''' -''' +Set more abstract information related to a peer. +Can be thought of as traditional 'contact' system +""" +import os + +import ujson as json +import unpaddedbase32 + +import onionrexceptions +from onionrusers import onionrusers +from onionrutils import bytesconverter, epoch +from utils import identifyhome +from onionrutils import mnemonickeys +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -16,16 +26,9 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' -import os, json -import unpaddedbase32 -import niceware +""" + -import onionrexceptions -from onionrusers import onionrusers -from onionrutils import bytesconverter, epoch -from utils import identifyhome -from onionrutils import mnemonickeys class ContactManager(onionrusers.OnionrUser): def __init__(self, publicKey, saveUser=False, recordExpireSeconds=5): try: @@ -43,10 +46,10 @@ class ContactManager(onionrusers.OnionrUser): self.recordExpire = recordExpireSeconds self.data = self._loadData() self.deleted = False - + if not os.path.exists(self.dataDir): os.mkdir(self.dataDir) - + def _writeData(self): data = json.dumps(self.data) with open(self.dataFile, 'w') as dataFile: @@ -68,7 +71,7 @@ class ContactManager(onionrusers.OnionrUser): if autoWrite: self._writeData() return - + def get_info(self, key, forceReload=False): if self.deleted: raise onionrexceptions.ContactDeleted From eea8b39b0f61b1506853d7195463894c5adf48b6 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 3 Apr 2020 04:17:45 -0500 Subject: [PATCH 007/105] refactored onionrusers formatting and json --- src/onionrusers/onionrusers.py | 42 ++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/onionrusers/onionrusers.py b/src/onionrusers/onionrusers.py index 153be743..50b1c00c 100755 --- a/src/onionrusers/onionrusers.py +++ b/src/onionrusers/onionrusers.py @@ -1,9 +1,20 @@ -''' - Onionr - Private P2P Communication +"""Onionr - Private P2P Communication. - Contains abstractions for interacting with users of Onionr -''' -''' +Contains abstractions for interacting with users of Onionr +""" +import sqlite3 +import time + +import onionrexceptions +from onionrutils import stringvalidators, bytesconverter, epoch + +import unpaddedbase32 +import nacl.exceptions + +from coredb import keydb, dbfiles +import onionrcrypto +from onionrcrypto import getourkeypair +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -16,14 +27,8 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' -import logger, onionrexceptions, json, sqlite3, time -from onionrutils import stringvalidators, bytesconverter, epoch -import unpaddedbase32 -import nacl.exceptions -from coredb import keydb, dbfiles -import onionrcrypto -from onionrcrypto import getourkeypair +""" + def deleteExpiredKeys(): # Fetch the keys we generated for the peer, that are still around @@ -37,6 +42,7 @@ def deleteExpiredKeys(): conn.close() return + def deleteTheirExpiredKeys(pubkey): conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10) c = conn.cursor() @@ -49,17 +55,19 @@ def deleteTheirExpiredKeys(pubkey): conn.commit() conn.close() + DEFAULT_KEY_EXPIRE = 604800 + class OnionrUser: def __init__(self, publicKey, saveUser=False): - ''' + """ OnionrUser is an abstraction for "users" of the network. Takes a base32 encoded ed25519 public key, and a bool saveUser saveUser determines if we should add a user to our peer database or not. - ''' + """ publicKey = unpaddedbase32.repad(bytesconverter.str_to_bytes(publicKey)).decode() self.trust = 0 @@ -75,7 +83,7 @@ class OnionrUser: return def setTrust(self, newTrust): - '''Set the peers trust. 0 = not trusted, 1 = friend, 2 = ultimate''' + """Set the peers trust. 0 = not trusted, 1 = friend, 2 = ultimate""" keydb.userinfo.set_user_info(self.publicKey, 'trust', newTrust) def isFriend(self): @@ -227,4 +235,4 @@ class OnionrUser: friendList = [] for x in keydb.listkeys.list_peers(trust=1): friendList.append(cls(x)) - return list(friendList) \ No newline at end of file + return list(friendList) From 37913d544bc7294351a1ce65a43f86cef8865331 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 3 Apr 2020 04:19:20 -0500 Subject: [PATCH 008/105] fixed localcommand formatting and imports --- src/onionrutils/localcommand.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/onionrutils/localcommand.py b/src/onionrutils/localcommand.py index 7a3bde84..b1330cf8 100644 --- a/src/onionrutils/localcommand.py +++ b/src/onionrutils/localcommand.py @@ -1,10 +1,9 @@ -""" - Onionr - Private P2P Communication +"""Onionr - Private P2P Communication. - send a command to the local API server +send a command to the local API server """ -import urllib, time -import json +import urllib +import time import functools from typing import TYPE_CHECKING, Callable @@ -29,7 +28,10 @@ import filepaths """ config.reload() -cache = deadsimplekv.DeadSimpleKV(filepaths.cached_storage, refresh_seconds=1000) +cache = deadsimplekv.DeadSimpleKV(filepaths.cached_storage, + refresh_seconds=1000) + + def get_hostname(): hostname = '' waited = 0 @@ -52,6 +54,7 @@ def get_hostname(): else: return hostname + def local_command(command, data='', silent = True, post=False, postData = {}, maxWait=20, is_json=False From a8ab5d20f121cb72b172e7c88c873da75cd345da Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 3 Apr 2020 04:20:55 -0500 Subject: [PATCH 009/105] moving onionrutils over to ujson --- src/onionrutils/blockmetadata/fromdata.py | 22 +++++++++++----------- src/onionrutils/validatemetadata.py | 17 +++++++++-------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/onionrutils/blockmetadata/fromdata.py b/src/onionrutils/blockmetadata/fromdata.py index a39f54c3..c437d6df 100644 --- a/src/onionrutils/blockmetadata/fromdata.py +++ b/src/onionrutils/blockmetadata/fromdata.py @@ -1,9 +1,11 @@ -''' - Onionr - Private P2P Communication +"""Onionr - Private P2P Communication. Return a useful tuple of (metadata (header), meta, and data) by accepting raw block data -''' -''' +""" +import ujson as json + +from onionrutils import bytesconverter +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -16,17 +18,15 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' +""" -import json -from onionrutils import bytesconverter def get_block_metadata_from_data(block_data): - ''' - accepts block contents as string, returns a tuple of - metadata, meta (meta being internal metadata, which will be + """ + accepts block contents as string, returns a tuple of + metadata, meta (meta being internal metadata, which will be returned as an encrypted base64 string if it is encrypted, dict if not). - ''' + """ meta = {} metadata = {} data = block_data diff --git a/src/onionrutils/validatemetadata.py b/src/onionrutils/validatemetadata.py index 9349ec9e..8cb7a1cf 100644 --- a/src/onionrutils/validatemetadata.py +++ b/src/onionrutils/validatemetadata.py @@ -1,8 +1,13 @@ -""" - Onionr - Private P2P Communication +"""Onionr - Private P2P Communication. - validate new block's metadata +validate new block's metadata """ +import ujson as json + +import logger, onionrexceptions +from etc import onionrvalues +from . import stringvalidators, epoch, bytesconverter +import config, filepaths, onionrcrypto """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -17,11 +22,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . """ -import json -import logger, onionrexceptions -from etc import onionrvalues -from . import stringvalidators, epoch, bytesconverter -import config, filepaths, onionrcrypto + def validate_metadata(metadata, block_data) -> bool: """Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string""" From 919ab12b76167b220acd9c9b9f0325589e35c0dc Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 3 Apr 2020 04:27:37 -0500 Subject: [PATCH 010/105] plugins now use ujson --- .../default-plugins/chat/controlapi.py | 21 +++++------ static-data/default-plugins/chat/main.py | 2 +- .../default-plugins/chat/peerserver.py | 26 ++++++++------ .../default-plugins/circles/flowapi.py | 3 +- static-data/default-plugins/encrypt/main.py | 5 ++- static-data/default-plugins/pms/mailapi.py | 35 ++++++++++--------- static-data/default-plugins/pms/main.py | 3 +- 7 files changed, 54 insertions(+), 41 deletions(-) diff --git a/static-data/default-plugins/chat/controlapi.py b/static-data/default-plugins/chat/controlapi.py index 7718c36a..0cbac652 100755 --- a/static-data/default-plugins/chat/controlapi.py +++ b/static-data/default-plugins/chat/controlapi.py @@ -1,9 +1,14 @@ -''' - Onionr - Private P2P Communication +"""Onionr - Private P2P Communication. - HTTP endpoints for controlling IMs -''' -''' +HTTP endpoints for controlling IMs +""" +import ujson as json + +from flask import Response, request, redirect, Blueprint, send_from_directory +import deadsimplekv as simplekv + +import filepaths +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -16,11 +21,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' -import json -from flask import Response, request, redirect, Blueprint, send_from_directory -import deadsimplekv as simplekv -import filepaths +""" flask_blueprint = Blueprint('chat_control', __name__) key_store = simplekv.DeadSimpleKV(filepaths.cached_storage, refresh_seconds=5) @flask_blueprint.route('/chatapi/ping') diff --git a/static-data/default-plugins/chat/main.py b/static-data/default-plugins/chat/main.py index 2b8f7eb6..a7d730d4 100755 --- a/static-data/default-plugins/chat/main.py +++ b/static-data/default-plugins/chat/main.py @@ -19,7 +19,7 @@ ''' # Imports some useful libraries -import locale, sys, os, threading, json +import locale, sys, os, threading, ujson as json locale.setlocale(locale.LC_ALL, '') import onionrservices, logger, config from onionrservices import bootstrapservice diff --git a/static-data/default-plugins/chat/peerserver.py b/static-data/default-plugins/chat/peerserver.py index 58dc3d7f..2d8e0905 100755 --- a/static-data/default-plugins/chat/peerserver.py +++ b/static-data/default-plugins/chat/peerserver.py @@ -1,9 +1,18 @@ -''' - Onionr - Private P2P Communication +"""Onionr - Private P2P Communication. - HTTP endpoints for communicating with peers -''' -''' +HTTP endpoints for communicating with peers +""" +import sys +import os + +import deadsimplekv as simplekv +import ujson as json +from flask import Response, request, redirect, Blueprint, abort, g + +from utils import identifyhome +from onionrutils import localcommand +import filepaths +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -16,12 +25,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' -import sys, os, json -from utils import identifyhome -from onionrutils import localcommand -import deadsimplekv as simplekv, filepaths -from flask import Response, request, redirect, Blueprint, abort, g +""" sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) direct_blueprint = Blueprint('chat', __name__) diff --git a/static-data/default-plugins/circles/flowapi.py b/static-data/default-plugins/circles/flowapi.py index 6c946d47..01f70a04 100755 --- a/static-data/default-plugins/circles/flowapi.py +++ b/static-data/default-plugins/circles/flowapi.py @@ -3,9 +3,10 @@ This file primarily serves to allow specific fetching of circles board messages """ import operator -import json import os +import ujson as json + from flask import Response, Blueprint from deadsimplekv import DeadSimpleKV diff --git a/static-data/default-plugins/encrypt/main.py b/static-data/default-plugins/encrypt/main.py index 239be339..8fd53fe2 100755 --- a/static-data/default-plugins/encrypt/main.py +++ b/static-data/default-plugins/encrypt/main.py @@ -19,7 +19,10 @@ ''' # Imports some useful libraries -import logger, config, threading, time, datetime, sys, json +import logger, config, threading, time, datetime, sys + +import ujson as json + from onionrutils import stringvalidators, bytesconverter from onionrcrypto import encryption, keypair, signing, getourkeypair import onionrexceptions, onionrusers diff --git a/static-data/default-plugins/pms/mailapi.py b/static-data/default-plugins/pms/mailapi.py index efa827f4..81c5a127 100755 --- a/static-data/default-plugins/pms/mailapi.py +++ b/static-data/default-plugins/pms/mailapi.py @@ -1,9 +1,22 @@ -''' - Onionr - Private P2P Communication +"""Onionr - Private P2P Communication. - HTTP endpoints for mail plugin. -''' -''' +HTTP endpoints for mail plugin +""" +import sys +import os + +import ujson as json +from flask import Response, request, redirect, Blueprint, abort +import deadsimplekv as simplekv + +from onionrusers import contactmanager +from onionrutils import stringvalidators +from utils import reconstructhash, identifyhome + +sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) +import loadinbox +import sentboxdb +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -16,17 +29,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' -import sys, os, json -from flask import Response, request, redirect, Blueprint, abort -from onionrusers import contactmanager -from onionrutils import stringvalidators -from utils import reconstructhash, identifyhome -import filepaths -import deadsimplekv as simplekv -sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) -import loadinbox, sentboxdb - +""" flask_blueprint = Blueprint('mail', __name__) kv = simplekv.DeadSimpleKV(identifyhome.identify_home() + '/mailcache.dat') diff --git a/static-data/default-plugins/pms/main.py b/static-data/default-plugins/pms/main.py index bda3f1fd..cf03c1b7 100755 --- a/static-data/default-plugins/pms/main.py +++ b/static-data/default-plugins/pms/main.py @@ -5,7 +5,8 @@ Private messages in an email like fashion import locale import sys import os -import json + +import ujson as json from onionrusers import contactmanager from utils import reconstructhash From 39d0be32acb22efc8ed243bcfd4cbbb9d9ca4f54 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 3 Apr 2020 04:33:30 -0500 Subject: [PATCH 011/105] import jsondecodeerror seperately since its not in ujson --- src/config/__init__.py | 5 +++-- src/httpapi/configapi/__init__.py | 5 +++-- src/onionrutils/localcommand.py | 6 ++++-- src/onionrutils/validatemetadata.py | 3 ++- static-data/default-plugins/chat/peerserver.py | 3 ++- static-data/default-plugins/circles/flowapi.py | 2 +- 6 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/config/__init__.py b/src/config/__init__.py index 2762a522..b292c4be 100755 --- a/src/config/__init__.py +++ b/src/config/__init__.py @@ -3,6 +3,7 @@ This file deals with configuration management. """ import os +from json import JSONDecodeError import ujson as json import logger @@ -105,7 +106,7 @@ def save(): try: with open(get_config_file(), 'w', encoding="utf8") as configfile: json.dump(get_config(), configfile, indent=2) - except json.JSONDecodeError: + except JSONDecodeError: logger.warn('Failed to write to configuration file.') @@ -115,7 +116,7 @@ def reload(): try: with open(get_config_file(), 'r', encoding="utf8") as configfile: set_config(json.loads(configfile.read())) - except (FileNotFoundError, json.JSONDecodeError) as e: + except (FileNotFoundError, JSONDecodeError) as e: pass #logger.debug('Failed to parse configuration file.') diff --git a/src/httpapi/configapi/__init__.py b/src/httpapi/configapi/__init__.py index ea8fbc85..a0aefab7 100755 --- a/src/httpapi/configapi/__init__.py +++ b/src/httpapi/configapi/__init__.py @@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . """ +from json import JSONDecodeError import ujson as json from flask import Blueprint, request, Response, abort @@ -42,7 +43,7 @@ def set_all_config(): """Overwrite existing JSON config with new JSON string""" try: new_config = request.get_json(force=True) - except json.JSONDecodeError: + except JSONDecodeError: abort(400) else: config.set_config(new_config) @@ -59,7 +60,7 @@ def set_by_key(key): """ try: data = json.loads(bytes_to_str(request.data)) - except (json.JSONDecodeError, KeyError): + except (JSONDecodeError, KeyError): abort(400) config.set(key, data, True) return Response('success') \ No newline at end of file diff --git a/src/onionrutils/localcommand.py b/src/onionrutils/localcommand.py index b1330cf8..b7e7aab2 100644 --- a/src/onionrutils/localcommand.py +++ b/src/onionrutils/localcommand.py @@ -4,12 +4,14 @@ send a command to the local API server """ import urllib import time -import functools from typing import TYPE_CHECKING, Callable import requests +import deadsimplekv + +import logger +import config -import logger, config, deadsimplekv from . import getclientapiserver import filepaths """ diff --git a/src/onionrutils/validatemetadata.py b/src/onionrutils/validatemetadata.py index 8cb7a1cf..a4e1ccea 100644 --- a/src/onionrutils/validatemetadata.py +++ b/src/onionrutils/validatemetadata.py @@ -2,6 +2,7 @@ validate new block's metadata """ +from json import JSONDecodeError import ujson as json import logger, onionrexceptions @@ -34,7 +35,7 @@ def validate_metadata(metadata, block_data) -> bool: if type(metadata) is str: try: metadata = json.loads(metadata) - except json.JSONDecodeError: + except JSONDecodeError: pass # Validate metadata dict for invalid keys to sizes that are too large diff --git a/static-data/default-plugins/chat/peerserver.py b/static-data/default-plugins/chat/peerserver.py index 2d8e0905..7bf0dda2 100755 --- a/static-data/default-plugins/chat/peerserver.py +++ b/static-data/default-plugins/chat/peerserver.py @@ -4,6 +4,7 @@ HTTP endpoints for communicating with peers """ import sys import os +from json import JSONDecodeError import deadsimplekv as simplekv import ujson as json @@ -50,7 +51,7 @@ def sendto(): """Endpoint peers send chat messages to""" try: msg = request.get_json(force=True) - except json.JSONDecodeError: + except JSONDecodeError: msg = '' else: msg = json.dumps(msg) diff --git a/static-data/default-plugins/circles/flowapi.py b/static-data/default-plugins/circles/flowapi.py index 01f70a04..e1cd5ad8 100755 --- a/static-data/default-plugins/circles/flowapi.py +++ b/static-data/default-plugins/circles/flowapi.py @@ -32,7 +32,7 @@ with open( os.path.dirname( os.path.realpath(__file__)) + '/info.json', 'r') as info_file: data = info_file.read().strip() - version = json.loads(data, strict=False)['version'] + version = json.loads(data)['version'] BOARD_CACHE_FILE = identifyhome.identify_home() + '/board-index.cache.json' From d47c546620a5eed5ffea2b9cea658db5b2d3741a Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 3 Apr 2020 23:34:23 -0500 Subject: [PATCH 012/105] added dns rebinding runtime test --- src/runtests/__init__.py | 4 ++- src/runtests/dnsrebindingtest.py | 46 ++++++++++++++++++++++++++++++++ tests/runtime-result.txt | 2 +- 3 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 src/runtests/dnsrebindingtest.py diff --git a/src/runtests/__init__.py b/src/runtests/__init__.py index c9a6e3dc..10694b70 100644 --- a/src/runtests/__init__.py +++ b/src/runtests/__init__.py @@ -16,6 +16,7 @@ from .clearnettor import test_clearnet_tor_request from .housekeeping import test_inserted_housekeeping from .lanservertest import test_lan_server from .sneakernettest import test_sneakernet_import +from .dnsrebindingtest import test_dns_rebinding """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -41,7 +42,8 @@ RUN_TESTS = [uicheck.check_ui, test_clearnet_tor_request, test_inserted_housekeeping, test_lan_server, - sneakernettest.test_sneakernet_import + sneakernettest.test_sneakernet_import, + test_dns_rebinding ] SUCCESS_FILE = os.path.dirname(os.path.realpath(__file__)) + '/../../tests/runtime-result.txt' diff --git a/src/runtests/dnsrebindingtest.py b/src/runtests/dnsrebindingtest.py new file mode 100644 index 00000000..8a430674 --- /dev/null +++ b/src/runtests/dnsrebindingtest.py @@ -0,0 +1,46 @@ +"""Onionr - Private P2P Communication. + +Test apis for dns rebinding +""" +import config +import requests +from filepaths import private_API_host_file, public_API_host_file +import logger +""" + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + + +def test_dns_rebinding(test_manager): + f = '' + with open(private_API_host_file, 'r') as f: + host = f.read() + private_api_port = config.get('client.client.port') + + if requests.get(f'http://{host}:{private_api_port}/ping', headers={'host': 'example.com'}) == 'pong!': + raise ValueError('DNS rebinding failed') + logger.info('It is normal to see 403 errors right now', terminal=True) + + if config.get('general.security_level', 0) > 0 or not config.get('transports.tor', True): + return + public_api_port = config.get('client.public.port') + f = '' + with open(public_API_host_file, 'r') as f: + host = f.read() + + if requests.get(f'http://{host}:{public_api_port}/ping', headers={'host': 'example.com'}) == 'pong!': + raise ValueError('DNS rebinding failed') + logger.info('It is normal to see 403 errors right now', terminal=True) + + diff --git a/tests/runtime-result.txt b/tests/runtime-result.txt index 8dc1d799..652cfcbb 100644 --- a/tests/runtime-result.txt +++ b/tests/runtime-result.txt @@ -1 +1 @@ -1585902463 \ No newline at end of file +1585974430 \ No newline at end of file From da1940cc8e66606da824edc45ac23a8be809693f Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 4 Apr 2020 00:58:02 -0500 Subject: [PATCH 013/105] fix circles escapeansi being passed bytes --- static-data/default-plugins/circles/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static-data/default-plugins/circles/main.py b/static-data/default-plugins/circles/main.py index 0e75dc8b..66178a8c 100755 --- a/static-data/default-plugins/circles/main.py +++ b/static-data/default-plugins/circles/main.py @@ -106,7 +106,7 @@ class OnionrFlow: content = block.getContent() # Escape new lines, remove trailing whitespace, and escape ansi sequences content = escapeansi.escape_ANSI(content.replace( - '\n', '\\n').replace('\r', '\\r').strip()) + b'\n', b'\\n').replace(b'\r', b'\\r').strip().decode('utf-8')) logger.info(block.getDate().strftime( "%m/%d %H:%M") + ' - ' + logger.colors.reset + content, prompt=False, terminal=True) self.alreadyOutputed.append(b_hash) From 653954714ffe22a4501f4a68e7085ba3d8007965 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 4 Apr 2020 00:58:49 -0500 Subject: [PATCH 014/105] added script to start many nodes --- start-many-nodes.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 start-many-nodes.py diff --git a/start-many-nodes.py b/start-many-nodes.py new file mode 100644 index 00000000..cc78a1fd --- /dev/null +++ b/start-many-nodes.py @@ -0,0 +1,8 @@ +import os +n = int(input("how many nodes: ")) + + + +for _ in range(n): + os.system('./start-ram.sh') + From aa729d862c658e86dc38cf5a99f8d0badba34351 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 4 Apr 2020 02:21:28 -0500 Subject: [PATCH 015/105] added escape_ansi unittest --- src/onionrutils/escapeansi.py | 2 ++ tests/runtime-result.txt | 2 +- tests/test_escape_ansi.py | 63 +++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 tests/test_escape_ansi.py diff --git a/src/onionrutils/escapeansi.py b/src/onionrutils/escapeansi.py index ca635097..1641da39 100644 --- a/src/onionrutils/escapeansi.py +++ b/src/onionrutils/escapeansi.py @@ -1,4 +1,6 @@ import re + + def escape_ANSI(line): ''' Remove ANSI escape codes from a string with regex diff --git a/tests/runtime-result.txt b/tests/runtime-result.txt index 652cfcbb..158d9521 100644 --- a/tests/runtime-result.txt +++ b/tests/runtime-result.txt @@ -1 +1 @@ -1585974430 \ No newline at end of file +1585984468 \ No newline at end of file diff --git a/tests/test_escape_ansi.py b/tests/test_escape_ansi.py new file mode 100644 index 00000000..b0cc8fc6 --- /dev/null +++ b/tests/test_escape_ansi.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +import sys, os, random +sys.path.append(".") +sys.path.append("src/") +import unittest, uuid +TEST_DIR_1 = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' +TEST_DIR_2 = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' +import time + +os.environ["ONIONR_HOME"] = TEST_DIR_1 +from utils import identifyhome, createdirs +from onionrsetup import setup_config +createdirs.create_dirs() + +from onionrutils.escapeansi import escape_ANSI + +class Colors: + """ ANSI color codes """ + BLACK = "\033[0;30m" + RED = "\033[0;31m" + GREEN = "\033[0;32m" + BROWN = "\033[0;33m" + BLUE = "\033[0;34m" + PURPLE = "\033[0;35m" + CYAN = "\033[0;36m" + LIGHT_GRAY = "\033[0;37m" + DARK_GRAY = "\033[1;30m" + LIGHT_RED = "\033[1;31m" + LIGHT_GREEN = "\033[1;32m" + YELLOW = "\033[1;33m" + LIGHT_BLUE = "\033[1;34m" + LIGHT_PURPLE = "\033[1;35m" + LIGHT_CYAN = "\033[1;36m" + LIGHT_WHITE = "\033[1;37m" + BOLD = "\033[1m" + FAINT = "\033[2m" + ITALIC = "\033[3m" + UNDERLINE = "\033[4m" + BLINK = "\033[5m" + NEGATIVE = "\033[7m" + CROSSED = "\033[9m" + END = "\033[0m" + +class OnionrForwardSecrecyTests(unittest.TestCase): + def test_no_replace(self): + msg = 'test message' + self.assertEqual(escape_ANSI(msg), msg) + + def test_escape_ansi(self): + msg = "test" + for color in dir(Colors): + color = getattr(Colors, color) + try: + if '[' not in color and r'\0' not in color: continue + except TypeError: + continue + try: + self.assertEqual(escape_ANSI(color + msg), msg) + except TypeError: + pass + self.assertEqual(escape_ANSI(msg), msg) + +unittest.main() From 7bedae48a4d41f34110e754ae58977149e0519bd Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 4 Apr 2020 04:22:11 -0500 Subject: [PATCH 016/105] * Mail compose no longer uses overlay --- static-data/www/mail/index.html | 38 ++++++++++++++++----------------- static-data/www/mail/mail.css | 1 - static-data/www/mail/mail.js | 6 +++--- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/static-data/www/mail/index.html b/static-data/www/mail/index.html index dbe7adb9..a320f552 100755 --- a/static-data/www/mail/index.html +++ b/static-data/www/mail/index.html @@ -113,25 +113,6 @@ -
-
-
- -
-
- -
- To: -
- Subject: -
- -
- -
-
-
- +
+
+
+ +
+
+
+ To: +
+ Subject: +
+ +
+ +
+
+
diff --git a/static-data/www/mail/mail.css b/static-data/www/mail/mail.css index 45099b0b..f2212a04 100755 --- a/static-data/www/mail/mail.css +++ b/static-data/www/mail/mail.css @@ -58,4 +58,3 @@ #settingsModal small{ font-size: 0.5em; } - diff --git a/static-data/www/mail/mail.js b/static-data/www/mail/mail.js index 12956d98..f83408d0 100755 --- a/static-data/www/mail/mail.js +++ b/static-data/www/mail/mail.js @@ -128,6 +128,7 @@ function openThread(bHash, sender, date, sigBool, pubkey, subjectLine){ sigEl.innerText = sigMsg overlay('messageDisplay') replyBtn.onclick = function(){ + document.getElementById('messageDisplay').style.visibility = 'hidden' openReply(bHash, messageDisplay.innerText, subjectLine) } addUnknownContact.onclick = function(){ @@ -144,6 +145,7 @@ function setActiveTab(tabName){ threadPart.innerHTML = "" noInbox.style.display = 'none' window.inboxActive = false + document.getElementById('sendMessage').classList.add('is-hidden') switch(tabName){ case 'inbox': window.inboxActive = true @@ -154,8 +156,7 @@ function setActiveTab(tabName){ getSentbox() break case 'compose': - overlay('sendMessage') - document.getElementById('inboxTab').click() + document.getElementById('sendMessage').classList.remove('is-hidden') break case 'settings': document.getElementById('settingsModal').classList.add('is-active') @@ -419,7 +420,6 @@ fetch('/friends/list', { .then(function(resp) { var friendSelectParent = document.getElementById('friendSelect') var keys = []; - var friend for(var k in resp) keys.push(k); friendSelectParent.appendChild(document.createElement('option')) From 4cd2302bc9676c13a8235d47128a59deaf11bdce Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 5 Apr 2020 03:52:40 -0500 Subject: [PATCH 017/105] added logic for stranger notification setting made page content refreshes happen on visibility --- docs/TODO.txt | 1 - static-data/default-plugins/pms/main.py | 3 +++ static-data/www/board/autorefresh.js | 9 ++++++++- static-data/www/mail/mail.js | 8 +++++++- static-data/www/shared/main/stats.js | 8 +++++++- 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/docs/TODO.txt b/docs/TODO.txt index 006f0b02..ed49b915 100644 --- a/docs/TODO.txt +++ b/docs/TODO.txt @@ -12,7 +12,6 @@ * make node "speed" setting such as when ui is open to reduce bandwidth usage * localization support -* add mark read in mail * add BCC support to mail diff --git a/static-data/default-plugins/pms/main.py b/static-data/default-plugins/pms/main.py index cf03c1b7..4a7753ce 100755 --- a/static-data/default-plugins/pms/main.py +++ b/static-data/default-plugins/pms/main.py @@ -76,4 +76,7 @@ def on_processblocks(api, data=None): if data['block'].decrypted: config.reload() if config.get('mail.notificationSetting', True): + if not config.get('mail.strangersNotification', True): + if not user.isFriend(): + return notifier.notification_with_sound(title="Onionr Mail - New Message", message="From: %s\n\nSubject: %s" % (signer, metadata['subject'])) diff --git a/static-data/www/board/autorefresh.js b/static-data/www/board/autorefresh.js index 066ef301..bb41fde1 100644 --- a/static-data/www/board/autorefresh.js +++ b/static-data/www/board/autorefresh.js @@ -34,4 +34,11 @@ function setupInterval(){ var refreshInterval = setInterval(autoRefresh, 3000) setupInterval() -checkbox.onchange = function(){setupInterval} \ No newline at end of file +checkbox.onchange = function(){setupInterval} + + +document.addEventListener("visibilitychange", function() { + if (document.visibilityState === 'visible') { + autoRefresh() + } + }) diff --git a/static-data/www/mail/mail.js b/static-data/www/mail/mail.js index f83408d0..f693e808 100755 --- a/static-data/www/mail/mail.js +++ b/static-data/www/mail/mail.js @@ -441,4 +441,10 @@ setActiveTab('inbox') setInterval(function(){mailPing()}, 10000) mailPing() window.inboxInterval = setInterval(function(){refreshPms(true)}, 3000) -refreshPms(true) \ No newline at end of file +refreshPms(true) + +document.addEventListener("visibilitychange", function() { + if (document.visibilityState === 'visible') { + refreshPms() + } + }) diff --git a/static-data/www/shared/main/stats.js b/static-data/www/shared/main/stats.js index a33068e4..00f3d2fb 100755 --- a/static-data/www/shared/main/stats.js +++ b/static-data/www/shared/main/stats.js @@ -136,5 +136,11 @@ var getStats = function(){ }) } +document.addEventListener("visibilitychange", function() { + if (document.visibilityState === 'visible') { + getStats() + } + }) + getStats() -setInterval(function(){getStats()}, 10000) \ No newline at end of file +setInterval(function(){getStats()}, 1000) \ No newline at end of file From ae8d1fc5ea36fc98995ca6783c24958591bd3b0a Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 5 Apr 2020 23:21:21 -0500 Subject: [PATCH 018/105] Finished implementing notification settings for sound and strangers --- static-data/default-plugins/pms/main.py | 7 ++++- static-data/www/mail/index.html | 14 +++++----- static-data/www/mail/loadsettings.js | 6 +++++ static-data/www/mail/settings.js | 36 +++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/static-data/default-plugins/pms/main.py b/static-data/default-plugins/pms/main.py index 4a7753ce..0b0a1c5b 100755 --- a/static-data/default-plugins/pms/main.py +++ b/static-data/default-plugins/pms/main.py @@ -62,6 +62,7 @@ def on_insertblock(api, data={}): def on_processblocks(api, data=None): if data['type'] != 'pm': return + notification_func = notifier.notify data['block'].decrypt() metadata = data['block'].bmetadata @@ -73,10 +74,14 @@ def on_processblocks(api, data=None): else: signer = signer[:5] + if data['block'].decrypted: config.reload() + + if config.get('mail.notificationSound', True): + notification_func = notifier.notification_with_sound if config.get('mail.notificationSetting', True): if not config.get('mail.strangersNotification', True): if not user.isFriend(): return - notifier.notification_with_sound(title="Onionr Mail - New Message", message="From: %s\n\nSubject: %s" % (signer, metadata['subject'])) + notification_func(title="Onionr Mail - New Message", message="From: %s\n\nSubject: %s" % (signer, metadata['subject'])) diff --git a/static-data/www/mail/index.html b/static-data/www/mail/index.html index a320f552..a67be33f 100755 --- a/static-data/www/mail/index.html +++ b/static-data/www/mail/index.html @@ -161,13 +161,13 @@
- Only show notifications for friends + Notifications for stranger's messages
- - + +
@@ -177,9 +177,9 @@
- - + +
diff --git a/static-data/www/mail/loadsettings.js b/static-data/www/mail/loadsettings.js index c6608c29..3ec81891 100644 --- a/static-data/www/mail/loadsettings.js +++ b/static-data/www/mail/loadsettings.js @@ -17,7 +17,13 @@ fetch('/config/get/mail', { if (mailSettings.notificationSetting === false){ document.getElementById('notificationSetting').checked = false } + if (mailSettings.notificationSound === false){ + document.getElementById('notificationSound').checked = false + } if (typeof mailSettings.signature != undefined && mailSettings.signature != null && mailSettings.signature != ""){ document.getElementById('mailSignatureSetting').value = mailSettings.signature } + if (mailSettings.strangersNotification == false){ + document.getElementById('strangersNotification').checked = false + } }) \ No newline at end of file diff --git a/static-data/www/mail/settings.js b/static-data/www/mail/settings.js index 0667a31b..f410660d 100644 --- a/static-data/www/mail/settings.js +++ b/static-data/www/mail/settings.js @@ -17,6 +17,8 @@ along with this program. If not, see . */ var notificationSetting = document.getElementById('notificationSetting') +var friendOnlyNotification = document.getElementById('strangersNotification') +var notificationSound = document.getElementById('notificationSound') var sigSetting = document.getElementById('mailSignatureSetting') document.getElementById('forwardSecrecySetting').onchange = function(e){ @@ -37,6 +39,40 @@ document.getElementById('forwardSecrecySetting').onchange = function(e){ }) } +notificationSound.onchange = function(e){ + var postData = JSON.stringify({"notificationSound": e.target.checked}) + fetch('/config/set/mail', { + method: 'POST', + body: postData, + headers: { + "content-type": "application/json", + "token": webpass + }}) + .then(function(data) { + mailSettings['notificationSound'] = notificationSound.checked + PNotify.success({ + text: 'Successfully notification sound' + }) + }) +} + +friendOnlyNotification.onchange = function(e){ + var postData = JSON.stringify({"strangersNotification": e.target.checked}) + fetch('/config/set/mail', { + method: 'POST', + body: postData, + headers: { + "content-type": "application/json", + "token": webpass + }}) + .then(function(data) { + mailSettings['strangersNotification'] = friendOnlyNotification.checked + PNotify.success({ + text: 'Successfully toggled notifications from strangers' + }) + }) +} + notificationSetting.onchange = function(e){ var notificationSettings = document.getElementsByClassName('notificationSetting') From d13d9a3039d52aa2f57870d1ba95826fc8bee183 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 5 Apr 2020 23:35:03 -0500 Subject: [PATCH 019/105] added is_friend test for onionrusers and fixed bug where isFriend was setting instead of getting the value --- src/onionrusers/onionrusers.py | 2 +- tests/test_onionrusers.py | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/onionrusers/onionrusers.py b/src/onionrusers/onionrusers.py index 50b1c00c..388d8cac 100755 --- a/src/onionrusers/onionrusers.py +++ b/src/onionrusers/onionrusers.py @@ -87,7 +87,7 @@ class OnionrUser: keydb.userinfo.set_user_info(self.publicKey, 'trust', newTrust) def isFriend(self): - if keydb.userinfo.set_peer_info(self.publicKey, 'trust') == 1: + if keydb.userinfo.get_user_info(self.publicKey, 'trust') == 1: return True return False diff --git a/tests/test_onionrusers.py b/tests/test_onionrusers.py index 73590cbb..de5b56a2 100644 --- a/tests/test_onionrusers.py +++ b/tests/test_onionrusers.py @@ -19,11 +19,10 @@ class OnionrUserTests(unittest.TestCase): ''' Tests both the onionrusers class and the contactmanager (which inherits it) ''' - + def test_users(self): keypair = crypto.generate() onionrusers.OnionrUser(keypair[0]) - return def test_contact_init_no_save(self): contact = crypto.generate()[0] @@ -34,7 +33,7 @@ class OnionrUserTests(unittest.TestCase): contact = crypto.generate()[0] contact = contactmanager.ContactManager(contact, saveUser=True) self.assertTrue(contact.publicKey in keydb.listkeys.list_peers()) - + def test_contact_set_info(self): contact = crypto.generate()[0] contact = contactmanager.ContactManager(contact, saveUser=True) @@ -44,10 +43,10 @@ class OnionrUserTests(unittest.TestCase): with open(fileLocation, 'r') as data: data = data.read() - + data = json.loads(data) self.assertEqual(data['alias'], 'bob') - + def test_contact_get_info(self): contact = crypto.generate()[0] contact = contactmanager.ContactManager(contact, saveUser=True) @@ -55,18 +54,25 @@ class OnionrUserTests(unittest.TestCase): with open(fileLocation, 'w') as contactFile: contactFile.write('{"alias": "bob"}') - + self.assertEqual(contact.get_info('alias', forceReload=True), 'bob') self.assertEqual(contact.get_info('fail', forceReload=True), None) self.assertEqual(contact.get_info('fail'), None) - + + def test_is_friend(self): + contact = crypto.generate()[0] + contact = onionrusers.OnionrUser(contact, saveUser=True) + self.assertFalse(contact.isFriend()) + contact.setTrust(1) + self.assertTrue(contact.isFriend()) + def test_encrypt(self): contactPair = crypto.generate() contact = contactmanager.ContactManager(contactPair[0], saveUser=True) encrypted = contact.encrypt('test') decrypted = crypto.encryption.pub_key_decrypt(encrypted, privkey=contactPair[1], encodedData=True).decode() self.assertEqual('test', decrypted) - + def test_delete_contact(self): contact = crypto.generate()[0] contact = contactmanager.ContactManager(contact, saveUser=True) From 12137d3c001b1d38d49e0c49df3006cc89926925 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2020 08:05:19 +0000 Subject: [PATCH 020/105] Bump flask from 1.1.1 to 1.1.2 Bumps [flask](https://github.com/pallets/flask) from 1.1.1 to 1.1.2. - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/master/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/1.1.1...1.1.2) Signed-off-by: dependabot-preview[bot] --- requirements.in | 2 +- requirements.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.in b/requirements.in index 8d04fa62..261d9d24 100644 --- a/requirements.in +++ b/requirements.in @@ -2,7 +2,7 @@ urllib3==1.25.8 requests==2.23.0 PyNaCl==1.3.0 gevent==1.4.0 -Flask==1.1.1 +Flask==1.1.2 PySocks==1.7.1 stem==1.8.0 deadsimplekv==0.3.1 diff --git a/requirements.txt b/requirements.txt index 31c9b8ff..1254699a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,9 +54,9 @@ filenuke==0.0.0 \ --hash=sha256:147011c0125121469cae0a8a7f4df399f470e54aa29a08f2d2c099bf0118dcee \ --hash=sha256:c55535dcecfdb27c5f4ce664d46e115950b5429763b5db75c198053646177f8f \ # via -r requirements.in -flask==1.1.1 \ - --hash=sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52 \ - --hash=sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6 \ +flask==1.1.2 \ + --hash=sha256:4efa1ae2d7c9865af48986de8aeb8504bf32c7f3d6fdc9353d34b21f4b127060 \ + --hash=sha256:8a4fdd8936eba2512e9c85df320a37e694c93945b33ef33c89946a340a238557 \ # via -r requirements.in gevent==1.4.0 \ --hash=sha256:0774babec518a24d9a7231d4e689931f31b332c4517a771e532002614e270a64 \ From b35d5be8dba1beeba2f40a9be6dcbc510115aced Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2020 08:38:07 +0000 Subject: [PATCH 021/105] Bump streamedrequests from 1.0.0 to 1.0.3 Bumps [streamedrequests](https://github.com/beardog108/StreamedRequests) from 1.0.0 to 1.0.3. - [Release notes](https://github.com/beardog108/StreamedRequests/releases) - [Changelog](https://github.com/beardog108/StreamedRequests/blob/master/CHANGELOG.md) - [Commits](https://github.com/beardog108/StreamedRequests/commits) Signed-off-by: dependabot-preview[bot] --- requirements.in | 2 +- requirements.txt | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements.in b/requirements.in index 261d9d24..34ae290d 100644 --- a/requirements.in +++ b/requirements.in @@ -7,7 +7,7 @@ PySocks==1.7.1 stem==1.8.0 deadsimplekv==0.3.1 unpaddedbase32==0.2.0 -streamedrequests==1.0.0 +streamedrequests==1.0.3 toomanyobjs==1.1.0 niceware==0.2.1 psutil==5.7.0 diff --git a/requirements.txt b/requirements.txt index 1254699a..4ebd6e03 100644 --- a/requirements.txt +++ b/requirements.txt @@ -210,8 +210,9 @@ six==1.12.0 \ stem==1.8.0 \ --hash=sha256:a0b48ea6224e95f22aa34c0bc3415f0eb4667ddeae3dfb5e32a6920c185568c2 \ # via -r requirements.in -streamedrequests==1.0.0 \ - --hash=sha256:1d9d07394804a6e1fd66bde74a804e71cab98e6920053865574a459f1cf7d3b7 \ +streamedrequests==1.0.3 \ + --hash=sha256:4388ffc0ee94dda719dafc4324b8ddd108cb2231ec59871de79e2592bf4eef0a \ + --hash=sha256:ee68417a1522e75c35b1b2d5f3b6f7e76a3a1a6c0ef5e0c573d08307910079d8 \ # via -r requirements.in toomanyobjs==1.1.0 \ --hash=sha256:99e27468f9dad19127be9e2fb086b42acd69aed9ad7e63cef74d6e4389be0534 \ From a52465a54fcbef3ed04b8b3bfa76312e6e40efe4 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 6 Apr 2020 08:51:20 -0500 Subject: [PATCH 022/105] work on lan --- src/httpapi/sse/README.md | 5 ++- src/lan/__init__.py | 1 - src/lan/client/__init__.py | 43 ++++++++++++++++++++--- src/lan/discover.py | 16 ++++----- src/lan/server/__init__.py | 13 +++++-- src/onionrblocks/__init__.py | 2 +- src/onionrblocks/blocklist.py | 41 +++++++++++++++++++++ src/onionrutils/basicrequests.py | 9 ++--- src/onionrutils/blockmetadata/fromdata.py | 3 +- src/runtests/lanservertest.py | 2 +- static-data/default_config.json | 16 ++++----- tests/runtime-result.txt | 2 +- tests/test_blocklist_class.py | 31 ++++++++++++++++ 13 files changed, 151 insertions(+), 33 deletions(-) create mode 100644 src/onionrblocks/blocklist.py create mode 100644 tests/test_blocklist_class.py diff --git a/src/httpapi/sse/README.md b/src/httpapi/sse/README.md index b01ca67a..9f3f6a41 100644 --- a/src/httpapi/sse/README.md +++ b/src/httpapi/sse/README.md @@ -1,3 +1,6 @@ # sse -This folder contains a wrapper for handling server sent event loops \ No newline at end of file +This folder contains a wrapper for handling server sent event loops + + + diff --git a/src/lan/__init__.py b/src/lan/__init__.py index 230312df..15a106f4 100644 --- a/src/lan/__init__.py +++ b/src/lan/__init__.py @@ -36,5 +36,4 @@ class LANManager: def start(self): Thread(target=learn_services, args=[self.too_many.get(Client)], daemon=True).start() Thread(target=advertise_service, daemon=True).start() - Thread(target=self.too_many.get(Client, (self.peers,)).start, daemon=True).start() diff --git a/src/lan/client/__init__.py b/src/lan/client/__init__.py index b7d5b913..77562cac 100644 --- a/src/lan/client/__init__.py +++ b/src/lan/client/__init__.py @@ -4,9 +4,15 @@ LAN transport client thread """ from typing import List +import watchdog +from requests.exceptions import ConnectionError + from onionrcrypto.cryptoutils.randomshuffle import random_shuffle from utils.bettersleep import better_sleep from onionrutils.basicrequests import do_post_request, do_get_request +from threading import Thread +from onionrblocks import BlockList +from onionrblocks.blockimporter import import_block_from_data """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -29,17 +35,46 @@ class Client: self.lookup_time = {} self.poll_delay = 10 + def get_lookup_time(self, peer): try: return self.lookup_time[peer] except KeyError: return 0 - def start(self): + def peer_work(self, peer): + port = 1024 + + self.peers.append(peer) + for port in range(port, 65535): + print(port) + try: + if do_get_request(f'http://{peer}:{port}/ping', proxyType='lan', ignoreAPI=True, connect_timeout=0.3) == 'onionr!': + port = port + print(f'{peer}:{port} found') + break + except (AttributeError, ConnectionError): + pass + else: + self.peers.remove(peer) + return + self.peers.append(peer) + while True: - self.peers = random_shuffle(self.peers) + block_list = self._too_many.get(BlockList).get() + last_time = self.get_lookup_time(peer) + new_blocks = set('\n'.join(do_get_request(f'http://{peer}:{port}/blist/{last_time}', proxyType='lan', ignoreAPI=True))) ^ set(block_list) + for bl in new_blocks: + import_block_from_data( + do_get_request( + f'http://{peer}:{port}/get/{bl}', proxyType='lan', ignoreAPI=True)) + better_sleep(10) + self.peers.remove(peer) - - better_sleep(self.poll_delay) + def connect_peer(self, peer): + if peer in self.peers: + return + print(f'connecting to {peer}') + Thread(target=self.peer_work, args=[peer], daemon=True).start() diff --git a/src/lan/discover.py b/src/lan/discover.py index 3f6ed819..33c87551 100644 --- a/src/lan/discover.py +++ b/src/lan/discover.py @@ -9,7 +9,7 @@ from typing import List from ipaddress import ip_address from socket import SHUT_RDWR -from .getip import lan_ips +from .getip import lan_ips, best_ip from utils.bettersleep import better_sleep """ This program is free software: you can redistribute it and/or modify @@ -52,17 +52,17 @@ def learn_services(lan_client): if 'onionr' not in service_ips: continue service_ips = service_ips.replace('onionr-', '').split('-') + print(service_ips) port = 0 for service in service_ips: try: ip_address(service) if not ip_address(service).is_private: raise ValueError if service in lan_ips: raise ValueError - if service in lan_client.peers: raise ValueError except ValueError: - service_ips.remove(service) - p = list(lan_client.peers) - lan_client.peers = list(set(service_ips + p)) + pass + else: + lan_client.connect_peer(service) def advertise_service(specific_ips=None): @@ -71,10 +71,8 @@ def advertise_service(specific_ips=None): # for all packets sent, after three hops on the network the packet will not # be re-sent/broadcast (see https://www.tldp.org/HOWTO/Multicast-HOWTO-6.html) MULTICAST_TTL = 3 - if specific_ips is None: - ips = '-'.join(lan_ips) - else: - ips = '-'.join(specific_ips) + + ips = best_ip sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, MULTICAST_TTL) diff --git a/src/lan/server/__init__.py b/src/lan/server/__init__.py index fe45a386..8c8ea737 100644 --- a/src/lan/server/__init__.py +++ b/src/lan/server/__init__.py @@ -6,6 +6,7 @@ from gevent.pywsgi import WSGIServer from flask import Flask from flask import Response from flask import request +from flask import abort from onionrblocks.onionrblockapi import Block from httpapi.fdsafehandler import FDSafeHandler @@ -15,6 +16,7 @@ from coredb.blockmetadb import get_block_list from lan.getip import best_ip from onionrutils import stringvalidators from httpapi.miscpublicapi.upload import accept_upload +import logger """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -41,6 +43,13 @@ class LANServer: self.host = best_ip self.port = None + @app.before_request + def dns_rebinding_prevention(): + if request.host != f'{self.host}:{self.port}': + logger.warn('Potential DNS rebinding attack on LAN server:') + logger.warn(f'Hostname {request.host} was used instead of {self.host}:{self.port}') + abort(403) + @app.route('/blist/