From e86f154522e7541c3a9b85becf1892d60594f9e3 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 13 Aug 2019 17:28:53 -0500 Subject: [PATCH] * prevent the same message from being inserted at the same time in the http api * bumped toomanyobjs * correct pysocks raise fixes #34 --- onionr/__init__.py | 4 ++-- onionr/communicator/__init__.py | 2 ++ onionr/communicatorutils/daemonqueuehandler.py | 2 ++ onionr/communicatorutils/downloadblocks/__init__.py | 4 +++- onionr/httpapi/insertblock.py | 13 +++++++++++-- onionr/httpapi/security/client.py | 8 +++++++- onionr/onionrblocks/insert.py | 1 + onionr/static-data/www/board/board.js | 6 +++++- requirements.in | 2 +- requirements.txt | 4 ++-- 10 files changed, 36 insertions(+), 10 deletions(-) diff --git a/onionr/__init__.py b/onionr/__init__.py index f4a6d36a..61e5c579 100755 --- a/onionr/__init__.py +++ b/onionr/__init__.py @@ -31,8 +31,8 @@ import sys # Ensure that PySocks is installed try: from urllib3.contrib.socks import SOCKSProxyManager -except ImportError: - raise ImportError("You need the PySocks module (for use with socks5 proxy to use Tor)") +except ModuleNotFoundError: + raise ModuleNotFoundError("You need the PySocks module (for use with socks5 proxy to use Tor)") # Onionr imports from etc import onionrvalues # For different Onionr related constants such as versions diff --git a/onionr/communicator/__init__.py b/onionr/communicator/__init__.py index 01c61c1e..27dbf806 100755 --- a/onionr/communicator/__init__.py +++ b/onionr/communicator/__init__.py @@ -65,6 +65,8 @@ class OnionrCommunicatorDaemon: self.announceProgress = {} self.announceCache = {} + self.generating_blocks = [] + # amount of threads running by name, used to prevent too many self.threadCounts = {} diff --git a/onionr/communicatorutils/daemonqueuehandler.py b/onionr/communicatorutils/daemonqueuehandler.py index 007b8603..ea29435e 100755 --- a/onionr/communicatorutils/daemonqueuehandler.py +++ b/onionr/communicatorutils/daemonqueuehandler.py @@ -30,6 +30,8 @@ def handle_daemon_commands(comm_inst): events.event('daemon_command', data = {'cmd' : cmd}) if cmd[0] == 'shutdown': comm_inst.shutdown = True + elif cmd[0] == 'remove_from_insert_list': + comm_inst.generating_blocks.remove(cmd[1]) elif cmd[0] == 'announceNode': if len(comm_inst.onlinePeers) > 0: comm_inst.announce(cmd[1]) diff --git a/onionr/communicatorutils/downloadblocks/__init__.py b/onionr/communicatorutils/downloadblocks/__init__.py index 2c20a3c1..3cffa526 100755 --- a/onionr/communicatorutils/downloadblocks/__init__.py +++ b/onionr/communicatorutils/downloadblocks/__init__.py @@ -25,11 +25,13 @@ from . import shoulddownload from communicator import peeraction, onlinepeers import onionrcrypto, onionrstorage, onionrblacklist, storagecounter def download_blocks_from_communicator(comm_inst): + '''Use Onionr communicator instance to download blocks in the communicator's queue''' assert isinstance(comm_inst, communicator.OnionrCommunicatorDaemon) blacklist = onionrblacklist.OnionrBlackList() storage_counter = storagecounter.StorageCounter() - LOG_SKIP_COUNT = 10 + LOG_SKIP_COUNT = 10 # for how many iterations we skip logging the counter count = 0 + # Iterate the block queue in the communicator for blockHash in list(comm_inst.blockQueue): count += 1 if len(comm_inst.onlinePeers) == 0: diff --git a/onionr/httpapi/insertblock.py b/onionr/httpapi/insertblock.py index 21ed1d42..ddba750f 100644 --- a/onionr/httpapi/insertblock.py +++ b/onionr/httpapi/insertblock.py @@ -18,8 +18,10 @@ along with this program. If not, see . ''' import json, threading -from flask import Blueprint, Response, request +from flask import Blueprint, Response, request, g import onionrblocks +from onionrcrypto import hashers +from onionrutils import bytesconverter ib = Blueprint('insertblock', __name__) @ib.route('/insertblock', methods=['POST']) @@ -27,10 +29,17 @@ def client_api_insert_block(): encrypt = False bData = request.get_json(force=True) message = bData['message'] + message_hash = bytesconverter.bytes_to_str(hashers.sha3_hash(message)) # Detect if message (block body) is not specified if type(message) is None: - return 'failure', 406 + return 'failure due to unspecified message', 400 + + # Detect if block with same message is already being inserted + if message_hash in g.too_many.get_by_string("OnionrCommunicatorDaemon").generating_blocks: + return 'failure due to duplicate insert', 400 + else: + g.too_many.get_by_string("OnionrCommunicatorDaemon").generating_blocks.append(message_hash) subject = 'temp' encryptType = '' diff --git a/onionr/httpapi/security/client.py b/onionr/httpapi/security/client.py index aa12a8f1..0019c4e5 100644 --- a/onionr/httpapi/security/client.py +++ b/onionr/httpapi/security/client.py @@ -18,7 +18,7 @@ along with this program. If not, see . ''' import hmac -from flask import Blueprint, request, abort +from flask import Blueprint, request, abort, g from onionrservices import httpheaders # Be extremely mindful of this. These are endpoints available without a password whitelist_endpoints = ('siteapi.site', 'www', 'staticfiles.onionrhome', 'staticfiles.homedata', @@ -49,6 +49,12 @@ class ClientAPISecurity: except KeyError: if not hmac.compare_digest(request.form['token'], client_api.clientToken): abort(403) + + # Add shared objects + try: + g.too_many = self.client_api._too_many + except KeyError: + g.too_many = None @client_api_security_bp.after_app_request def after_req(resp): diff --git a/onionr/onionrblocks/insert.py b/onionr/onionrblocks/insert.py index d297f706..68be1360 100644 --- a/onionr/onionrblocks/insert.py +++ b/onionr/onionrblocks/insert.py @@ -143,4 +143,5 @@ def insert_block(data, header='txt', sign=False, encryptType='', symKey='', asym events.event('insertdeniable', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': bytesconverter.bytes_to_str(asymPeer)}, threaded = True) else: events.event('insertblock', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': bytesconverter.bytes_to_str(asymPeer)}, threaded = True) + coredb.daemonqueue.daemon_queue_add('remove_from_insert_list', data=dataNonce) return retData \ No newline at end of file diff --git a/onionr/static-data/www/board/board.js b/onionr/static-data/www/board/board.js index 38e86ff0..7b018454 100755 --- a/onionr/static-data/www/board/board.js +++ b/onionr/static-data/www/board/board.js @@ -107,7 +107,11 @@ newPostForm.onsubmit = function(){ .then((resp) => resp.text()) // Transform the data into json .then(function(data) { newPostForm.style.display = 'block' - alert('Queued for submission!') + if (data == 'failure due to duplicate insert'){ + alert('This message is already queued') + return + } + alert('Queued for submission! ' + data) setTimeout(function(){getBlocks()}, 3000) }) return false diff --git a/requirements.in b/requirements.in index 83f56118..8ad2c519 100644 --- a/requirements.in +++ b/requirements.in @@ -9,4 +9,4 @@ deadsimplekv==0.1.1 unpaddedbase32==0.1.0 streamedrequests==1.0.0 jinja2==2.10.1 -toomanyobjs==1.0.0 +toomanyobjs==1.1.0 diff --git a/requirements.txt b/requirements.txt index c07870bf..3fc3671c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -173,8 +173,8 @@ stem==1.7.1 \ --hash=sha256:c9eaf3116cb60c15995cbd3dec3a5cbc50e9bb6e062c4d6d42201e566f498ca2 streamedrequests==1.0.0 \ --hash=sha256:1d9d07394804a6e1fd66bde74a804e71cab98e6920053865574a459f1cf7d3b7 -toomanyobjs==1.0.0 \ - --hash=sha256:040390188063dd00e5d903fd82a08850a175f9a384e09880d50acffb1e60ca70 +toomanyobjs==1.1.0 \ + --hash=sha256:99e27468f9dad19127be9e2fb086b42acd69aed9ad7e63cef74d6e4389be0534 unpaddedbase32==0.1.0 \ --hash=sha256:5e4143fcaf77c9c6b4f60d18301c7570f0dac561dcf9b9aed8b5ba6ead7f218c urllib3==1.24.2 \