based block size protection added
This commit is contained in:
Kevin Froman
parent
fe142528ca
commit
b822a74c53
|
|
@ -301,7 +301,7 @@ class OnionrCommunicatorDaemon:
|
|||
score = str(self.getPeerProfileInstance(i).score)
|
||||
logger.info(i + ', score: ' + score, terminal=True)
|
||||
|
||||
def peerAction(self, peer, action, data='', returnHeaders=False):
|
||||
def peerAction(self, peer, action, data='', returnHeaders=False, max_resp_size=5242880):
|
||||
'''Perform a get request to a peer'''
|
||||
if len(peer) == 0:
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -20,20 +20,7 @@
|
|||
import communicator, onionrexceptions
|
||||
import logger, onionrpeers
|
||||
from onionrutils import blockmetadata, stringvalidators, validatemetadata
|
||||
|
||||
def _should_download(comm_inst, block_hash):
|
||||
ret_data = True
|
||||
if block_hash in comm_inst._core.getBlockList():
|
||||
ret_data = False
|
||||
else:
|
||||
if comm_inst._core._blacklist.inBlacklist(block_hash):
|
||||
ret_data = False
|
||||
if ret_data is False:
|
||||
try:
|
||||
del comm_inst.blockQueue[block_hash]
|
||||
except KeyError:
|
||||
pass
|
||||
return ret_data
|
||||
from . import shoulddownload
|
||||
|
||||
def download_blocks_from_communicator(comm_inst):
|
||||
assert isinstance(comm_inst, communicator.OnionrCommunicatorDaemon)
|
||||
|
|
@ -47,13 +34,13 @@ def download_blocks_from_communicator(comm_inst):
|
|||
blockPeers = []
|
||||
removeFromQueue = True
|
||||
|
||||
if _should_download(comm_inst, blockHash):
|
||||
if shoulddownload.should_download(comm_inst, blockHash):
|
||||
continue
|
||||
|
||||
if comm_inst.shutdown or not comm_inst.isOnline or comm_inst._core.storage_counter.isFull():
|
||||
# Exit loop if shutting down or offline, or disk allocation reached
|
||||
break
|
||||
# Do not download blocks being downloaded or that are already saved (edge cases)
|
||||
# Do not download blocks being downloaded
|
||||
if blockHash in comm_inst.currentDownloading:
|
||||
#logger.debug('Already downloading block %s...' % blockHash)
|
||||
continue
|
||||
|
|
@ -67,7 +54,7 @@ def download_blocks_from_communicator(comm_inst):
|
|||
|
||||
if not comm_inst.shutdown and peerUsed.strip() != '':
|
||||
logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed))
|
||||
content = comm_inst.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata)
|
||||
content = comm_inst.peerAction(peerUsed, 'getdata/' + blockHash, max_resp_size=3000000) # block content from random peer (includes metadata)
|
||||
if content != False and len(content) > 0:
|
||||
try:
|
||||
content = content.encode()
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
'''
|
||||
Onionr - Private P2P Communication
|
||||
|
||||
Check if a block should be downloaded (if we already have it or its blacklisted or not)
|
||||
'''
|
||||
'''
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
def should_download(comm_inst, block_hash):
|
||||
ret_data = True
|
||||
if block_hash in comm_inst._core.getBlockList(): # Dont download block we have
|
||||
ret_data = False
|
||||
else:
|
||||
if comm_inst._core._blacklist.inBlacklist(block_hash): # Dont download blacklisted block
|
||||
ret_data = False
|
||||
if ret_data is False:
|
||||
# Remove block from communicator queue if it shouldnt be downloaded
|
||||
try:
|
||||
del comm_inst.blockQueue[block_hash]
|
||||
except KeyError:
|
||||
pass
|
||||
return ret_data
|
||||
|
|
@ -17,7 +17,7 @@
|
|||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
import requests
|
||||
import requests, streamedrequests
|
||||
import logger, onionrexceptions
|
||||
def do_post_request(core_inst, url, data={}, port=0, proxyType='tor'):
|
||||
'''
|
||||
|
|
@ -43,7 +43,7 @@ def do_post_request(core_inst, url, data={}, port=0, proxyType='tor'):
|
|||
retData = False
|
||||
return retData
|
||||
|
||||
def do_get_request(core_inst, url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False):
|
||||
def do_get_request(core_inst, url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False, max_size=5242880):
|
||||
'''
|
||||
Do a get request through a local tor or i2p instance
|
||||
'''
|
||||
|
|
@ -61,16 +61,16 @@ def do_get_request(core_inst, url, port=0, proxyType='tor', ignoreAPI=False, ret
|
|||
response_headers = dict()
|
||||
try:
|
||||
proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
|
||||
r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30), )
|
||||
r = streamedrequests.get(url, request_headers=headers, allow_redirects=False, proxy=proxies, connect_timeout=15, max_size=max_size)
|
||||
# Check server is using same API version as us
|
||||
if not ignoreAPI:
|
||||
try:
|
||||
response_headers = r.headers
|
||||
if r.headers['X-API'] != str(API_VERSION):
|
||||
response_headers = r[0].headers
|
||||
if r[0].headers['X-API'] != str(API_VERSION):
|
||||
raise onionrexceptions.InvalidAPIVersion
|
||||
except KeyError:
|
||||
raise onionrexceptions.InvalidAPIVersion
|
||||
retData = r.text
|
||||
retData = r[1]
|
||||
except KeyboardInterrupt:
|
||||
raise KeyboardInterrupt
|
||||
except ValueError as e:
|
||||
|
|
|
|||
|
|
@ -7,5 +7,5 @@ PySocks==1.6.8
|
|||
stem==1.7.1
|
||||
deadsimplekv==0.1.1
|
||||
unpaddedbase32==0.1.0
|
||||
streamedrequests==0.0.0
|
||||
streamedrequests==1.0.0
|
||||
jinja2==2.10.1
|
||||
|
|
|
|||
|
|
@ -171,8 +171,8 @@ six==1.12.0 \
|
|||
# via pynacl
|
||||
stem==1.7.1 \
|
||||
--hash=sha256:c9eaf3116cb60c15995cbd3dec3a5cbc50e9bb6e062c4d6d42201e566f498ca2
|
||||
streamedrequests==0.0.0 \
|
||||
--hash=sha256:94c729078099c0f2691510be0bb90f421277a729078b8e150fe2740c78f8e53e
|
||||
streamedrequests==1.0.0 \
|
||||
--hash=sha256:1d9d07394804a6e1fd66bde74a804e71cab98e6920053865574a459f1cf7d3b7
|
||||
unpaddedbase32==0.1.0 \
|
||||
--hash=sha256:5e4143fcaf77c9c6b4f60d18301c7570f0dac561dcf9b9aed8b5ba6ead7f218c
|
||||
urllib3==1.24.2 \
|
||||
|
|
|
|||
Loading…
Reference in New Issue