based block size protection added

onionr/communicator.py

@@ -301,7 +301,7 @@ class OnionrCommunicatorDaemon:
                 score = str(self.getPeerProfileInstance(i).score)
                 logger.info(i + ', score: ' + score, terminal=True)
 
-    def peerAction(self, peer, action, data='', returnHeaders=False):
+    def peerAction(self, peer, action, data='', returnHeaders=False, max_resp_size=5242880):
         '''Perform a get request to a peer'''
         if len(peer) == 0:
             return False

onionr/communicatorutils/downloadblocks/__init__.py

@@ -20,20 +20,7 @@
 import communicator, onionrexceptions
 import logger, onionrpeers
 from onionrutils import blockmetadata, stringvalidators, validatemetadata
-
-def _should_download(comm_inst, block_hash):
-    ret_data = True
-    if block_hash in comm_inst._core.getBlockList():
-        ret_data = False
-    else:
-        if comm_inst._core._blacklist.inBlacklist(block_hash):
-            ret_data = False
-    if ret_data is False:
-        try:
-            del comm_inst.blockQueue[block_hash]
-        except KeyError:
-            pass
-    return ret_data
+from . import shoulddownload
 
 def download_blocks_from_communicator(comm_inst):
     assert isinstance(comm_inst, communicator.OnionrCommunicatorDaemon)
@@ -47,13 +34,13 @@ def download_blocks_from_communicator(comm_inst):
             blockPeers = []
         removeFromQueue = True
 
-        if _should_download(comm_inst, blockHash):
+        if shoulddownload.should_download(comm_inst, blockHash):
             continue
 
         if comm_inst.shutdown or not comm_inst.isOnline or comm_inst._core.storage_counter.isFull():
             # Exit loop if shutting down or offline, or disk allocation reached
             break
-        # Do not download blocks being downloaded or that are already saved (edge cases)
+        # Do not download blocks being downloaded
         if blockHash in comm_inst.currentDownloading:
             #logger.debug('Already downloading block %s...' % blockHash)
             continue
@@ -67,7 +54,7 @@ def download_blocks_from_communicator(comm_inst):
 
         if not comm_inst.shutdown and peerUsed.strip() != '':
             logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed))
-        content = comm_inst.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata)
+        content = comm_inst.peerAction(peerUsed, 'getdata/' + blockHash, max_resp_size=3000000) # block content from random peer (includes metadata)
         if content != False and len(content) > 0:
             try:
                 content = content.encode()

onionr/communicatorutils/downloadblocks/shoulddownload.py

@@ -0,0 +1,33 @@
+'''
+    Onionr - Private P2P Communication
+
+    Check if a block should be downloaded (if we already have it or its blacklisted or not)
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+def should_download(comm_inst, block_hash):
+    ret_data = True
+    if block_hash in comm_inst._core.getBlockList(): # Dont download block we have
+        ret_data = False
+    else:
+        if comm_inst._core._blacklist.inBlacklist(block_hash): # Dont download blacklisted block
+            ret_data = False
+    if ret_data is False:
+        # Remove block from communicator queue if it shouldnt be downloaded
+        try:
+            del comm_inst.blockQueue[block_hash]
+        except KeyError:
+            pass
+    return ret_data

onionr/onionrutils/basicrequests.py

@@ -17,7 +17,7 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import requests
+import requests, streamedrequests
 import logger, onionrexceptions
 def do_post_request(core_inst, url, data={}, port=0, proxyType='tor'):
     '''
@@ -43,7 +43,7 @@ def do_post_request(core_inst, url, data={}, port=0, proxyType='tor'):
         retData = False
     return retData
 
-def do_get_request(core_inst, url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False):
+def do_get_request(core_inst, url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False, max_size=5242880):
     '''
     Do a get request through a local tor or i2p instance
     '''
@@ -61,16 +61,16 @@ def do_get_request(core_inst, url, port=0, proxyType='tor', ignoreAPI=False, ret
     response_headers = dict()
     try:
         proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
-        r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30), )
+        r = streamedrequests.get(url, request_headers=headers, allow_redirects=False, proxy=proxies, connect_timeout=15, max_size=max_size)
         # Check server is using same API version as us
         if not ignoreAPI:
             try:
-                response_headers = r.headers
-                if r.headers['X-API'] != str(API_VERSION):
+                response_headers = r[0].headers
+                if r[0].headers['X-API'] != str(API_VERSION):
                     raise onionrexceptions.InvalidAPIVersion
             except KeyError:
                 raise onionrexceptions.InvalidAPIVersion
-        retData = r.text
+        retData = r[1]
     except KeyboardInterrupt:
         raise KeyboardInterrupt
     except ValueError as e:

requirements.in

@@ -7,5 +7,5 @@ PySocks==1.6.8
 stem==1.7.1
 deadsimplekv==0.1.1
 unpaddedbase32==0.1.0
-streamedrequests==0.0.0
+streamedrequests==1.0.0
 jinja2==2.10.1

requirements.txt

@@ -171,8 +171,8 @@ six==1.12.0 \
     # via pynacl
 stem==1.7.1 \
     --hash=sha256:c9eaf3116cb60c15995cbd3dec3a5cbc50e9bb6e062c4d6d42201e566f498ca2
-streamedrequests==0.0.0 \
-    --hash=sha256:94c729078099c0f2691510be0bb90f421277a729078b8e150fe2740c78f8e53e
+streamedrequests==1.0.0 \
+    --hash=sha256:1d9d07394804a6e1fd66bde74a804e71cab98e6920053865574a459f1cf7d3b7
 unpaddedbase32==0.1.0 \
     --hash=sha256:5e4143fcaf77c9c6b4f60d18301c7570f0dac561dcf9b9aed8b5ba6ead7f218c
 urllib3==1.24.2 \