refactored timeouts, added http server for testing

This commit is contained in:
Kevin Froman 2019-07-06 02:36:44 -05:00
parent dac0da773a
commit cb8353336e
4 changed files with 49 additions and 14 deletions

View File

@ -16,19 +16,17 @@
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import requests import requests
from . import exceptions, responsesize, dodownload from . import exceptions, responsesize, dodownload, setuptimeout
def get(url, query_parameters=None, request_headers=None, sync=True, def get(url, query_parameters=None, request_headers=None, sync=True,
max_size=0, chunk_size=1000, connect_timeout=60, stream_timeout=0, max_size=0, chunk_size=1000, connect_timeout=60, stream_timeout=0,
proxy=None, callback=None, allow_redirects=True): proxy={}, callback=None, allow_redirects=True):
chunk_count = responsesize.SizeValidator(max_size) # Class to verify if the stream is staying within the max_size chunk_count = responsesize.SizeValidator(max_size) # Class to verify if the stream is staying within the max_size
timeouts = (connect_timeout, stream_timeout)
if stream_timeout == 0: # If timeout for stream is default, use connect timeout for both timeouts = setuptimeout.__setup_timeout(connect_timeout, stream_timeout)
timeouts = connect_timeout
req = requests.get(url, params=query_parameters, headers=request_headers, req = requests.get(url, params=query_parameters, headers=request_headers,
timeout=timeouts, stream=True, allow_redirects=allow_redirects) timeout=timeouts, stream=True, allow_redirects=allow_redirects, proxies=proxy)
return dodownload.__do_download(req, max_size, chunk_size, callback, sync) return dodownload.__do_download(req, max_size, chunk_size, callback, sync)

View File

@ -17,5 +17,6 @@
''' '''
import requests import requests
def post(url, post_data=None, request_headers=None, sync=True, def post(url, post_data=None, request_headers=None, sync=True,
max_size=0, connect_timeout=0, stream_timeout=0, proxy=None): max_size=0, chunk_size=1000, connect_timeout=60, stream_timeout=0,
proxy={}, callback=None, allow_redirects=True):
return return

View File

@ -0,0 +1,23 @@
'''
StreamedRequests. A simple library for streaming HTTP requests
Copyright (C) 2019 Kevin Froman https://chaoswebs.net/
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
def __setup_timeout(connect_timeout, stream_timeout):
timeouts = (connect_timeout, stream_timeout)
if stream_timeout == 0: # If timeout for stream is default, use connect timeout for both
timeouts = connect_timeout
return timeouts

View File

@ -15,34 +15,47 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import sys, os, unittest import sys, os, unittest, threading
from http.server import HTTPServer, SimpleHTTPRequestHandler
import requests
sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/../") sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/../")
import streamedrequests import streamedrequests
def get_test_id(): def get_test_id():
return str(uuid.uuid4()) + '.dat' return str(uuid.uuid4()) + '.dat'
def run(server_class=HTTPServer, handler_class=SimpleHTTPRequestHandler):
server_address = ('127.0.0.1', 8000)
httpd = server_class(server_address, handler_class)
httpd.serve_forever()
def _test_callback(text): def _test_callback(text):
print('got', text) return
#print('got', text)
class TestInit(unittest.TestCase): class TestInit(unittest.TestCase):
def test_requests(self):
requests.get('http://127.0.0.1:8000/')
def test_basic(self): def test_basic(self):
streamedrequests.get('https://example.com/') streamedrequests.get('http://127.0.0.1:8000/')
def test_callback(self): def test_callback(self):
pass pass
streamedrequests.get('https://example.com/', chunk_size=1, callback=_test_callback) streamedrequests.get('http://127.0.0.1:8000/', chunk_size=1, callback=_test_callback)
def test_async(self): def test_async(self):
streamedrequests.get('https://example.com/', chunk_size=1, callback=_test_callback, sync=False) streamedrequests.get('http://127.0.0.1:8000/', chunk_size=1, callback=_test_callback, sync=False)
def test_zero_chunk_size(self): def test_zero_chunk_size(self):
try: try:
streamedrequests.get('https://example.com/', chunk_size=0) streamedrequests.get('http://127.0.0.1:8000/', chunk_size=0)
except ValueError: except ValueError:
pass pass
else: else:
self.assertTrue(failUnless) self.assertTrue(failUnless)
threading.Thread(target=run, daemon=True).start()
unittest.main() unittest.main()