From dac0da773a92b9c8e657b8d80e10a951a3e9ffb5 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 5 Jul 2019 03:38:08 -0500 Subject: [PATCH] GET requests partly doen --- .gitignore | 2 +- streamedrequests/__init__.py | 14 ++++-------- streamedrequests/dodownload.py | 38 ++++++++++++++++++++++++++++++++ streamedrequests/exceptions.py | 2 ++ streamedrequests/get.py | 34 ++++++++++++++++++++++++++++ streamedrequests/post.py | 21 ++++++++++++++++++ streamedrequests/responsesize.py | 30 +++++++++++++++++++++++++ tests/test_basic.py | 23 +++++++++++++++---- 8 files changed, 149 insertions(+), 15 deletions(-) create mode 100644 streamedrequests/dodownload.py create mode 100644 streamedrequests/exceptions.py create mode 100644 streamedrequests/get.py create mode 100644 streamedrequests/post.py create mode 100644 streamedrequests/responsesize.py diff --git a/.gitignore b/.gitignore index 29da480..c3669a6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ venv/* -deadsimplekv/__pycache__/* +streamedrequests/__pycache__/* diff --git a/streamedrequests/__init__.py b/streamedrequests/__init__.py index aa631c7..f9b7bea 100644 --- a/streamedrequests/__init__.py +++ b/streamedrequests/__init__.py @@ -15,13 +15,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -class Request: - def __init__(self, prepared_request): - return - -def get_request(url, query_parameters=None, request_headers=None, -sync=True, max_size=0, connect_timeout=0, stream_timeout=0, proxy=None): - return - -def post_request(): - return \ No newline at end of file +from requests import Request, session +from . import get, post +get = get.get +post = post.post \ No newline at end of file diff --git a/streamedrequests/dodownload.py b/streamedrequests/dodownload.py new file mode 100644 index 0000000..20ca830 --- /dev/null +++ b/streamedrequests/dodownload.py @@ -0,0 +1,38 @@ +''' + StreamedRequests. A simple library for streaming HTTP requests + Copyright (C) 2019 Kevin Froman https://chaoswebs.net/ + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import threading + +def __run_callback(data, sync, callback=None): + if callback is None: + return + if sync: + callback(data) + else: + threading.Thread(target=callback, args=(data,)).start() + +def __do_download(req, max_size, chunk_size, callback, sync): + ret_data = b'' + if chunk_size == 0: + raise ValueError("Chunk size cannot be zero") + for chunk in req.iter_content(chunk_size=chunk_size): + if max_size > 0: + chunk_count.add(chunk_size) + if not callback is None: + __run_callback(chunk, sync, callback) + ret_data += chunk + return ret_data \ No newline at end of file diff --git a/streamedrequests/exceptions.py b/streamedrequests/exceptions.py new file mode 100644 index 0000000..57dcb27 --- /dev/null +++ b/streamedrequests/exceptions.py @@ -0,0 +1,2 @@ +class ResponseLimitReached(Exception): + '''For when the response of a streamed request exceeds the max''' \ No newline at end of file diff --git a/streamedrequests/get.py b/streamedrequests/get.py new file mode 100644 index 0000000..b3f1be6 --- /dev/null +++ b/streamedrequests/get.py @@ -0,0 +1,34 @@ +''' + StreamedRequests. A simple library for streaming HTTP requests + Copyright (C) 2019 Kevin Froman https://chaoswebs.net/ + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import requests +from . import exceptions, responsesize, dodownload +def get(url, query_parameters=None, request_headers=None, sync=True, + max_size=0, chunk_size=1000, connect_timeout=60, stream_timeout=0, + proxy=None, callback=None, allow_redirects=True): + + chunk_count = responsesize.SizeValidator(max_size) # Class to verify if the stream is staying within the max_size + timeouts = (connect_timeout, stream_timeout) + + if stream_timeout == 0: # If timeout for stream is default, use connect timeout for both + timeouts = connect_timeout + + req = requests.get(url, params=query_parameters, headers=request_headers, + timeout=timeouts, stream=True, allow_redirects=allow_redirects) + + return dodownload.__do_download(req, max_size, chunk_size, callback, sync) + \ No newline at end of file diff --git a/streamedrequests/post.py b/streamedrequests/post.py new file mode 100644 index 0000000..163299c --- /dev/null +++ b/streamedrequests/post.py @@ -0,0 +1,21 @@ +''' + StreamedRequests. A simple library for streaming HTTP requests + Copyright (C) 2019 Kevin Froman https://chaoswebs.net/ + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import requests +def post(url, post_data=None, request_headers=None, sync=True, + max_size=0, connect_timeout=0, stream_timeout=0, proxy=None): + return \ No newline at end of file diff --git a/streamedrequests/responsesize.py b/streamedrequests/responsesize.py new file mode 100644 index 0000000..1171f5d --- /dev/null +++ b/streamedrequests/responsesize.py @@ -0,0 +1,30 @@ +''' + StreamedRequests. A simple library for streaming HTTP requests + Copyright (C) 2019 Kevin Froman https://chaoswebs.net/ + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +from . import exceptions +class SizeValidator: + def __init__(self, max_size): + self.max_size = 0 + self.size = 0 + + def add(self, amount): + self.size += amount + if self.size >= self.max_size: + raise exceptions.ResponseLimitReached("The request has reached the maximum size limit of %s" % [self.max_size]) + + def reset(self): + self.size = 0 \ No newline at end of file diff --git a/tests/test_basic.py b/tests/test_basic.py index b207479..e8bb4b1 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -22,12 +22,27 @@ import streamedrequests def get_test_id(): return str(uuid.uuid4()) + '.dat' +def _test_callback(text): + print('got', text) + class TestInit(unittest.TestCase): - def test_init(self): - streamedrequests.get_request("test", request_headers=None, sync=True, max_size=0, - connect_timeout=0, stream_timeout=0, proxy=None) + def test_basic(self): + streamedrequests.get('https://example.com/') - return + def test_callback(self): + pass + streamedrequests.get('https://example.com/', chunk_size=1, callback=_test_callback) + + def test_async(self): + streamedrequests.get('https://example.com/', chunk_size=1, callback=_test_callback, sync=False) + + def test_zero_chunk_size(self): + try: + streamedrequests.get('https://example.com/', chunk_size=0) + except ValueError: + pass + else: + self.assertTrue(failUnless) unittest.main() \ No newline at end of file