GET requests partly doen

.gitignore

@@ -1,2 +1,2 @@
 venv/*
-deadsimplekv/__pycache__/*
+streamedrequests/__pycache__/*

streamedrequests/__init__.py

@@ -15,13 +15,7 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-class Request:
-    def __init__(self, prepared_request):
-        return
-
-def get_request(url, query_parameters=None, request_headers=None, 
-sync=True, max_size=0, connect_timeout=0, stream_timeout=0, proxy=None):
-    return
-
-def post_request():
-    return
+from requests import Request, session
+from . import get, post
+get = get.get
+post = post.post

streamedrequests/dodownload.py

@@ -0,0 +1,38 @@
+'''
+    StreamedRequests. A simple library for streaming HTTP requests
+    Copyright (C) 2019 Kevin Froman https://chaoswebs.net/
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import threading
+
+def __run_callback(data, sync, callback=None):
+    if callback is None:
+        return
+    if sync:
+        callback(data)
+    else:
+        threading.Thread(target=callback, args=(data,)).start()
+    
+def __do_download(req, max_size, chunk_size, callback, sync):
+    ret_data = b''
+    if chunk_size == 0:
+        raise ValueError("Chunk size cannot be zero")
+    for chunk in req.iter_content(chunk_size=chunk_size):
+        if max_size > 0:
+            chunk_count.add(chunk_size)
+        if not callback is None:
+            __run_callback(chunk, sync, callback)
+        ret_data += chunk
+    return ret_data

streamedrequests/exceptions.py

@@ -0,0 +1,2 @@
+class ResponseLimitReached(Exception):
+    '''For when the response of a streamed request exceeds the max'''

streamedrequests/get.py

@@ -0,0 +1,34 @@
+'''
+    StreamedRequests. A simple library for streaming HTTP requests
+    Copyright (C) 2019 Kevin Froman https://chaoswebs.net/
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import requests
+from . import exceptions, responsesize, dodownload
+def get(url, query_parameters=None, request_headers=None, sync=True, 
+    max_size=0, chunk_size=1000, connect_timeout=60, stream_timeout=0,
+    proxy=None, callback=None, allow_redirects=True):
+
+    chunk_count = responsesize.SizeValidator(max_size) # Class to verify if the stream is staying within the max_size
+    timeouts = (connect_timeout, stream_timeout)
+
+    if stream_timeout == 0: # If timeout for stream is default, use connect timeout for both
+        timeouts = connect_timeout
+    
+    req = requests.get(url, params=query_parameters, headers=request_headers,
+        timeout=timeouts, stream=True, allow_redirects=allow_redirects)
+    
+    return dodownload.__do_download(req, max_size, chunk_size, callback, sync)
+    

streamedrequests/post.py

@@ -0,0 +1,21 @@
+'''
+    StreamedRequests. A simple library for streaming HTTP requests
+    Copyright (C) 2019 Kevin Froman https://chaoswebs.net/
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import requests
+def post(url, post_data=None, request_headers=None, sync=True, 
+        max_size=0, connect_timeout=0, stream_timeout=0, proxy=None):
+        return

streamedrequests/responsesize.py

@@ -0,0 +1,30 @@
+'''
+    StreamedRequests. A simple library for streaming HTTP requests
+    Copyright (C) 2019 Kevin Froman https://chaoswebs.net/
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from . import exceptions
+class SizeValidator:
+    def __init__(self, max_size):
+        self.max_size = 0
+        self.size = 0
+
+    def add(self, amount):
+        self.size += amount
+        if self.size >= self.max_size:
+            raise exceptions.ResponseLimitReached("The request has reached the maximum size limit of %s" % [self.max_size])
+    
+    def reset(self):
+        self.size = 0

tests/test_basic.py

@@ -22,12 +22,27 @@ import streamedrequests
 def get_test_id():
     return str(uuid.uuid4()) + '.dat'
 
+def _test_callback(text):
+    print('got', text)
+
 class TestInit(unittest.TestCase):
 
-    def test_init(self):
-        streamedrequests.get_request("test", request_headers=None, sync=True, max_size=0,
-        connect_timeout=0, stream_timeout=0, proxy=None)
+    def test_basic(self):
+        streamedrequests.get('https://example.com/')
 
-        return
+    def test_callback(self):
+        pass
+        streamedrequests.get('https://example.com/', chunk_size=1, callback=_test_callback)
+    
+    def test_async(self):
+        streamedrequests.get('https://example.com/', chunk_size=1, callback=_test_callback, sync=False)
+
+    def test_zero_chunk_size(self):
+        try:
+            streamedrequests.get('https://example.com/', chunk_size=0)
+        except ValueError:
+            pass
+        else:
+            self.assertTrue(failUnless)
 
 unittest.main()