GET requests partly doen

This commit is contained in:
Kevin Froman 2019-07-05 03:38:08 -05:00
parent c79012d183
commit dac0da773a
8 changed files with 149 additions and 15 deletions

2
.gitignore vendored
View File

@ -1,2 +1,2 @@
venv/* venv/*
deadsimplekv/__pycache__/* streamedrequests/__pycache__/*

View File

@ -15,13 +15,7 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
class Request: from requests import Request, session
def __init__(self, prepared_request): from . import get, post
return get = get.get
post = post.post
def get_request(url, query_parameters=None, request_headers=None,
sync=True, max_size=0, connect_timeout=0, stream_timeout=0, proxy=None):
return
def post_request():
return

View File

@ -0,0 +1,38 @@
'''
StreamedRequests. A simple library for streaming HTTP requests
Copyright (C) 2019 Kevin Froman https://chaoswebs.net/
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import threading
def __run_callback(data, sync, callback=None):
if callback is None:
return
if sync:
callback(data)
else:
threading.Thread(target=callback, args=(data,)).start()
def __do_download(req, max_size, chunk_size, callback, sync):
ret_data = b''
if chunk_size == 0:
raise ValueError("Chunk size cannot be zero")
for chunk in req.iter_content(chunk_size=chunk_size):
if max_size > 0:
chunk_count.add(chunk_size)
if not callback is None:
__run_callback(chunk, sync, callback)
ret_data += chunk
return ret_data

View File

@ -0,0 +1,2 @@
class ResponseLimitReached(Exception):
'''For when the response of a streamed request exceeds the max'''

34
streamedrequests/get.py Normal file
View File

@ -0,0 +1,34 @@
'''
StreamedRequests. A simple library for streaming HTTP requests
Copyright (C) 2019 Kevin Froman https://chaoswebs.net/
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import requests
from . import exceptions, responsesize, dodownload
def get(url, query_parameters=None, request_headers=None, sync=True,
max_size=0, chunk_size=1000, connect_timeout=60, stream_timeout=0,
proxy=None, callback=None, allow_redirects=True):
chunk_count = responsesize.SizeValidator(max_size) # Class to verify if the stream is staying within the max_size
timeouts = (connect_timeout, stream_timeout)
if stream_timeout == 0: # If timeout for stream is default, use connect timeout for both
timeouts = connect_timeout
req = requests.get(url, params=query_parameters, headers=request_headers,
timeout=timeouts, stream=True, allow_redirects=allow_redirects)
return dodownload.__do_download(req, max_size, chunk_size, callback, sync)

21
streamedrequests/post.py Normal file
View File

@ -0,0 +1,21 @@
'''
StreamedRequests. A simple library for streaming HTTP requests
Copyright (C) 2019 Kevin Froman https://chaoswebs.net/
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import requests
def post(url, post_data=None, request_headers=None, sync=True,
max_size=0, connect_timeout=0, stream_timeout=0, proxy=None):
return

View File

@ -0,0 +1,30 @@
'''
StreamedRequests. A simple library for streaming HTTP requests
Copyright (C) 2019 Kevin Froman https://chaoswebs.net/
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
from . import exceptions
class SizeValidator:
def __init__(self, max_size):
self.max_size = 0
self.size = 0
def add(self, amount):
self.size += amount
if self.size >= self.max_size:
raise exceptions.ResponseLimitReached("The request has reached the maximum size limit of %s" % [self.max_size])
def reset(self):
self.size = 0

View File

@ -22,12 +22,27 @@ import streamedrequests
def get_test_id(): def get_test_id():
return str(uuid.uuid4()) + '.dat' return str(uuid.uuid4()) + '.dat'
def _test_callback(text):
print('got', text)
class TestInit(unittest.TestCase): class TestInit(unittest.TestCase):
def test_init(self): def test_basic(self):
streamedrequests.get_request("test", request_headers=None, sync=True, max_size=0, streamedrequests.get('https://example.com/')
connect_timeout=0, stream_timeout=0, proxy=None)
return def test_callback(self):
pass
streamedrequests.get('https://example.com/', chunk_size=1, callback=_test_callback)
def test_async(self):
streamedrequests.get('https://example.com/', chunk_size=1, callback=_test_callback, sync=False)
def test_zero_chunk_size(self):
try:
streamedrequests.get('https://example.com/', chunk_size=0)
except ValueError:
pass
else:
self.assertTrue(failUnless)
unittest.main() unittest.main()