Switching placekey-py over to the new calls and refactoring calls to …

placekey/api.py

@@ -6,6 +6,7 @@
 import backoff
 import requests
 from ratelimit import limits, RateLimitException
+from .general import _post_request_function
 
 from .__version__ import __version__
 
@@ -106,13 +107,15 @@ def __init__(self, api_key=None, max_retries=DEFAULT_MAX_RETRIES, logger=log,
                     self.headers['User-Agent'] + " " + self.user_agent_comment).strip()
 
         # Rate-limited function for a single requests
-        self.make_request = self._get_request_function(
+        self.make_request = _post_request_function(
+            headers=self.headers,
             url=self.URL,
             calls=self.REQUEST_LIMIT,
             period=self.REQUEST_WINDOW,
             max_tries=self.max_retries)
 
-        self.make_bulk_request = self._get_request_function(
+        self.make_bulk_request = _post_request_function(
+            headers=self.headers,
             url=self.BULK_URL,
             calls=self.BULK_REQUEST_LIMIT,
             period=self.BULK_REQUEST_WINDOW,
@@ -290,37 +293,4 @@ def _safe_parse_json(self, result):
             return []
         except Exception as e:
             self.logger.error(f"Error parsing: {e}, returning empty list")
-            return []
-
-    def _get_request_function(self, url, calls, period, max_tries):
-        """
-        Construct a rate limited function for making requests.
-
-        :param url: request URL
-        :param calls: number of calls that can be made in time period
-        :param  period: length of rate limiting time period in seconds
-        :param max_tries: the maximum number of retries before giving up
-        """
-
-        @backoff.on_exception(backoff.fibo, (RateLimitException, requests.exceptions.RequestException),
-                              max_tries=max_tries)
-        @limits(calls=calls, period=period)
-        def make_request(data):
-            try:
-                response = requests.post(
-                    url, headers=self.headers,
-                    data=json.dumps(data).encode('utf-8')
-                )
-
-                if response.status_code == 429:
-                    raise RateLimitException("Rate limit exceeded", 0)
-                elif response.status_code == 503:
-                    raise requests.exceptions.RequestException("Service Unavailable")
-                elif response.status_code == 504:
-                    raise requests.exceptions.RequestException("Gateway Timeout")
-
-                return response
-            except requests.exceptions.RequestException as e:
-                raise e
-
-        return make_request
+            return []

placekey/general.py

@@ -0,0 +1,80 @@
+import itertools
+import json
+import logging
+from json import JSONDecodeError
+
+import requests
+from ratelimit import limits, RateLimitException
+import backoff
+
+def _post_request_function(headers, url, calls, period, max_tries):
+        """
+        Construct a rate limited function for making requests.
+
+        :param url: request URL
+        :param calls: number of calls that can be made in time period
+        :param  period: length of rate limiting time period in seconds
+        :param max_tries: the maximum number of retries before giving up
+        """
+
+        @backoff.on_exception(backoff.fibo, (RateLimitException, requests.exceptions.RequestException),
+                              max_tries=max_tries)
+        @limits(calls=calls, period=period)
+        def make_request(request_data = None):
+            try:
+                payload = {
+                    "url": url,
+                    "headers": headers,
+                }
+                if request_data:
+                    payload["data"] = json.dumps(request_data).encode('utf-8')
+                response = requests.post(**payload)
+
+                if response.status_code == 429:
+                    raise RateLimitException("Rate limit exceeded", 0)
+                elif response.status_code == 503:
+                    raise requests.exceptions.RequestException("Service Unavailable")
+                elif response.status_code == 504:
+                    raise requests.exceptions.RequestException("Gateway Timeout")
+
+                return response
+            except requests.exceptions.RequestException as e:
+                raise e
+
+        return make_request
+
+def _get_request_function(headers, url, calls, period, max_tries):
+        """
+        Construct a rate limited function for making requests.
+
+        :param url: request URL
+        :param calls: number of calls that can be made in time period
+        :param  period: length of rate limiting time period in seconds
+        :param max_tries: the maximum number of retries before giving up
+        """
+
+        @backoff.on_exception(backoff.fibo, (RateLimitException, requests.exceptions.RequestException),
+                              max_tries=max_tries)
+        @limits(calls=calls, period=period)
+        def make_request(params = None):
+            try:
+                payload = {
+                    "url": url,
+                    "headers": headers,
+                }
+                if params:
+                    payload["params"] = params
+                response = requests.get(**payload)
+
+                if response.status_code == 429:
+                    raise RateLimitException("Rate limit exceeded", 0)
+                elif response.status_code == 503:
+                    raise requests.exceptions.RequestException("Service Unavailable")
+                elif response.status_code == 504:
+                    raise requests.exceptions.RequestException("Gateway Timeout")
+
+                return response
+            except requests.exceptions.RequestException as e:
+                raise e
+
+        return make_request

placekey/placekey.py

@@ -7,6 +7,7 @@
 import re
 import json
 from math import asin, cos, radians, sqrt
+import ast
 
 import h3
 import h3.api.basic_int as h3_int
@@ -18,6 +19,7 @@
 import boto3
 from botocore import UNSIGNED
 from botocore.config import Config
+from .general import _get_request_function
 
 RESOLUTION = 10
 BASE_RESOLUTION = 12
@@ -57,12 +59,18 @@ def list_free_datasets():
     """
     :return: The names of every free placekey'd dataset Placekey offers
     """
-    folders = set()
-    paginator = s3.get_paginator("list_objects_v2")
-    for page in paginator.paginate(Bucket='placekey-free-datasets', Prefix='', Delimiter="/"):
-        for common_prefix in page.get("CommonPrefixes", []):
-            folders.add(common_prefix["Prefix"].replace("/", ""))
-    return folders
+    func = _get_request_function(
+        headers={},
+        url="https://api.placekey.io/placekey-py/v1/get-public-dataset-names",
+        calls=3,
+        period=60,
+        max_tries=20
+    )
+    response = func()
+    if response.status_code == 200:
+        return ast.literal_eval(response.text)
+    else:
+        raise Exception("Something went wrong. Please contact Placekey.")
 
 def return_free_datasets_location_by_name(name: str, url: bool = False):
     """
@@ -72,21 +80,23 @@ def return_free_datasets_location_by_name(name: str, url: bool = False):
     :param name: Return a URL or S3 URI? Default is False (S3 URI)
     :return: The public S3 location of the placekey'd dataset
     """
-    response = s3.list_objects_v2(Bucket='placekey-free-datasets', Prefix=name+'/csv')
-
-    # Extract files from the response
-    files = [obj["Key"] for obj in response.get("Contents", [])]
-
-    if len(files) == 1:
-        if url:
-            return "https://placekey-free-datasets.s3.us-west-2.amazonaws.com/"+files[0]
-        else:
-            return "s3://placekey-free-datasets/"+files[0]
-    elif len(files) == 0:
-        print()
-        raise FileNotFoundError("No files found in the specified S3 directory. Please notify Placekey.")
+    func = _get_request_function(
+        headers={},
+        url="https://api.placekey.io/placekey-py/v1/get-public-dataset-location-from-name",
+        calls=3,
+        period=60,
+        max_tries=20
+    )
+    response = func(params={
+        'name': name,
+        'url': url
+    })
+    if response.status_code == 200:
+        return response.text
+    elif response.status_code >= 400 and response.status_code < 500:
+        raise ValueError(response.reason)
     else:
-        raise ValueError(f"Something went wrong. Please notify Placekey.")
+        raise Exception("Something went wrong. Please contact Placekey.")
 
 def _get_header_int():
     """