add flat files replace

Author: victoreram

FlatFilesExport.md

@@ -7,7 +7,7 @@ market trades, market quotes, and market candles.
 This program was set up to easily access a flat files server provided to Coin Metrics commercial clients. If you are a 
 community API user or a client without access to this server, a 403 error will be returned. If you are a community API 
 user and looking to get historical asset prices, Coin Metrics does offer [historical asset prices for download](https://coinmetrics.io/community-network-data/).  
-The `CoinMetricsDataExporter` class will return a `CoinMetricsUnauthorizedException` if your API key is not authorized
+The `CoinMetricsDataExporter` class will return a `CoinMetricsClientFlatFilesUnauthorizedException` if your API key is not authorized
 to access this server. The flat files export is considered a separate product from the Coin Metrics API, if you would
 like to gain access or believe you should have access but do not, please contact coinmetrics support. 
 

coinmetrics/_exceptions.py

@@ -4,25 +4,80 @@
 from functools import reduce
 
 
-class CoinMetricsUnauthorizedException(HTTPError):
+class CoinMetricsClientFlatFilesUnauthorizedException(HTTPError):
     """
     Raised when a request is made that will return an error due to being unauthorized to flat files server
     """
+    def __init__(self, response: Response, *args: Any, **kwargs: Any):
+        if response.status_code != 403:
+            response.raise_for_status()
+        self.response = response
+        self.request = response.request
+        error_message = """The provided API Key is not authorized to access """
+        self.msg = error_message
+        super().__init__(response=response, request=response.request, *args, **kwargs)
+
+    def __str__(self) -> str:
+        return self.msg
+
+
+class CoinMetricsClientNotFoundError(Exception):
+    """Raised when a CoinMetricsClient instance is not found."""
+    def __init__(self, message="CoinMetricsClient not found"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class CoinMetricsClientBadParameterError(HTTPError):
+    """
+    Raised when a request is made with bad parameters (HTTP 400).
+    """
+    def __init__(self, response: Response, *args: Any, **kwargs: Any):
+        if response.status_code != 400:
+            response.raise_for_status()
+        self.response = response
+        self.request = response.request
+        error_message = "Bad Parameter: The request contains invalid parameters."
+        self.msg = error_message
+        super().__init__(response=response, request=response.request, *args, **kwargs)
+
+    def __str__(self) -> str:
+        return self.msg
+
 
+class CoinMetricsClientUnauthorizedError(HTTPError):
+    """
+    Raised when a request is unauthorized due to invalid or missing API key (HTTP 401).
+    """
     def __init__(self, response: Response, *args: Any, **kwargs: Any):
-        if response.status_code not in [401, 403]:
+        if response.status_code != 401:
             response.raise_for_status()
         self.response = response
         self.request = response.request
-        error_message = """The provided API key is not authorized to access the Coin Metrics Flat Files server. This product is separate from the API. If you'd like access granted or believe this is a mistake please contact Coin Metrics support.
-                    """
+        error_message = "Unauthorized: The API key is invalid or missing."
         self.msg = error_message
         super().__init__(response=response, request=response.request, *args, **kwargs)
 
     def __str__(self) -> str:
         return self.msg
 
 
+class CoinMetricsClientForbiddenError(HTTPError):
+    """
+    Raised when a request is forbidden due to insufficient permissions (HTTP 403).
+    """
+    def __init__(self, response: Response, *args: Any, **kwargs: Any):
+        if response.status_code != 403:
+            response.raise_for_status()
+        self.response = response
+        self.request = response.request
+        error_message = "Forbidden: The API key does not have permission to access this resource."
+        self.msg = error_message
+        super().__init__(response=response, request=response.request, *args, **kwargs)
+
+    def __str__(self) -> str:
+        return self.msg
+
 class CoinMetricsClientQueryParamsException(HTTPError):
     """
     Raised when a request is made that will return an error due to the logic or contents of the request
@@ -52,8 +107,20 @@ def __str__(self) -> str:
         return self.msg
 
 
-class CoinMetricsClientNotFoundError(Exception):
-    """Raised when a CoinMetricsClient instance is not found."""
-    def __init__(self, message="CoinMetricsClient not found"):
-        self.message = message
-        super().__init__(self.message)
+class CoinMetricsClientRateLimitError(HTTPError):
+    """
+    Raised when the rate limit is exceeded (HTTP 429).
+    """
+    def __init__(self, response: Response, *args: Any, **kwargs: Any):
+        if response.status_code != 429:
+            response.raise_for_status()
+        self.response = response
+        self.request = response.request
+        self.rate_limit_remaining = response.headers.get("x-ratelimit-remaining", "0")
+        self.rate_limit_reset = response.headers.get("x-ratelimit-reset", "0")
+        error_message = f"Rate Limit Exceeded: Too many requests. Remaining: {self.rate_limit_remaining}, Reset in: {self.rate_limit_reset} seconds."
+        self.msg = error_message
+        super().__init__(response=response, request=response.request, *args, **kwargs)
+
+    def __str__(self) -> str:
+        return self.msg

coinmetrics/api_client.py

@@ -13,7 +13,13 @@
 
 from coinmetrics._utils import retry, transform_url_params_values_to_str, deprecated, alias
 from coinmetrics import __version__ as version
-from coinmetrics._exceptions import CoinMetricsClientQueryParamsException
+from coinmetrics._exceptions import (
+    CoinMetricsClientQueryParamsException,
+    CoinMetricsClientBadParameterError,
+    CoinMetricsClientForbiddenError,
+    CoinMetricsClientUnauthorizedError,
+    CoinMetricsClientRateLimitError,
+)
 from coinmetrics._typing import (
     DataReturnType,
     MessageHandlerType
@@ -132,6 +138,8 @@ def __init__(
         host: Optional[str] = None,
         port: Optional[int] = None,
         schema: str = "https",
+        ignore_unauthorized_errors: bool = False,
+        ignore_forbidden_errors: bool = False,
     ):
         """
         :param api_key: The API key for the CoinMetrics API.
@@ -152,10 +160,16 @@ def __init__(
         :type port: int
         :param schema: The schema for accessing the Coin Metrics API. Default is "https".
         :type schema: str
+        :param ignore_unauthorized_errors: Whether to ignore 401 Unauthorized errors. Default is False.
+        :type ignore_unauthorized_errors: bool
+        :param ignore_forbidden_errors: Whether to ignore 403 Forbidden errors. Default is False.
+        :type ignore_forbidden_errors: bool
         """
         self._api_key_url_str = "api_key={}".format(api_key) if api_key else ""
 
         self._verify_ssl_certs = verify_ssl_certs
+        self._ignore_unauthorized_errors = ignore_unauthorized_errors
+        self._ignore_forbidden_errors = ignore_forbidden_errors
 
         if host is None:
             self._host = "api.coinmetrics.io" if api_key else "community-api.coinmetrics.io"
@@ -8268,23 +8282,35 @@ def _get_data(self, url: str, params: Dict[str, Any]) -> DataReturnType:
         else:
             self._log(f"Response status code: {resp.status_code} for url: {resp.url} took: {elapsed}")
 
-        # Raise early on HTTP errors
-        try:
-            resp.raise_for_status()
-        except HTTPError:
-            if resp.status_code == 414:
-                # URI is too long
-                raise CoinMetricsClientQueryParamsException(response=resp)
-            if resp.headers.get("content-type") == "application/json":
-                try:
-                    data = json.loads(resp.content)
-                    if isinstance(data, dict) and "error" in data:
-                        error_msg = (f"Error found for the query: \n {actual_url}\nError details: {data.get('error')}")
-                        logger.error(error_msg)
-                        resp.raise_for_status()
-                except ValueError:
-                    raise ValueError(f"Failed to parse error response as JSON. Status code: {resp.status_code}, Content: {resp.content}")
-            raise
+        # Handle HTTP errors with custom exceptions
+        if resp.status_code == 400:
+            raise CoinMetricsClientBadParameterError(response=resp)
+        elif resp.status_code == 401:
+            if not self._ignore_unauthorized_errors:
+                raise CoinMetricsClientUnauthorizedError(response=resp)
+        elif resp.status_code == 403:
+            if not self._ignore_forbidden_errors:
+                raise CoinMetricsClientForbiddenError(response=resp)
+        elif resp.status_code == 414:
+            # URI is too long
+            raise CoinMetricsClientQueryParamsException(response=resp)
+        elif resp.status_code == 429:
+            raise CoinMetricsClientRateLimitError(response=resp)
+        elif resp.status_code >= 400:
+            # Handle other HTTP errors
+            try:
+                resp.raise_for_status()
+            except HTTPError:
+                if resp.headers.get("content-type") == "application/json":
+                    try:
+                        data = json.loads(resp.content)
+                        if isinstance(data, dict) and "error" in data:
+                            error_msg = (f"Error found for the query: \n {actual_url}\nError details: {data.get('error')}")
+                            logger.error(error_msg)
+                            resp.raise_for_status()
+                    except ValueError:
+                        raise ValueError(f"Failed to parse error response as JSON. Status code: {resp.status_code}, Content: {resp.content}")
+                raise
 
         if is_json_stream:
             # Return a generator: caller can iterate without loading into memory
@@ -8318,21 +8344,46 @@ def _get_stream_data(self, url: str, params: Dict[str, Any]) -> CmStream:
         )
         return CmStream(ws_url=actual_url)
 
-    @retry((socket.gaierror, HTTPError), retries=5, wait_time_between_retries=5)
+    @retry((socket.gaierror, HTTPError, CoinMetricsClientRateLimitError), retries=5, wait_time_between_retries=5)
     def _send_request(self, actual_url: str, is_json_stream: False) -> Response:  # type: ignore
         """
         Wrapper for requests.get with retry on certain exceptions.
         """
-        response = self._session.get(
-            actual_url,
-            headers=self._session.headers,
-            proxies=self._session.proxies,
-            verify=self._session.verify,
-            stream=is_json_stream,
-        )
-        if response.status_code == 429 or response.headers.get("x-ratelimit-remaining", None) == "0":
-            logger.info("Sleeping for a rate limit window because 429 (too many requests) error was returned. Please "
-                        "see Coin Metrics APIV4 documentation for more information: https://docs.coinmetrics.io/api/v4/#tag/Rate-limits")
-            time.sleep(int(response.headers["x-ratelimit-reset"]))
-            response = self._send_request(actual_url=actual_url)
-        return response
+        max_retries = 5
+        retry_count = 0
+
+        while retry_count < max_retries:
+            response = self._session.get(
+                actual_url,
+                headers=self._session.headers,
+                proxies=self._session.proxies,
+                verify=self._session.verify,
+                stream=is_json_stream,
+            )
+
+            # Handle rate limiting with proper header-based retry logic
+            if response.status_code == 429:
+                retry_count += 1
+                rate_limit_reset = response.headers.get("x-ratelimit-reset", "60")
+                rate_limit_remaining = response.headers.get("x-ratelimit-remaining", "0")
+
+                logger.info(
+                    "Rate limit exceeded (429). Remaining requests: %s, Reset in: %s seconds. "
+                    "Retry attempt: %d/%d. See Coin Metrics API v4 documentation for more information: "
+                    "https://docs.coinmetrics.io/api/v4/#tag/Rate-limits",
+                    rate_limit_remaining, rate_limit_reset, retry_count, max_retries
+                )
+
+                if retry_count >= max_retries:
+                    raise CoinMetricsClientRateLimitError(response=response)
+
+                # Sleep for the reset time specified in the header
+                sleep_time = int(rate_limit_reset)
+                time.sleep(sleep_time)
+                continue
+
+            # If we get here, the request was successful or had a different error
+            return response
+
+        # This should never be reached, but just in case
+        raise CoinMetricsClientRateLimitError(response=response)

coinmetrics/data_exporter.py

@@ -9,7 +9,7 @@
 from requests import Response, HTTPError
 from requests.auth import HTTPBasicAuth
 
-from coinmetrics._exceptions import CoinMetricsUnauthorizedException
+from coinmetrics._exceptions import CoinMetricsClientFlatFilesUnauthorizedException
 from coinmetrics._utils import retry
 from datetime import datetime
 from typing import Union, Optional, List, Dict, Any, Iterable
@@ -472,7 +472,7 @@ def _send_request(self, actual_url: str, stream: bool = False) -> Response:
             actual_url, verify=self._verify_ssl_certs, auth=self._auth, stream=stream, proxies=self.proxies  # type: ignore
         )
         if response.status_code == 403 or response.status_code == 401:
-            raise CoinMetricsUnauthorizedException(response=response)
+            raise CoinMetricsClientFlatFilesUnauthorizedException(response=response)
         if response.status_code != 200:
             response.raise_for_status()
         return response

docs/docs/tools/FlatFilesExport.md

@@ -7,7 +7,7 @@ market trades, market quotes, and market candles.
 This program was set up to easily access a flat files server provided to Coin Metrics commercial clients. If you are a 
 community API user or a client without access to this server, a 403 error will be returned. If you are a community API 
 user and looking to get historical asset prices, Coin Metrics does offer [historical asset prices for download](https://coinmetrics.io/community-network-data/).  
-The `CoinMetricsDataExporter` class will return a `CoinMetricsUnauthorizedException` if your API key is not authorized
+The `CoinMetricsDataExporter` class will return a `CoinMetricsClientFlatFilesUnauthorizedException` if your API key is not authorized
 to access this server. The flat files export is considered a separate product from the Coin Metrics API, if you would
 like to gain access or believe you should have access but do not, please contact coinmetrics support. 
 

docs/site/search/search_index.json

@@ -1046,7 +1046,7 @@ <h2 id="authorization-for-flat-files-download">Authorization for flat files down
 <p>This program was set up to easily access a flat files server provided to Coin Metrics commercial clients. If you are a 
 community API user or a client without access to this server, a 403 error will be returned. If you are a community API 
 user and looking to get historical asset prices, Coin Metrics does offer <a href="https://coinmetrics.io/community-network-data/">historical asset prices for download</a>.<br />
-The <code>CoinMetricsDataExporter</code> class will return a <code>CoinMetricsUnauthorizedException</code> if your API key is not authorized
+The <code>CoinMetricsDataExporter</code> class will return a <code>CoinMetricsClientFlatFilesUnauthorizedException</code> if your API key is not authorized
 to access this server. The flat files export is considered a separate product from the Coin Metrics API, if you would
 like to gain access or believe you should have access but do not, please contact coinmetrics support. </p>
 <h3 id="installation-and-set-up">Installation and set up<a class="headerlink" href="#installation-and-set-up" title="Permanent link">&para;</a></h3>

docs/site/tools/FlatFilesExport.html

@@ -3,7 +3,7 @@
 from datetime import datetime
 from coinmetrics._exceptions import (
     CoinMetricsClientQueryParamsException,
-    # CoinMetricsUnauthorizedException,
+    # CoinMetricsClientFlatFilesUnauthorizedException,
 )
 from coinmetrics.api_client import CoinMetricsClient
 from coinmetrics.data_exporter import CoinMetricsDataExporter
@@ -58,7 +58,7 @@ def test_custom_exception_not_raised_for_403() -> None:
 #         data_exporter.export_market_trades_spot_data(
 #             start_date=start_date, end_date=end_date, exchanges=exchanges, threaded=True
 #         )
-#     except CoinMetricsUnauthorizedException as e:
+#     except CoinMetricsClientFlatFilesUnauthorizedException as e:
 #         assert e.response is not None
 #         assert e.response.status_code == 403 or e.response.status_code == 401
 #         print(e)