Fix: Eliminate uses of requests library (#7633)#7745
Open
achave11-ucsc wants to merge 23 commits intodevelopfrom
Open
Fix: Eliminate uses of requests library (#7633)#7745achave11-ucsc wants to merge 23 commits intodevelopfrom
achave11-ucsc wants to merge 23 commits intodevelopfrom
Conversation
achave11-ucsc
force-pushed
the
issues/achave11-ucsc/7633-eliminate-requests-usages
branch
from
feb9b32 to
b9bc81d
Compare
|
coverage: 85.014% (+0.05%) from 84.968% — issues/achave11-ucsc/7633-eliminate-requests-usages into develop |
achave11-ucsc
force-pushed
the
issues/achave11-ucsc/7633-eliminate-requests-usages
branch
3 times, most recently
from
dadf91d to
aa2bfcf
Compare
Codecov Report❌ Patch coverage is Additional details and impacted files@@ Coverage Diff @@
## develop #7745 +/- ##
===========================================
+ Coverage 84.88% 84.93% +0.04%
===========================================
Files 164 165 +1
Lines 23511 23571 +60
===========================================
+ Hits 19958 20020 +62
+ Misses 3553 3551 -2 ☔ View full report in Codecov by Sentry. 🚀 New features to boost your workflow:
|
achave11-ucsc
force-pushed
the
issues/achave11-ucsc/7633-eliminate-requests-usages
branch
6 times, most recently
from
eafd283 to
032ef7c
Compare
achave11-ucsc
force-pushed
the
issues/achave11-ucsc/7633-eliminate-requests-usages
branch
15 times, most recently
from
e5f9720 to
e0a85ba
Compare
hannes-ucsc
reviewed
Apr 21, 2026
Member
hannes-ucsc
left a comment
hannes-ucsc
left a comment
There was a problem hiding this comment.
Index: test/urllib3_mock.py
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/test/urllib3_mock.py b/test/urllib3_mock.py
--- a/test/urllib3_mock.py (revision 3e0cdf3331ed51c75de715694346d702b45ffd73)
+++ b/test/urllib3_mock.py (date 1776745455251)
@@ -1,3 +1,6 @@
+from collections import (
+ defaultdict,
+)
import json
from unittest.mock import (
PropertyMock,
@@ -16,18 +19,24 @@
JSON,
)
+# TODO: document structure here, use furl so that users don't accidentally swap the arguments
+type CannedResponses = dict[tuple[str, furl], list[urllib3.BaseHTTPResponse]]
+
class Urllib3Mock:
"""
- Context manager that replaces ``_http_client`` on one or more classes with
- a :class:`MockHttpClient` backed by a shared response registry. Responses
- are registered via :meth:`add` and dispatched in FIFO order per
- ``(method, url)`` key.
+ Context manager that patches the ``_http_client`` property of one or more
+ target classes with a mock client that returns mock responses from a
+ registry.
"""
def __init__(self, *targets: type) -> None:
- self.registry: dict[tuple[str, str], list[urllib3.BaseHTTPResponse]] = {}
- self._mock_client = self._MockHttpClient(self.registry)
+ """
+ Create the context manager instance that patches the given targets on
+ entry, and restores them on exit.
+ """
+ self._registry: CannedResponses = defaultdict(list)
+ self._mock_client = self._MockHttpClient(self._registry)
self._patches = [
patch.object(target,
'_http_client',
@@ -35,40 +44,6 @@
for target in targets
]
- def __enter__(self):
- for p in self._patches:
- p.start()
- return self
-
- def __exit__(self, *_args):
- for p in self._patches:
- p.stop()
- self.registry.clear()
-
- class _MockHttpClient(HttpClient):
- """
- A mock HTTP client backed by a response registry keyed by
- ``(method, url)``. Subclasses :class:`HttpClient`
- (``urllib3.request.RequestMethods``) so that ``request()`` correctly
- encodes ``fields`` into the URL for GET/HEAD requests before delegating
- to ``urlopen()``. Multiple responses per key are served in FIFO order.
- """
-
- def __init__(self,
- registry: dict[tuple[str, str], list[urllib3.BaseHTTPResponse]]
- ) -> None:
- super().__init__()
- self._registry = registry
-
- def urlopen(self,
- method: str,
- url: str,
- *args,
- **kwargs
- ) -> urllib3.BaseHTTPResponse:
- key = (method, _normalize_url(str(url)))
- return self._registry[key].pop(0)
-
def add(self,
method: str,
url: str,
@@ -78,6 +53,19 @@
body: bytes | str | JSON = b'',
reason: str | None = None,
) -> None:
+ """
+ Register a mock response for the given pair of HTTP method and URL.
+ If multiple responses are registered for the same pair, they are
+ returned in the order they were registered in.
+
+ :param method: the request method, e.g. 'GET'
+ :param url: the request URL
+ :param status: the status of the returned response
+ :param headers: the headers of the returned response
+ :param body: the body of the returned response
+ :param reason: optional text to follow the numeric response status
+ :return:
+ """
if headers is None:
headers = {}
if isinstance(body, dict):
@@ -95,7 +83,36 @@
request_url=url,
preload_content=True,
)
- self.registry.setdefault((method, _normalize_url(url)), []).append(response)
+ key = (method, _normalize_url(url))
+ self._registry[key].append(response)
+
+ def __enter__(self):
+ for p in self._patches:
+ p.start()
+ return self
+
+ def __exit__(self, *_args):
+ for p in self._patches:
+ p.stop()
+ self._registry.clear()
+
+ class _MockHttpClient(HttpClient):
+ # REVIEW: PL, please
+
+ def __init__(self,
+ registry: dict[tuple[str, str], list[urllib3.BaseHTTPResponse]]
+ ) -> None:
+ super().__init__()
+ self._registry = registry
+
+ def urlopen(self,
+ method: str,
+ url: str,
+ *args,
+ **kwargs
+ ) -> urllib3.BaseHTTPResponse:
+ key = (method, _normalize_url(str(url)))
+ return self._registry[key].pop(0)
def _normalize_url(url: str) -> str:
hannes-ucsc
requested changes
Apr 23, 2026
Member
hannes-ucsc
left a comment
hannes-ucsc
left a comment
There was a problem hiding this comment.
Index: test/urllib3_mock.py
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/test/urllib3_mock.py b/test/urllib3_mock.py
--- a/test/urllib3_mock.py (revision 92d2641a8dd22a29155a8203f352dfc8096267e2)
+++ b/test/urllib3_mock.py (date 1776960146618)
@@ -1,5 +1,6 @@
from collections import (
defaultdict,
+ deque,
)
import json
from unittest.mock import (
@@ -10,7 +11,10 @@
from furl import (
furl,
)
-import urllib3
+from urllib3 import (
+ BaseHTTPResponse,
+ HTTPResponse,
+)
from azul.http import (
HttpClient,
@@ -19,15 +23,17 @@
JSON,
)
-# Maps each (method, url) pair to a queue of responses
-type _CannedResponses = dict[tuple[str, str], list[urllib3.BaseHTTPResponse]]
+type _QueuedResponses = dict[tuple[str, str], deque[BaseHTTPResponse]]
class Urllib3Mock:
"""
Context manager that patches the ``_http_client`` property of one or more
- target classes with a mock client that returns mock responses from a
- registry.
+ target classes with a mock client that returns previously queued mock
+ responses. Each distinct combination of HTTP method and URL has a separate
+ queue. When the patched target's mock client makes a request matching one of
+ those combinations, the mock responses are returned in the order they were
+ queued in.
"""
def __init__(self, *targets: type) -> None:
@@ -35,14 +41,14 @@
Create the context manager instance that patches the given targets on
entry, and restores them on exit.
"""
- self._registry: _CannedResponses = defaultdict(list)
- self._mock_client = self._MockHttpClient(self._registry)
- self._patches = [
+ self._responses: _QueuedResponses = defaultdict(deque)
+ self._client = _MockHttpClient(self._responses)
+ self._patches = deque(
patch.object(target,
'_http_client',
- new=PropertyMock(return_value=self._mock_client))
+ new=PropertyMock(return_value=self._client))
for target in targets
- ]
+ )
def add(self,
*,
@@ -54,9 +60,9 @@
reason: str | None = None,
) -> None:
"""
- Register a mock response for the given pair of HTTP method and URL.
- If multiple responses are registered for the same pair, they are
- returned in the order they were registered in.
+ Queue a mock response for the given combination of HTTP method and URL.
+ If multiple responses are queued for the same combination, they are
+ returned in the order they were queued in.
:param method: the request method, e.g. 'GET'
@@ -78,17 +84,15 @@
if isinstance(body, str):
body = body.encode()
assert isinstance(body, bytes), type(body)
- response = urllib3.HTTPResponse(
- body=body,
- headers=headers,
- status=status,
- reason=reason,
- request_method=method,
- request_url=str(url),
- preload_content=True,
- )
+ response = HTTPResponse(body=body,
+ headers=headers,
+ status=status,
+ reason=reason,
+ request_method=method,
+ request_url=str(url),
+ preload_content=True)
key = (method, _normalize_url(url))
- self._registry[key].append(response)
+ self._responses[key].append(response)
def __enter__(self):
for p in self._patches:
@@ -96,29 +100,24 @@
return self
def __exit__(self, *_args):
- for p in self._patches:
+ for p in reversed(self._patches):
p.stop()
- self._registry.clear()
+ self._responses.clear()
+
- class _MockHttpClient(HttpClient):
+class _MockHttpClient(HttpClient):
- def __init__(self,
- registry: _CannedResponses
- ) -> None:
- super().__init__()
- self._registry = registry
+ def __init__(self, responses: _QueuedResponses) -> None:
+ super().__init__()
+ self._responses = responses
- def urlopen(self,
- method: str,
- url: str,
- *args,
- **kwargs
- ) -> urllib3.BaseHTTPResponse:
- key = (method, _normalize_url(url))
- return self._registry[key].pop(0)
+ def urlopen(self, method: str, url: str, *args, **kwargs) -> BaseHTTPResponse:
+ key = (method, _normalize_url(url))
+ return self._responses[key].pop()
def _normalize_url(url: str) -> str:
+ # REVIEW: Think about how many `furl` instances your code creates
f = furl(url).copy()
f.set(args=dict(sorted(f.args.items())))
return str(f)
Index: .mypy.ini
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/.mypy.ini b/.mypy.ini
--- a/.mypy.ini (revision 92d2641a8dd22a29155a8203f352dfc8096267e2)
+++ b/.mypy.ini (date 1776960256735)
@@ -93,4 +93,3 @@
[mypy-aws_requests_auth.boto_utils]
follow_untyped_imports = True
-
hannes-ucsc
requested changes
Apr 29, 2026
Member
hannes-ucsc
left a comment
hannes-ucsc
left a comment
There was a problem hiding this comment.
Index: src/azul/http.py
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/src/azul/http.py b/src/azul/http.py
--- a/src/azul/http.py (revision 526cbd982f01fb3ba24dc4a2d3d1b0a20e57d1f3)
+++ b/src/azul/http.py (date 1777479946473)
@@ -174,10 +174,28 @@
return StatusRetryHttpClient(client)
+# REVIEW: I'm not sure we should inherit the urllib3 exception class, try
+# inheriting Exception instead, or provide reason why that's
+# difficult/impossible. If it IS possible, we might be able to reduce
+# the boiler plate with attrs.define and a custom init or a pre init
+# (https://www.attrs.org/en/stable/init.html)
+
+class HTTPStatusError(urllib3.exceptions.HTTPError):
+ url: str | None
+ status: int
+ reason: str | None
+
+ def __init__(self, url: str | None, status: int, reason: str | None = None):
+ self.url = url
+ self.status = status
+ self.reason = reason
+ # URL is intentionally passed as the last arg, as they tend to be long
+ super().__init__('Unexpected response status', status, reason, url)
+
+
def raise_on_status(response: urllib3.BaseHTTPResponse) -> None:
if not 200 <= response.status <= 399:
- msg = f'{response.reason} for url: {response.url}'
- raise urllib3.exceptions.HTTPError(msg)
+ raise HTTPStatusError(response.url, response.status, response.reason)
class LimitedTimeoutException(Exception):
Index: src/azul/service/drs_controller.py
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/src/azul/service/drs_controller.py b/src/azul/service/drs_controller.py
--- a/src/azul/service/drs_controller.py (revision 526cbd982f01fb3ba24dc4a2d3d1b0a20e57d1f3)
+++ b/src/azul/service/drs_controller.py (date 1777477434587)
@@ -247,11 +247,10 @@
'token': token
})
if response.status == 301:
- headers: dict[str, str | list[str]] = {
- 'retry-after': response.headers['retry-after']
- }
+ header_name = 'retry-after'
+ retry_after = response.headers[header_name]
# DRS says no body for 202 responses
- return Response(body='', status_code=202, headers=headers)
+ return Response(body='', status_code=202, headers={header_name: retry_after})
elif response.status == 302:
retry_url = response.headers['location']
return Response(self._access_url(retry_url))![github-advanced-security[bot]](https://avatars.githubusercontent.com/in/57789?s=60&v=4){kind=small}
Member
Author
|
The custom init or a pre init wasn't necessary… |
hannes-ucsc
requested changes
Apr 30, 2026
Member
hannes-ucsc
left a comment
hannes-ucsc
left a comment
There was a problem hiding this comment.
PL, please, to clear up the misunderstanding regarding my previous patch.
|
|
||
| def raise_on_status(response: urllib3.BaseHTTPResponse) -> None: | ||
| if not 200 <= response.status <= 399: | ||
| raise HTTPStatusError(response.status, response.reason, response.url) |
Member
There was a problem hiding this comment.
My patch had a human-readable message.
hannes-ucsc
reviewed
Apr 30, 2026
| class HTTPStatusError(Exception): | ||
| status: int | ||
| reason: str | None | ||
| url: str | None # URL is intentionally the last arg, as it tends to be long |
Member
There was a problem hiding this comment.
My patch had this first, and that was intentional. The "long" concern was meant to apply to the log super().init call, that's why my patch had this comment before that call.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
…/dss/__init__.py (#7633) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
…adata/helpers/schema_validation.py (#7633)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
6 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Linked issues: #7633
Checklist
Author
developissues/<GitHub handle of author>/<issue#>-<slug>1 when the issue title describes a problem, the corresponding PR
title is
Fix:followed by the issue titleAuthor (partiality)
ptag to titles of partial commitspartialor completely resolves all linked issuespartiallabelAuthor (reindex)
rtag to commit title or the changes introduced by this PR will not require reindexing of any deploymentreindex:devor the changes introduced by it will not require reindexing ofdevreindex:anvildevor the changes introduced by it will not require reindexing ofanvildevreindex:anvilprodor the changes introduced by it will not require reindexing ofanvilprodreindex:prodor the changes introduced by it will not require reindexing ofprodreindex:partialand its description documents the specific reindexing procedure fordev,anvildev,anvilprodandprodor requires a full reindex or carries none of the labelsreindex:dev,reindex:anvildev,reindex:anvilprodandreindex:prodAuthor (API changes)
APIor this PR does not modify a REST APIa(A) tag to commit title for backwards (in)compatible changes or this PR does not modify a REST APIapp.pyor this PR does not modify a REST APIAuthor (upgrading deployments)
make docker_images.jsonand committed the resulting changes or this PR does not modifyazul_docker_images, or any other variables referenced in the definition of that variableutag to commit title or this PR does not require upgrading deploymentsupgradeor does not require upgrading deploymentsdeploy:sharedor does not modifydocker_images.json, and does not require deploying thesharedcomponent for any other reasondeploy:gitlabor does not require deploying thegitlabcomponentdeploy:runneror does not require deploying therunnerimageAuthor (hotfixes)
Ftag to main commit title or this PR does not include permanent fix for a temporary hotfixanvilprodandprod) have temporary hotfixes for any of the issues linked to this PRAuthor (before every review)
develop, squashed fixups from prior reviewsmake requirements_updateor this PR does not modifyDockerfile,environment,requirements*.txt,common.mk,Makefileorenvironment.bootRtag to commit title or this PR does not modifyrequirements*.txtreqsor does not modifyrequirements*.txtmake integration_testpasses in personal deployment or this PR does not modify functionality that could affect the IT outcomePeer reviewer (after approval)
Note that after requesting changes, the PR must be assigned to only the author.
System administrator (after approval)
demoorno demono demono sandboxN reviewslabel is accurateOperator
reindex:…labels andrcommit title tagno demodevelopOperator (deploy
.sharedand.gitlabcomponents)_select dev.shared && CI_COMMIT_REF_NAME=develop make -C terraform/shared apply_keep_unusedor this PR is not labeleddeploy:shared_select dev.gitlab && CI_COMMIT_REF_NAME=develop make -C terraform/gitlab applyor this PR is not labeleddeploy:gitlab_select anvildev.shared && CI_COMMIT_REF_NAME=develop make -C terraform/shared apply_keep_unusedor this PR is not labeleddeploy:shared_select anvildev.gitlab && CI_COMMIT_REF_NAME=develop make -C terraform/gitlab applyor this PR is not labeleddeploy:gitlabdeploy:gitlabdeploy:gitlabSystem administrator (post-deploy of
.gitlabcomponent)dev.gitlabare complete or this PR is not labeleddeploy:gitlabanvildev.gitlabare complete or this PR is not labeleddeploy:gitlabOperator (deploy runner image)
_select dev.gitlab && make -C terraform/gitlab/runneror this PR is not labeleddeploy:runner_select anvildev.gitlab && make -C terraform/gitlab/runneror this PR is not labeleddeploy:runnerOperator (sandbox build)
sandboxlabel or PR is labeledno sandboxdevor PR is labeledno sandboxanvildevor PR is labeledno sandboxsandboxdeployment or PR is labeledno sandboxanvilboxdeployment or PR is labeledno sandboxsandboxdeployment or PR is labeledno sandboxanvilboxdeployment or PR is labeledno sandboxsandboxor this PR does not remove catalogs or otherwise causes unreferenced indices insandboxanvilboxor this PR does not remove catalogs or otherwise causes unreferenced indices inanvilboxsandboxor this PR is not labeledreindex:devanvilboxor this PR is not labeledreindex:anvildevsandboxor this PR is not labeledreindex:devanvilboxor this PR is not labeledreindex:anvildevOperator (merge the branch)
pif the PR is also labeledpartialOperator (main build)
devanvildevdevdevanvildevanvildev_select dev.shared && make -C terraform/shared applyor this PR is not labeleddeploy:shared_select anvildev.shared && make -C terraform/shared applyor this PR is not labeleddeploy:shareddevanvildevOperator (reindex)
devor this PR is neither labeledreindex:partialnorreindex:devanvildevor this PR is neither labeledreindex:partialnorreindex:anvildevdevor this PR is neither labeledreindex:partialnorreindex:devanvildevor this PR is neither labeledreindex:partialnorreindex:anvildevdevor this PR is neither labeledreindex:partialnorreindex:devanvildevor this PR is neither labeledreindex:partialnorreindex:anvildevdevor this PR does not require reindexingdevanvildevor this PR does not require reindexinganvildevdevor this PR does not require reindexingdevanvildevor this PR does not require reindexinganvildevdevor this PR does not require reindexingdevanvildevor this PR does not require reindexinganvildevdevor this PR does not require reindexingdevdevor this PR does not require reindexingdevdeploy_browserjob in the GitLab pipeline for this PR indevor this PR does not require reindexingdevanvildevor this PR does not require reindexinganvildevdeploy_browserjob in the GitLab pipeline for this PR inanvildevor this PR does not require reindexinganvildevOperator (mirroring)
devor this PR does not require mirroringdevanvildevor this PR does not require mirroringanvildevdevor this PR does not require mirroringdevanvildevor this PR does not require mirroringanvildevdevor this PR does not require mirroringdevanvildevor this PR does not require mirroringanvildevOperator
deploy:shared,deploy:gitlab,deploy:runner,API,reindex:partial,reindex:anvilprodandreindex:prodlabels to the next promotion PRs or this PR carries none of these labelsdeploy:shared,deploy:gitlab,deploy:runner,API,reindex:partial,reindex:anvilprodandreindex:prodlabels, from the description of this PR to that of the next promotion PRs or this PR carries none of these labelsShorthand for review comments
Lline is too longWline wrapping is wrongQbad quotesFother formatting problem