diff --git a/.dockerignore b/.dockerignore
index 71356e1..b79276d 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -153,3 +153,8 @@ cython_debug/
/mdast_cli/downloaded_apps/
/README_DOCKER.md
/mdast_cli.egg-info/
+
+# Local credentials and cross-platform binaries are not part of the build context.
+.git
+appstore_sessions/
+mdast_cli/distribution_systems/appstore_client/bin/
diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
index 63640e2..4bd1372 100644
--- a/.github/workflows/python-publish.yml
+++ b/.github/workflows/python-publish.yml
@@ -56,6 +56,14 @@ jobs:
build
--user
+ - uses: actions/setup-go@v6
+ with:
+ go-version: '1.27.1'
+ cache: false
+
+ - name: Build SAP helpers for release platforms
+ run: python tools/sap/build.py --target all
+
- name: Build a binary wheel and a source tarball
run: >-
python -m
@@ -65,6 +73,9 @@ jobs:
--outdir dist/
.
+ - name: Verify packaged SAP helpers
+ run: python tools/sap/verify_artifacts.py
+
- name: Publish distribution 📦 to Test PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 731413f..ddf156d 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -19,6 +19,13 @@ jobs:
- name: Install dependencies
run: pip install -r requirements.txt -r requirements-dev.txt
+ - uses: actions/setup-go@v6
+ with:
+ go-version: '1.27.1'
+ cache: false
+ - name: Build native SAP helper
+ run: python tools/sap/build.py --target host
+
- name: Run full test suite
run: python -m pytest -q
diff --git a/.gitignore b/.gitignore
index 59c188a..9bbaed7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -90,4 +90,6 @@ mdast_cli.egg-info/
Thumbs.db
# Application sessions (may contain sensitive data)
-appstore_sessions/
\ No newline at end of file
+appstore_sessions/
+# Reproducible native SAP helpers are built by CI, not committed.
+mdast_cli/distribution_systems/appstore_client/bin/
diff --git a/Dockerfile b/Dockerfile
index e11fd63..b224fb6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,14 +1,20 @@
-FROM python:3.9-slim
+FROM golang:1.27.1-bookworm AS go-toolchain
+FROM python:3.12-slim-bookworm AS sap-builder
+COPY --from=go-toolchain /usr/local/go /usr/local/go
+ENV PATH="/usr/local/go/bin:${PATH}"
+WORKDIR /src
+COPY tools/sap tools/sap
+RUN mkdir -p mdast_cli/distribution_systems/appstore_client/bin \
+ && python tools/sap/build.py --target host
+FROM python:3.12-slim-bookworm
WORKDIR /mdast_cli
-
COPY ./ /mdast_cli
-
-# Make apkeep_linux executable (if it exists)
+COPY --from=sap-builder /src/mdast_cli/distribution_systems/appstore_client/bin/ \
+ /mdast_cli/mdast_cli/distribution_systems/appstore_client/bin/
+COPY --from=sap-builder /src/mdast_cli/distribution_systems/appstore_client/IPATOOL-LICENSE \
+ /mdast_cli/mdast_cli/distribution_systems/appstore_client/IPATOOL-LICENSE
RUN if [ -f /mdast_cli/apkeep_linux ]; then chmod +x /mdast_cli/apkeep_linux; fi
-
-RUN pip install -r requirements.txt
-
-ENV PYTHONPATH "${PYTHONPATH}:/mdast_cli"
-
-ENTRYPOINT ["python3", "mdast_cli/mdast_scan.py"]
\ No newline at end of file
+RUN pip install --no-cache-dir -r requirements.txt
+ENV PYTHONPATH="/mdast_cli"
+ENTRYPOINT ["python3", "mdast_cli/mdast_scan.py"]
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..daa4bf0
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,5 @@
+include tools/sap/main.go tools/sap/build.py tools/sap/upstream.json
+include docs/STG-5076-appstore-sap.md
+recursive-include mdast_cli/distribution_systems/appstore_client/bin mdast-sap-*
+include mdast_cli/distribution_systems/appstore_client/IPATOOL-LICENSE
+include tools/sap/verify_artifacts.py
diff --git a/README.md b/README.md
index 2c346f2..b1bddfe 100644
--- a/README.md
+++ b/README.md
@@ -1179,3 +1179,22 @@ See LICENSE file for details.
For issues, questions, or contributions, please visit the GitHub repository or contact support.
**Note:** This documentation is maintained alongside the codebase. For the latest information, always refer to the version-specific documentation or the `--help` command output.
+
+### App Store authentication (2026.9.1)
+
+App Store login uses SAP-signed requests. The PyPI wheel includes native helpers
+for Linux, macOS and Windows (x86-64 and ARM64); Go is not required at runtime.
+On first login the signer downloads checksum-pinned Unicorn and Apple runtime
+assets. Allow outbound HTTPS to Apple, `swcdn.apple.com`, `s.mzstatic.com` and
+`files.pythonhosted.org` (Windows also uses the upstream runtime download hosts).
+The runtime cache is under the OS user cache directory in `ipatool/sap` and
+`ipatool/unicorn`. It contains runtime assets, not account sessions. A writable
+user cache directory is required. Alpine/musl is supported through its system
+loader. Offline first login is not supported; warm runtime assets avoid repeating
+these downloads. Account session caching continues to use `appstore_sessions`.
+
+For development from a source checkout, install Go 1.27.1 and run
+`python tools/sap/build.py --target host`. Release builds use `--target all`
+before building the wheel. See `docs/STG-5076-appstore-sap.md` for the protocol,
+source pin and acceptance criteria. The Apple frameworks are fetched directly
+from Apple at runtime and are not included in the package.
diff --git a/docs/STG-5076-appstore-sap.md b/docs/STG-5076-appstore-sap.md
new file mode 100644
index 0000000..31ad52f
--- /dev/null
+++ b/docs/STG-5076-appstore-sap.md
@@ -0,0 +1,69 @@
+# STG-5076: App Store SAP authentication
+
+## Problem and acceptance
+
+Fresh App Store authentication currently exhausts unsigned endpoint retries. A real
+CLI run from main 28c0b18 failed after 17 POSTs and 769 seconds. PyPI 2026.8.6 also
+fails and predates PR #149. Upstream ipatool v2.5.0 uses SAP-signed authentication.
+
+Implement locally first. Publishing a PR, merging and releasing are conditional on
+successful cold authentication and a valid IPA through the actual CLI entry point.
+A second process must download using its saved session. Then integrate the proven
+implementation into Markea and the monolith, test on dev and finish STG-5071 pod
+replacement acceptance. Do not automatically promote staging or production.
+
+## Design
+
+- Retain Python StoreClient and download/session interfaces.
+- Fetch Apple's bag over verified TLS and validate its authentication and SAP URLs.
+- Use a small subprocess adapter around the unmodified SAP signer from pinned
+ ipatool v2.5.0 (d5d0b56faf64e3fdef885d49e7928b390aadb6c7).
+- Build the adapter inside a verified temporary upstream source tree. Bundle
+ compiled helpers for Linux and macOS on amd64/arm64 in release artifacts, with
+ Windows support where build checks permit. No Go toolchain is required at runtime.
+ Keep the adapter source and reproducible build instructions in this repository.
+- The helper receives configuration and exact request bytes as bounded JSON lines
+ through stdin, returns signatures through stdout and never logs payloads. No
+ account credentials in process arguments. Initialize once per login, sign each
+ POST, close in finally. Enforce setup and sign deadlines and reap failed helpers.
+- Use X-Apple-ActionSignature (base64) for the exact serialized plist bytes. GUID
+ is the hex encoding of the hardware bytes passed to the signer.
+- Use only the bag authentication endpoint; validate every redirect before
+ forwarding credentials. Preserve body/attempt across pod redirects and transient
+ response retries. Limit redirects, protocol retries and HTTP retries explicitly.
+- No unsigned fallback. Classify missing/invalid SAP configuration, signature
+ failures, temporary Apple responses and authentication failures separately.
+- Upstream downloads its pinned, checksum-verified Unicorn runtime and Apple
+ framework assets on the first use. Document/cache these separately from account
+ sessions. Do not redistribute Apple frameworks in our package.
+
+## Checks
+
+- Unit coverage: exact signed bytes, bag validation, malicious redirects, malformed
+ and oversized helper responses, timeout/crash cleanup, 2FA/provider failures,
+ bounded retry behavior and unchanged successful download flow.
+- Helper build and protocol tests; complete existing CLI test suite.
+- Actual cold and warm CLI download with isolated cache and private credentials.
+- Verify published wheel and Docker version/content only after local acceptance.
+- Preserve Markea JSON session persistence and database locking.
+
+## Sources
+
+- https://tracker.yandex.ru/STG-5076
+- https://github.com/majd/ipatool/pull/525
+- https://github.com/majd/ipatool/releases/tag/v2.5.0
+
+## Local acceptance, 2026-09-07
+
+- Actual CLI cold login with empty account cache: signed POSTs 302 -> 200,
+ valid IPA 113,364,008 bytes, exit 0 in 35.13 seconds including first SAP setup.
+- Second CLI process: saved session restored, zero auth POSTs, valid IPA,
+ exit 0 in 7.13 seconds.
+- Installed wheel 2026.9.1, separate empty account cache: signed POSTs 302 -> 200,
+ valid IPA 113,364,009 bytes, exit 0 in 10.16 seconds (runtime assets already cached).
+- 241 Python tests pass. All six native targets build from the pinned Go module.
+- Linux amd64 Debian helper starts; Alpine amd64 completes real anonymous SAP
+ setup and creates a 501-byte signature in 25 seconds. Alpine uses its installed
+ musl loader explicitly because purego's default ELF interpreter names glibc.
+- Docker base aligned to Python 3.12, already required by the Python package.
+ Apple assets remain runtime downloads and are not bundled in released images.
diff --git a/mdast_cli/__init__.py b/mdast_cli/__init__.py
index 1db54d3..9cf81fe 100644
--- a/mdast_cli/__init__.py
+++ b/mdast_cli/__init__.py
@@ -1 +1 @@
-__version__ = '2026.8.6'
+__version__ = '2026.9.1'
diff --git a/mdast_cli/distribution_systems/appstore.py b/mdast_cli/distribution_systems/appstore.py
index 2b1ed9e..511a1f5 100644
--- a/mdast_cli/distribution_systems/appstore.py
+++ b/mdast_cli/distribution_systems/appstore.py
@@ -143,9 +143,7 @@ def login(self, force=False):
pickle.dump(self.store, file)
logger.info(f'Dumped session for {self.apple_id}')
except StoreException as e:
- raise RuntimeError(f'Failed to log into iTunes. This is either wrong credentials / an expired 2FA '
- f'code, or Apple refusing the request from this host. '
- f'Message: {e.req} {e.err_msg} {e.err_type}')
+ raise RuntimeError(f'Failed to log into iTunes: {e.err_msg} (type: {e.err_type})') from e
def get_app_info(self, app_id=None, bundle_id=None, country='US'):
if not app_id and not bundle_id:
diff --git a/mdast_cli/distribution_systems/appstore_client/IPATOOL-LICENSE b/mdast_cli/distribution_systems/appstore_client/IPATOOL-LICENSE
new file mode 100644
index 0000000..55c8aa7
--- /dev/null
+++ b/mdast_cli/distribution_systems/appstore_client/IPATOOL-LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Majd Alfhaily
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
\ No newline at end of file
diff --git a/mdast_cli/distribution_systems/appstore_client/sap.py b/mdast_cli/distribution_systems/appstore_client/sap.py
new file mode 100644
index 0000000..39b8a37
--- /dev/null
+++ b/mdast_cli/distribution_systems/appstore_client/sap.py
@@ -0,0 +1,177 @@
+"""Bounded subprocess bridge to the pinned ipatool SAP signer."""
+import base64
+import json
+import logging
+import platform
+import queue
+import subprocess
+import threading
+from pathlib import Path
+from urllib.parse import urlsplit
+
+logger = logging.getLogger(__name__)
+AUTH_PATH = '/WebObjects/MZFinance.woa/wa/authenticate'
+MAX_MESSAGE = 1 << 20
+SETUP_TIMEOUT = 1800
+SIGN_TIMEOUT = 120
+
+
+class SAPError(Exception):
+ pass
+
+
+def validate_endpoint(value, authentication=False):
+ try:
+ url = urlsplit(value)
+ host = (url.hostname or '').lower()
+ valid = (url.scheme == 'https' and url.port in (None, 443)
+ and not url.username and not url.password and not url.fragment
+ and (host.endswith('.apple.com') or host == 's.mzstatic.com'))
+ if authentication:
+ valid = (valid and url.path == AUTH_PATH
+ and (host == 'buy.itunes.apple.com'
+ or host.endswith('-buy.itunes.apple.com')))
+ if not valid:
+ raise ValueError
+ except (ValueError, TypeError, AttributeError):
+ raise SAPError('Invalid Apple authentication/SAP endpoint') from None
+ return value
+
+
+def parse_bag(data):
+ if not isinstance(data, dict):
+ raise SAPError('Apple bag is not a dictionary')
+ values = data.get('urlBag') or data.get('URLBag') or data
+ if not isinstance(values, dict):
+ raise SAPError('Apple bag has no SAP configuration')
+ if str(values.get('sign-sap-version')) != '200':
+ raise SAPError('Apple bag has missing or unsupported SAP version')
+ return {
+ 'auth_url': validate_endpoint(values.get('authenticateAccount'), True),
+ 'setup_url': validate_endpoint(values.get('sign-sap-setup')),
+ 'certificate_url': validate_endpoint(values.get('sign-sap-setup-cert')),
+ 'version': 200,
+ }
+
+
+def helper_path():
+ system = platform.system().lower()
+ arch = {'x86_64': 'amd64', 'aarch64': 'arm64', 'arm64': 'arm64',
+ 'amd64': 'amd64'}.get(platform.machine().lower())
+ suffix = '.exe' if system == 'windows' else ''
+ path = Path(__file__).with_name('bin') / f'mdast-sap-{system}-{arch}{suffix}'
+ if arch is None or not path.is_file():
+ raise SAPError('SAP helper is unavailable for this platform; install a complete mdast-cli release')
+ return path
+
+
+def helper_command():
+ binary = helper_path()
+ # Go/purego embeds the glibc interpreter in its Linux binary. On musl,
+ # invoke the installed system loader explicitly; upstream selects the
+ # corresponding checksum-pinned musllinux Unicorn runtime itself.
+ if platform.system() == 'Linux':
+ machine = {'amd64': 'x86_64', 'arm64': 'aarch64'}.get(
+ platform.machine().lower(), platform.machine().lower())
+ for directory in ('/lib', '/usr/lib'):
+ loader = Path(directory) / ('ld-musl-' + machine + '.so.1')
+ if loader.is_file():
+ return [str(loader), str(binary)]
+ return [str(binary)]
+
+
+class SAPSigner:
+ def __init__(self, config, guid):
+ self.process = None
+ try:
+ hardware = bytes.fromhex(guid)
+ if len(hardware) != 6:
+ raise ValueError
+ except (ValueError, TypeError):
+ raise SAPError('SAP requires a 12-character hexadecimal GUID') from None
+ try:
+ self.process = subprocess.Popen(
+ helper_command(), stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL,
+ )
+ logger.info('Preparing App Store SAP signer; first use downloads verified runtime assets')
+ self._exchange({
+ 'op': 'init', 'version': config['version'], 'hardware_id': guid,
+ 'setup_url': config['setup_url'], 'certificate_url': config['certificate_url'],
+ }, SETUP_TIMEOUT)
+ logger.info('App Store SAP signer ready')
+ except BaseException:
+ self.close()
+ raise
+
+ def _exchange(self, message, timeout):
+ data = json.dumps(message, separators=(',', ':')).encode() + b'\n'
+ if len(data) > MAX_MESSAGE:
+ raise SAPError('SAP request exceeds the size limit')
+ result = queue.Queue(maxsize=1)
+
+ process = self.process
+
+ def communicate():
+ try:
+ process.stdin.write(data)
+ process.stdin.flush()
+ result.put(process.stdout.readline(MAX_MESSAGE + 1))
+ except (OSError, ValueError):
+ result.put(None)
+
+ worker = threading.Thread(target=communicate, daemon=True)
+ worker.start()
+ try:
+ raw = result.get(timeout=timeout)
+ if not raw or len(raw) > MAX_MESSAGE or not raw.endswith(b'\n'):
+ raise SAPError('SAP helper stopped or returned an invalid response')
+ reply = json.loads(raw)
+ if not isinstance(reply, dict) or reply.get('ok') is not True:
+ # Upstream setup errors contain public asset/Apple URLs, but never
+ # expose arbitrary helper output or authentication payloads here.
+ raise SAPError('SAP helper failed during ' + message['op'])
+ return reply
+ except queue.Empty:
+ self.close()
+ raise SAPError('SAP helper timed out during ' + message['op']) from None
+ except (ValueError, TypeError):
+ self.close()
+ raise SAPError('SAP helper returned malformed JSON') from None
+ finally:
+ if self.process is None or self.process.poll() is not None:
+ worker.join(timeout=1)
+
+ def sign(self, payload):
+ reply = self._exchange({'op': 'sign', 'payload': base64.b64encode(payload).decode()}, SIGN_TIMEOUT)
+ try:
+ signature = base64.b64decode(reply['signature'], validate=True)
+ if not signature or len(signature) > 65536:
+ raise ValueError
+ return base64.b64encode(signature).decode('ascii')
+ except (KeyError, ValueError, TypeError):
+ raise SAPError('SAP helper returned an invalid signature') from None
+
+ def close(self):
+ process = self.process
+ if process is None:
+ return
+ # Terminate before closing pipes, which may still be blocked on setup.
+ # No signer state is persisted; all native resources belong to this process.
+ if process.poll() is None:
+ process.terminate()
+ try:
+ process.wait(timeout=3)
+ except subprocess.TimeoutExpired:
+ process.kill()
+ process.wait(timeout=3)
+ for pipe in (process.stdin, process.stdout):
+ if pipe is not None:
+ pipe.close()
+ self.process = None
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_):
+ self.close()
diff --git a/mdast_cli/distribution_systems/appstore_client/store.py b/mdast_cli/distribution_systems/appstore_client/store.py
index 79b2890..20cd97d 100755
--- a/mdast_cli/distribution_systems/appstore_client/store.py
+++ b/mdast_cli/distribution_systems/appstore_client/store.py
@@ -2,63 +2,20 @@
import logging
import os
import plistlib
-import random
import re
import time
-from typing import Optional
import requests
+from requests.adapters import HTTPAdapter
+from mdast_cli.distribution_systems.appstore_client.sap import SAPError, SAPSigner, parse_bag, validate_endpoint
logger = logging.getLogger(__name__)
-# Apple "bag" service: returns endpoint definitions (auth URL). Required since ~2025.
BAG_URL_TEMPLATE = "https://init.itunes.apple.com/bag.xml?guid=%s"
-# Apple's bag advertises the native auth endpoint, which only answers correctly when the
-# path ends with "/fast/" (trailing slash). Since July 2026 that endpoint also answers
-# 204/403/404/503 with an empty, non-plist body for many clients; the legacy MZFinance
-# endpoint still works, but replies 302 to an assigned pod host, and the original plist
-# body (with attempt=1) has to be reposted there. See majd/ipatool#513 / PR #514.
-AUTH_HOST = "auth.itunes.apple.com"
-DEFAULT_AUTH_URL = "https://" + AUTH_HOST + "/auth/v1/native/fast/"
LEGACY_AUTH_URL = "https://buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/authenticate"
-# Statuses that mean "this endpoint is not usable right now" when the body is not a plist.
-# Apple's edge also emits bare 301/302 responses that carry no Location header at all, so
-# the redirect statuses belong here too: without them a broken redirect aborts the login.
-AUTH_FALLBACK_STATUSES = (204, 301, 302, 303, 307, 308, 403, 404, 429, 500, 502, 503)
-# Apple answers this endpoint erratically and getting through is partly luck. Measured
-# in August 2026: 80 closely spaced requests got 0 usable answers, another burst got in
-# on the 34th, and a single request after two minutes of silence logged in immediately.
-# Spacing attempts out is not a guarantee, but it reached a working login in ~10 requests
-# where bursts needed dozens, so the backoff grows exponentially rather than hammering.
-# Jitter keeps parallel CLI runs from lining up into a burst of their own.
-AUTH_MAX_ROUNDS = int(os.environ.get("MDAST_APPSTORE_AUTH_ROUNDS", "8"))
-AUTH_ROUND_BACKOFF = float(os.environ.get("MDAST_APPSTORE_AUTH_BACKOFF", "20"))
-AUTH_MAX_BACKOFF = float(os.environ.get("MDAST_APPSTORE_AUTH_MAX_BACKOFF", "150"))
-AUTH_BACKOFF_JITTER = 5.0
AUTH_MAX_REDIRECTS = 4
BUY_DOMAIN = "buy.itunes.apple.com"
-def _normalize_auth_endpoint(endpoint):
- """Add the trailing slash the native auth endpoint requires.
-
- Apple's bag returns ".../auth/v1/native/fast"; posting without the trailing slash
- gets a 301/204 with an HTML or empty body that the plist parser chokes on
- (majd/ipatool#507). The legacy MZFinance endpoint is left untouched.
- """
- if endpoint and "/native/" in endpoint and not endpoint.endswith("/"):
- return endpoint + "/"
- return endpoint
-
-
-class _AuthEndpointUnusable(Exception):
- """Raised internally when an auth endpoint should be retried elsewhere."""
-
- def __init__(self, status_code, detail=""):
- self.status_code = status_code
- self.detail = detail
- super().__init__("auth endpoint unusable (HTTP %s) %s" % (status_code, detail))
-
-
# buyProduct lives on MZFinance, not MZBuy: the MZBuy variant answers HTTP 200 with
# m-allowed=False / cancel-purchase-batch=True ("Unable to process your request.") for
# every app, so no license is ever created and the download that follows fails with
@@ -114,49 +71,6 @@ def __init__(self, req, err_msg, err_type=None):
)
-def _parse_bag_response(content: bytes) -> Optional[str]:
- """Extract authenticateAccount URL from Apple bag plist/XML. Returns None if not found."""
- if not content or len(content) < 10:
- return None
- # Try direct plist parse (binary or XML)
- try:
- data = plistlib.loads(content)
- if isinstance(data, dict):
- # 'authenticateAccount' moved to the bag root (majd/ipatool#486); older bags
- # keep it under 'urlBag'. Check the root first, then fall back to urlBag.
- endpoint = data.get("authenticateAccount") or data.get("authenticate")
- if not endpoint:
- url_bag = data.get("urlBag") or data.get("URLBag")
- if isinstance(url_bag, dict):
- endpoint = url_bag.get("authenticateAccount") or url_bag.get("authenticate")
- return endpoint
- return None
- except Exception:
- pass
- # Try XML: unwrap Document and find plist/dict (ipatool-style normalization)
- try:
- text = content.decode("utf-8", errors="replace")
- # Extract inner body of ...
- doc_match = re.search(r"]*>(.*)", text, re.DOTALL | re.IGNORECASE)
- if doc_match:
- text = doc_match.group(1).strip()
- # Find authenticateAccountURL or similar
- key_match = re.search(
- r"\s*authenticateAccount\s*\s*([^<]+)",
- text,
- re.IGNORECASE,
- )
- if key_match:
- return key_match.group(1).strip()
- # Fallback: any key with "authenticate" and string value
- for m in re.finditer(r"\s*([^<]+)\s*\s*([^<]+)", text):
- if "authenticate" in m.group(1).lower():
- return m.group(2).strip()
- except Exception:
- pass
- return None
-
-
def _log_response_on_plist_error(r: requests.Response, context: str) -> None:
"""Log raw response details when plist parsing fails (e.g. HTML error page)."""
content = r.content
@@ -189,133 +103,96 @@ def __init__(self, sess: requests.Session, guid: str = None):
self.account_name = None
self.pod = None # Pod from auth response; used for purchase/download host (e.g. p25-buy.)
- def get_bag(self) -> str:
- """Fetch Apple bag and return auth endpoint URL (required since Apple changed endpoints)."""
- url = BAG_URL_TEMPLATE % self.guid
- r = self.sess.get(
- url,
+ def get_bag(self):
+ response = self.sess.get(
+ BAG_URL_TEMPLATE % self.guid,
headers={"Accept": "application/xml", "User-Agent": APPSTORE_USER_AGENT},
- verify=False,
- timeout=30,
+ verify=True, timeout=30,
)
- if r.status_code != 200:
- logger.warning(
- "Bag request failed: status=%s, falling back to default auth URL",
- r.status_code,
- )
- return DEFAULT_AUTH_URL
- auth_endpoint = _normalize_auth_endpoint(_parse_bag_response(r.content))
- if auth_endpoint:
- logger.debug("Using auth endpoint from bag: %s", auth_endpoint[:60] + "...")
- return auth_endpoint
- logger.warning("Could not parse bag response, falling back to default auth URL")
- return DEFAULT_AUTH_URL
-
- def _post_authenticate(self, url, appleId, password, attempt):
+ if response.status_code != 200:
+ raise SAPError("Apple bag request failed (HTTP %s)" % response.status_code)
+ try:
+ content = response.content
+ # Apple also wraps its plist in an outer Document element.
+ if b"", content, re.DOTALL)
+ if match:
+ content = match.group(0)
+ return parse_bag(plistlib.loads(content))
+ except (ValueError, TypeError, plistlib.InvalidFileException):
+ raise SAPError("Apple bag is not a valid SAP configuration") from None
+
+ def _post_authenticate(self, url, appleId, password, attempt, signer):
+ validate_endpoint(url, authentication=True)
req = StoreAuthenticateReq(
- appleId=appleId,
- password=password,
- attempt=str(attempt),
- createSession=None,
- guid=self.guid,
- rmp='0',
- why='signIn',
+ appleId=appleId, password=password, attempt=str(attempt),
+ createSession=None, guid=self.guid, rmp='0', why='signIn',
)
+ body = plistlib.dumps(req.as_dict())
+ signature = signer.sign(body)
+ # Signed POST retries are handled explicitly below, including re-signing.
+ # A general-purpose caller may have enabled urllib3 POST retries.
+ self.sess.mount(url.split('?')[0], HTTPAdapter(max_retries=0))
return self.sess.post(
url,
headers={
- "Accept": "*/*",
- "Content-Type": "application/x-www-form-urlencoded",
+ "Accept": "*/*", "Content-Type": "application/x-www-form-urlencoded",
"User-Agent": APPSTORE_USER_AGENT,
+ "X-Apple-ActionSignature": signature,
},
- data=plistlib.dumps(req.as_dict()),
- allow_redirects=False,
- verify=False,
- timeout=60,
+ data=body, allow_redirects=False, verify=True, timeout=60,
)
- def _authenticate_at(self, auth_url, appleId, password):
- """Authenticate against a single endpoint, following Apple's pod redirect.
-
- The legacy endpoint answers 302 with a Location pointing at the account's pod
- (e.g. https://p7-buy.itunes.apple.com/...?Pod=7&PRH=7). The original plist body
- must be reposted there unchanged - in particular attempt stays 1, otherwise Apple
- rejects the request (majd/ipatool#514).
- """
- url = auth_url
- attempt = 1
- redirects = 0
- r = None
+ def _authenticate_at(self, auth_url, appleId, password, signer):
+ url, attempt, redirects, transient = auth_url, 1, 0, 0
while True:
- r = self._post_authenticate(url, appleId, password, attempt)
- if r.status_code in (301, 302, 303, 307, 308) and r.headers.get('Location'):
+ response = self._post_authenticate(url, appleId, password, attempt, signer)
+ status = response.status_code
+ logger.info("App Store signed authentication response: HTTP %s", status)
+ if status in (301, 302, 303, 307, 308):
+ location = response.headers.get('Location')
+ if not location:
+ raise StoreException('authenticate', 'Apple redirect has no Location (HTTP %s)' % status)
+ validate_endpoint(location, authentication=True)
redirects += 1
if redirects > AUTH_MAX_REDIRECTS:
- raise _AuthEndpointUnusable(r.status_code, "too many redirects")
- url = r.headers['Location']
- logger.debug("Auth redirected to pod endpoint: %s", url)
- continue # attempt is intentionally NOT incremented here
+ raise StoreException('authenticate', 'Too many Apple authentication redirects')
+ url = location
+ continue
try:
- resp = StoreAuthenticateResp.from_dict(plistlib.loads(r.content))
- except plistlib.InvalidFileException:
- _log_response_on_plist_error(r, "authenticate")
- raise _AuthEndpointUnusable(r.status_code, "non-plist body") from None
- if resp.m_allowed:
- return r, resp
- # Apple sometimes rejects the very first attempt with an invalid-credentials
- # failure; a single retry with attempt=2 clears it (ipatool does the same).
+ data = plistlib.loads(response.content)
+ if not isinstance(data, dict):
+ raise ValueError
+ except (ValueError, TypeError, plistlib.InvalidFileException):
+ transient += 1
+ if (status in (204, 404) or status >= 500) and transient < 3:
+ time.sleep(0.25 * transient)
+ continue
+ raise StoreException('authenticate', 'Apple returned a non-plist authentication response (HTTP %s)' % status) from None
+ resp = StoreAuthenticateResp.from_dict(data)
+ # Both response shapes are in use: older clients read download-queue-info.
+ dsid = data.get('dsPersonId') or (data.get('download-queue-info') or {}).get('dsid')
+ if status == 200 and resp.passwordToken and dsid and not resp.failureType:
+ if not resp.download_queue_info:
+ resp = StoreAuthenticateResp.from_dict(dict(data, **{'download-queue-info': {'dsid': dsid}}))
+ return response, resp
if attempt == 1 and str(resp.failureType) == '-5000':
attempt = 2
+ transient = 0
continue
- raise StoreException("authenticate", resp.customerMessage, resp.failureType)
+ raise StoreException('authenticate', resp.customerMessage or 'Apple rejected authentication', resp.failureType)
def authenticate(self, appleId, password):
if not self.guid:
self.guid = self._generateGuid(appleId)
-
- endpoints = []
- for candidate in (self.get_bag(), DEFAULT_AUTH_URL, LEGACY_AUTH_URL):
- if candidate and candidate not in endpoints:
- endpoints.append(candidate)
-
- last_failure = None
- for round_no in range(1, AUTH_MAX_ROUNDS + 1):
- for auth_url in endpoints:
- try:
- r, resp = self._authenticate_at(auth_url, appleId, password)
- self._store_auth_result(r, resp, auth_url)
- return resp
- except _AuthEndpointUnusable as e:
- if e.status_code not in AUTH_FALLBACK_STATUSES:
- raise StoreException(
- "authenticate",
- "Server response is not valid plist (HTTP %s). See log for details."
- % e.status_code,
- None,
- ) from e
- last_failure = e
- logger.warning(
- "Auth endpoint %s unusable (HTTP %s, %s), trying next endpoint",
- auth_url, e.status_code, e.detail,
- )
- if round_no < AUTH_MAX_ROUNDS:
- delay = min(AUTH_ROUND_BACKOFF * (2 ** (round_no - 1)), AUTH_MAX_BACKOFF)
- delay += random.uniform(0, AUTH_BACKOFF_JITTER)
- logger.info(
- "All App Store auth endpoints failed (round %s/%s), waiting %.0fs before "
- "retrying - Apple's auth endpoint is erratic, spacing attempts out helps",
- round_no, AUTH_MAX_ROUNDS, delay,
- )
- time.sleep(delay)
-
- raise StoreException(
- "authenticate",
- "Apple rejected every authentication endpoint (last status: HTTP %s). "
- "This is an Apple-side/network block rather than a credentials problem: "
- "retry later or from a different egress IP (see majd/ipatool#513)."
- % (last_failure.status_code if last_failure else "unknown"),
- None,
- )
+ try:
+ config = self.get_bag()
+ with SAPSigner(config, self.guid) as signer:
+ response, result = self._authenticate_at(config['auth_url'], appleId, password, signer)
+ self._store_auth_result(response, result, config['auth_url'])
+ return result
+ except SAPError as exc:
+ raise StoreException('authenticate', str(exc), 'sap') from exc
def _store_auth_result(self, r, resp, auth_url):
self.sess.headers['X-Dsid'] = self.sess.headers['iCloud-Dsid'] = str(resp.download_queue_info.dsid)
@@ -336,7 +213,10 @@ def _store_auth_result(self, r, resp, auth_url):
if self.pod:
logger.debug("Using pod for buy host: %s", self.pod)
- self.account_name = resp.accountInfo.address.firstName + " " + resp.accountInfo.address.lastName
+ address = getattr(resp.accountInfo, 'address', None)
+ self.account_name = " ".join(filter(None, (
+ getattr(address, 'firstName', None), getattr(address, 'lastName', None),
+ )))
def _buy_host(self) -> str:
"""Host for purchase/download (pod-specific if set)."""
diff --git a/setup.py b/setup.py
index 56054c9..1f50661 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
setup(
name="mdast_cli",
- version='2026.8.6',
+ version='2026.9.1',
python_requires='>=3.12',
@@ -17,7 +17,8 @@
url="https://github.com/Dynamic-Mobile-Security/mdast-cli",
packages=find_packages(),
include_package_data=True,
- package_data={'': ['device.properties']},
+ package_data={'': ['device.properties'],
+ 'mdast_cli.distribution_systems.appstore_client': ['bin/mdast-sap-*', 'IPATOOL-LICENSE']},
install_requires=[
'altgraph==0.17',
'beautifulsoup4==4.10.0',
diff --git a/tests/test_appstore_auth.py b/tests/test_appstore_auth.py
index e69e22c..deb26f3 100644
--- a/tests/test_appstore_auth.py
+++ b/tests/test_appstore_auth.py
@@ -1,168 +1,179 @@
-"""Regression tests for the App Store authentication flow.
-
-Apple broke the native auth endpoint in July 2026: it answers 204/403/404/503 with an
-empty, non-plist body. The working flow (mirrored from majd/ipatool#514) is:
-
- native /auth/v1/native/fast/ -> 204/403/404/503
- legacy MZFinance authenticate -> 302 Location: https://pN-buy.itunes.apple.com/...
- repost the SAME plist body (attempt still 1) to the pod URL -> 200 + plist
-"""
+"""Signed App Store authentication and existing download regressions."""
+import base64
import plistlib
import pytest
from mdast_cli.distribution_systems.appstore_client import store as store_mod
-from mdast_cli.distribution_systems.appstore_client.store import (
- LEGACY_AUTH_URL,
- StoreClient,
- StoreException,
- _normalize_auth_endpoint,
-)
+from mdast_cli.distribution_systems.appstore_client.store import LEGACY_AUTH_URL, StoreClient, StoreException
POD_URL = "https://p7-buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/authenticate?Pod=7&PRH=7"
-
SUCCESS_PLIST = {
- "m-allowed": True,
- "passwordToken": "token-123",
- "download-queue-info": {"dsid": 4242},
+ "m-allowed": True, "passwordToken": "token-123", "download-queue-info": {"dsid": 4242},
"accountInfo": {"address": {"firstName": "Test", "lastName": "User"}},
}
+BAG_CONFIG = {
+ 'auth_url': LEGACY_AUTH_URL, 'setup_url': 'https://fpinit.itunes.apple.com/setup',
+ 'certificate_url': 'https://s.mzstatic.com/sap/setupCert.plist', 'version': 200,
+}
class FakeResponse:
def __init__(self, status_code, content=b"", headers=None, url=""):
- self.status_code = status_code
- self.content = content
- self.headers = headers or {}
- self.url = url
- self.text = content.decode("utf-8", "replace")
+ self.status_code, self.content = status_code, content
+ self.headers, self.url = headers or {}, url
class FakeSession:
- """Records POSTs and replays a scripted list of responses."""
-
def __init__(self, responses):
- self._responses = list(responses)
- self.headers = {}
- self.calls = []
+ self._responses, self.headers, self.calls = list(responses), {}, []
+
+ def mount(self, *args):
+ pass
def post(self, url, headers=None, data=None, **kwargs):
- self.calls.append({"url": url, "body": plistlib.loads(data)})
+ self.calls.append({'url': url, 'body': plistlib.loads(data), 'data': data, 'headers': headers, **kwargs})
return self._responses.pop(0)
-def _client(responses):
- client = StoreClient(FakeSession(responses), guid="12367150C7F5")
- return client
-
-
-@pytest.fixture(autouse=True)
-def _no_sleep(monkeypatch):
- monkeypatch.setattr(store_mod.time, "sleep", lambda *_: None)
-
+class FakeSigner:
+ def __init__(self, config, guid):
+ self.payloads, self.closed = [], False
+ FakeSigner.last = self
-@pytest.fixture
-def _bag_returns_legacy(monkeypatch):
- monkeypatch.setattr(StoreClient, "get_bag", lambda self: LEGACY_AUTH_URL)
+ def sign(self, payload):
+ self.payloads.append(payload)
+ return base64.b64encode(b'signed:' + payload).decode()
+ def __enter__(self):
+ return self
-@pytest.mark.parametrize(
- "endpoint,expected",
- [
- ("https://auth.itunes.apple.com/auth/v1/native/fast", "https://auth.itunes.apple.com/auth/v1/native/fast/"),
- ("https://auth.itunes.apple.com/auth/v1/native/fast/", "https://auth.itunes.apple.com/auth/v1/native/fast/"),
- (LEGACY_AUTH_URL, LEGACY_AUTH_URL),
- (None, None),
- ],
-)
-def test_normalize_auth_endpoint(endpoint, expected):
- assert _normalize_auth_endpoint(endpoint) == expected
+ def __exit__(self, *_):
+ self.closed = True
-def test_pod_redirect_reposts_body_with_attempt_one(_bag_returns_legacy):
- """Apple rejects the pod repost if `attempt` is bumped, so it must stay 1."""
- client = _client([
- FakeResponse(302, headers={"Location": POD_URL}),
- FakeResponse(200, plistlib.dumps(SUCCESS_PLIST), headers={"pod": "7"}, url=POD_URL),
- ])
+def _client(responses):
+ return StoreClient(FakeSession(responses), guid='12367150C7F5')
- resp = client.authenticate("user@example.com", "secret123456")
- assert resp.passwordToken == "token-123"
- assert client.account_name == "Test User"
- assert client.pod == "7"
- urls = [c["url"] for c in client.sess.calls]
- assert urls == [LEGACY_AUTH_URL, POD_URL]
- assert [c["body"]["attempt"] for c in client.sess.calls] == ["1", "1"]
- assert client.sess.calls[0]["body"] == client.sess.calls[1]["body"]
+@pytest.fixture(autouse=True)
+def _signed_bag(monkeypatch):
+ monkeypatch.setattr(store_mod.time, 'sleep', lambda *_: None)
+ monkeypatch.setattr(StoreClient, 'get_bag', lambda self: BAG_CONFIG)
+ monkeypatch.setattr(store_mod, 'SAPSigner', FakeSigner)
-@pytest.mark.parametrize("status", [204, 403, 404, 503])
-def test_falls_back_to_next_endpoint_on_empty_body(monkeypatch, status):
- """A non-plist body on the bag endpoint must not abort the login."""
- monkeypatch.setattr(StoreClient, "get_bag", lambda self: "https://auth.itunes.apple.com/auth/v1/native/fast/")
+def test_pod_redirect_reposts_body_with_attempt_one():
client = _client([
- FakeResponse(status, b""),
- FakeResponse(302, headers={"Location": POD_URL}),
- FakeResponse(200, plistlib.dumps(SUCCESS_PLIST), url=POD_URL),
+ FakeResponse(302, headers={'Location': POD_URL}),
+ FakeResponse(200, plistlib.dumps(SUCCESS_PLIST), headers={'pod': '7'}, url=POD_URL),
])
-
- client.authenticate("user@example.com", "secret123456")
-
- urls = [c["url"] for c in client.sess.calls]
- assert urls == ["https://auth.itunes.apple.com/auth/v1/native/fast/", LEGACY_AUTH_URL, POD_URL]
- assert client.pod == "7" # derived from the pod URL when the header is absent
+ response = client.authenticate('user@example.com', 'secret123456')
+ assert response.passwordToken == 'token-123'
+ assert client.account_name == 'Test User'
+ assert client.pod == '7'
+ assert [c['url'] for c in client.sess.calls] == [LEGACY_AUTH_URL, POD_URL]
+ assert [c['body']['attempt'] for c in client.sess.calls] == ['1', '1']
+ assert FakeSigner.last.payloads == [c['data'] for c in client.sess.calls]
+ assert len(set(FakeSigner.last.payloads)) == 1
+ assert FakeSigner.last.closed
+
+
+@pytest.mark.parametrize('status', [204, 404, 500, 503, 504])
+def test_transient_response_retries_same_signed_endpoint(status):
+ client = _client([FakeResponse(status), FakeResponse(200, plistlib.dumps(SUCCESS_PLIST))])
+ client.authenticate('user@example.com', 'secret123456')
+ assert [c['url'] for c in client.sess.calls] == [LEGACY_AUTH_URL] * 2
+ assert len(set(FakeSigner.last.payloads)) == 1
+ assert len(FakeSigner.last.payloads) == 2
+
+
+def test_invalid_credentials_are_reported():
+ failure = {'failureType': '1234', 'customerMessage': 'Incorrect password'}
+ client = _client([FakeResponse(200, plistlib.dumps(failure))])
+ with pytest.raises(StoreException, match='Incorrect password'):
+ client.authenticate('user@example.com', 'wrong')
+ assert len(client.sess.calls) == 1
+ assert FakeSigner.last.closed
-def test_invalid_credentials_are_reported_not_retried_forever(_bag_returns_legacy):
- failure = {"m-allowed": False, "customerMessage": "Your Apple ID or password was incorrect.",
- "failureType": "1234"}
- client = _client([FakeResponse(200, plistlib.dumps(failure))])
+def test_first_attempt_invalid_credentials_retried_once():
+ failure = {'failureType': '-5000', 'customerMessage': 'retry'}
+ client = _client([FakeResponse(200, plistlib.dumps(failure)), FakeResponse(200, plistlib.dumps(SUCCESS_PLIST))])
+ client.authenticate('user@example.com', 'secret')
+ assert [c['body']['attempt'] for c in client.sess.calls] == ['1', '2']
+ assert FakeSigner.last.payloads[0] != FakeSigner.last.payloads[1]
- with pytest.raises(StoreException) as exc:
- client.authenticate("user@example.com", "wrong")
- assert "incorrect" in str(exc.value)
+def test_forbidden_response_fails_without_unsigned_fallback():
+ client = _client([FakeResponse(403)])
+ with pytest.raises(StoreException, match='HTTP 403'):
+ client.authenticate('user@example.com', 'secret')
assert len(client.sess.calls) == 1
+ assert FakeSigner.last.closed
-def test_first_attempt_invalid_credentials_is_retried_once(_bag_returns_legacy):
- """Apple spuriously fails attempt 1 with -5000; attempt 2 clears it."""
- failure = {"m-allowed": False, "customerMessage": "retry", "failureType": "-5000"}
- client = _client([
- FakeResponse(200, plistlib.dumps(failure)),
- FakeResponse(200, plistlib.dumps(SUCCESS_PLIST)),
- ])
+def test_transient_response_budget_is_three():
+ client = _client([FakeResponse(204) for _ in range(3)])
+ with pytest.raises(StoreException, match='HTTP 204'):
+ client.authenticate('user@example.com', 'secret')
+ assert len(client.sess.calls) == 3
- client.authenticate("user@example.com", "secret123456")
- assert [c["body"]["attempt"] for c in client.sess.calls] == ["1", "2"]
+@pytest.mark.parametrize('status', [301, 302])
+def test_redirect_without_location_is_rejected(status):
+ client = _client([FakeResponse(status)])
+ with pytest.raises(StoreException, match='no Location'):
+ client.authenticate('user@example.com', 'secret')
+ assert len(client.sess.calls) == 1
-def test_all_endpoints_blocked_raises_actionable_error(_bag_returns_legacy):
- client = _client([FakeResponse(403, b"") for _ in range(2 * store_mod.AUTH_MAX_ROUNDS)])
+@pytest.mark.parametrize('url', [
+ 'https://evil.example/WebObjects/MZFinance.woa/wa/authenticate',
+ 'http://buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/authenticate',
+ 'https://buy.itunes.apple.com.evil.example/WebObjects/MZFinance.woa/wa/authenticate',
+ 'https://user:pass@buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/authenticate',
+ 'https://buy.itunes.apple.com:8443/WebObjects/MZFinance.woa/wa/authenticate',
+ 'https://buy.itunes.apple.com/unexpected',
+])
+def test_redirect_does_not_forward_credentials_to_invalid_endpoint(url):
+ client = _client([FakeResponse(302, headers={'Location': url})])
+ with pytest.raises(StoreException, match='endpoint'):
+ client.authenticate('user@example.com', 'secret')
+ assert len(client.sess.calls) == 1
+ assert FakeSigner.last.closed
- with pytest.raises(StoreException) as exc:
- client.authenticate("user@example.com", "secret123456")
- assert "Apple-side/network block" in str(exc.value)
+def test_action_signature_covers_exact_request_bytes():
+ client = _client([FakeResponse(200, plistlib.dumps(SUCCESS_PLIST))])
+ client.authenticate('user@example.com', 'password123456')
+ call = client.sess.calls[0]
+ assert base64.b64decode(call['headers']['X-Apple-ActionSignature']) == b'signed:' + call['data']
+ assert call['body']['password'] == 'password123456'
+ assert call['verify'] is True
+ assert call['allow_redirects'] is False
-@pytest.mark.parametrize("status", [301, 302])
-def test_redirect_without_location_is_retried(_bag_returns_legacy, status):
- """Apple's edge emits bare 30x responses with no Location; that must not abort login."""
- client = _client([
- FakeResponse(status, b""), # bag/legacy endpoint, broken redirect
- FakeResponse(status, b""), # native endpoint, same
- FakeResponse(302, headers={"Location": POD_URL}), # next round: real redirect
- FakeResponse(200, plistlib.dumps(SUCCESS_PLIST), url=POD_URL),
- ])
+def test_current_response_shape_without_download_queue():
+ data = dict(SUCCESS_PLIST, dsPersonId='4242')
+ del data['download-queue-info']
+ client = _client([FakeResponse(200, plistlib.dumps(data))])
+ client.authenticate('user@example.com', 'secret')
+ assert str(client.dsid) == '4242'
- client.authenticate("user@example.com", "secret123456")
- assert [c["url"] for c in client.sess.calls][-1] == POD_URL
+def test_redirect_loop_is_bounded():
+ client = _client([FakeResponse(302, headers={'Location': POD_URL}) for _ in range(5)])
+ with pytest.raises(StoreException, match='Too many'):
+ client.authenticate('user@example.com', 'secret')
+ assert len(client.sess.calls) == 5
+def test_two_factor_required_is_reported_without_retries():
+ client = _client([FakeResponse(200, plistlib.dumps({'customerMessage': 'MZFinance.BadLogin.Configurator_message'}))])
+ with pytest.raises(StoreException, match='BadLogin'):
+ client.authenticate('user@example.com', 'secret')
+ assert len(client.sess.calls) == 1
# --- purchase / download -----------------------------------------------------------
# buyProduct on MZBuy answers HTTP 200 with m-allowed=False for every app, so no license
# is ever created and the download that follows fails with failureType 9610. The license
diff --git a/tests/test_appstore_sap.py b/tests/test_appstore_sap.py
new file mode 100644
index 0000000..c3bb9f2
--- /dev/null
+++ b/tests/test_appstore_sap.py
@@ -0,0 +1,139 @@
+"""Protocol boundaries and process cleanup for the native SAP bridge."""
+import base64
+import io
+import json
+import subprocess
+import time
+
+import pytest
+import requests
+import responses
+
+from mdast_cli.distribution_systems.appstore_client import sap
+from mdast_cli.distribution_systems.appstore_client.store import StoreClient
+
+BAG = {
+ 'authenticateAccount': 'https://buy.itunes.apple.com/WebObjects/MZFinance.woa/wa/authenticate',
+ 'sign-sap-setup': 'https://fpinit.itunes.apple.com/v1/signSapSetup/legacy',
+ 'sign-sap-setup-cert': 'https://s.mzstatic.com/sap/setupCert.plist',
+ 'sign-sap-version': '200',
+}
+
+
+@pytest.mark.parametrize('wrapper', [lambda x:x, lambda x:{'urlBag':x}, lambda x:{'URLBag':x}])
+def test_bag_accepts_apple_certificate_cdn(wrapper):
+ assert sap.parse_bag(wrapper(BAG))['certificate_url'] == BAG['sign-sap-setup-cert']
+
+
+@pytest.mark.parametrize('field,value', [
+ ('sign-sap-version', None), ('sign-sap-version', '201'),
+ ('authenticateAccount', 'https://evil.example/'),
+ ('sign-sap-setup', 'http://fpinit.itunes.apple.com/setup'),
+ ('sign-sap-setup-cert', 'https://s.mzstatic.com.evil.example/setupCert.plist'),
+ ('sign-sap-setup-cert', None),
+])
+def test_bag_fails_closed_on_invalid_configuration(field, value):
+ with pytest.raises(sap.SAPError):
+ sap.parse_bag(dict(BAG, **{field:value}))
+
+
+@responses.activate
+def test_bag_uses_configurator_user_agent_and_requires_sap():
+ import plistlib
+ url = 'https://init.itunes.apple.com/bag.xml?guid=AABBCCDDEEFF'
+ responses.get(url, body=plistlib.dumps({'urlBag': BAG}), status=200)
+ client = StoreClient(requests.Session(), guid='AABBCCDDEEFF')
+ assert client.get_bag()['version'] == 200
+ assert responses.calls[0].request.headers['User-Agent'].startswith('Configurator/')
+
+
+class Process:
+ def __init__(self, output):
+ self.stdin, self.stdout = io.BytesIO(), io.BytesIO(output)
+ self.returncode = None
+ self.terminated = False
+
+ def poll(self):
+ return self.returncode
+
+ def terminate(self):
+ self.terminated = True
+ self.returncode = -15
+
+ def wait(self, timeout=None):
+ return self.returncode
+
+
+def make_signer(monkeypatch, output):
+ process = Process(output)
+ monkeypatch.setattr(sap, 'helper_command', lambda: ['/test/mdast-sap'])
+ captured = {}
+ def popen(args, **kwargs):
+ captured.update(args=args, **kwargs)
+ return process
+ monkeypatch.setattr(sap.subprocess, 'Popen', popen)
+ return process, captured
+
+
+def test_protocol_keeps_credentials_out_of_process_arguments(monkeypatch):
+ process, captured = make_signer(monkeypatch, b'{"ok":true}\n{"ok":true,"signature":"YWJj"}\n')
+ with sap.SAPSigner(sap.parse_bag(BAG), 'AABBCCDDEEFF') as signer:
+ assert signer.sign(b'private request bytes') == base64.b64encode(b'abc').decode()
+ sent = [json.loads(line) for line in process.stdin.getvalue().splitlines()]
+ assert base64.b64decode(sent[1]['payload']) == b'private request bytes'
+ assert captured['args'] == ['/test/mdast-sap']
+ assert captured['stderr'] == subprocess.DEVNULL
+ assert process.terminated
+
+
+@pytest.mark.parametrize('output', [b'', b'no json\n', b'[]\n', b'{}\n', b'{"ok":false,"detail":"private"}\n', b'x'*(sap.MAX_MESSAGE+1)])
+def test_invalid_init_response_reaps_helper_without_leaking_output(monkeypatch, output):
+ process, _ = make_signer(monkeypatch, output)
+ with pytest.raises(sap.SAPError) as error:
+ sap.SAPSigner(sap.parse_bag(BAG), 'AABBCCDDEEFF')
+ assert 'private' not in str(error.value)
+ assert process.terminated
+
+
+def test_setup_timeout_reaps_helper(monkeypatch):
+ process, _ = make_signer(monkeypatch, b'')
+ class SlowOutput(io.BytesIO):
+ def readline(self, *_):
+ time.sleep(0.03)
+ return b''
+ process.stdout = SlowOutput()
+ monkeypatch.setattr(sap, 'SETUP_TIMEOUT', 0.001)
+ with pytest.raises(sap.SAPError, match='timed out'):
+ sap.SAPSigner(sap.parse_bag(BAG), 'AABBCCDDEEFF')
+ assert process.terminated
+
+
+@pytest.mark.parametrize('signature', [None, '', '@@@', ' '])
+def test_invalid_signature_rejected(monkeypatch, signature):
+ output = b'{"ok":true}\n' + json.dumps({'ok':True, 'signature':signature}).encode()+b'\n'
+ process, _ = make_signer(monkeypatch, output)
+ with sap.SAPSigner(sap.parse_bag(BAG), 'AABBCCDDEEFF') as signer:
+ with pytest.raises(sap.SAPError, match='signature'):
+ signer.sign(b'private')
+ assert process.terminated
+
+
+def test_compiled_helper_protocol_without_network():
+ try:
+ binary = sap.helper_path()
+ except sap.SAPError:
+ pytest.skip('helper not built in this source checkout')
+ result = subprocess.run([str(binary)], input=b'{"op":"sign","payload":"YWJj"}\n',
+ capture_output=True, timeout=10)
+ assert result.returncode == 0
+ assert json.loads(result.stdout)['error'] == 'invalid_sign_state'
+ assert b'YWJj' not in result.stderr
+
+
+@pytest.mark.parametrize('machine,loader', [('x86_64','/lib/ld-musl-x86_64.so.1'), ('aarch64','/lib/ld-musl-aarch64.so.1')])
+def test_musl_uses_system_loader(monkeypatch, machine, loader):
+ monkeypatch.setattr(sap, 'helper_path', lambda: '/test/mdast-sap')
+ monkeypatch.setattr(sap.platform, 'system', lambda: 'Linux')
+ monkeypatch.setattr(sap.platform, 'machine', lambda: machine)
+ monkeypatch.setattr(sap.Path, 'is_file', lambda self: str(self) == loader)
+ assert sap.helper_command() == [loader, '/test/mdast-sap']
diff --git a/tools/sap/build.py b/tools/sap/build.py
new file mode 100644
index 0000000..bbc1b60
--- /dev/null
+++ b/tools/sap/build.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""Build the SAP bridge from Go-checksum-verified, pinned ipatool sources."""
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+
+ROOT = Path(__file__).resolve().parents[2]
+TARGETS = ('linux-amd64', 'linux-arm64', 'darwin-amd64', 'darwin-arm64',
+ 'windows-amd64', 'windows-arm64')
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--go', default='go')
+ parser.add_argument('--target', choices=(*TARGETS, 'host', 'all'), default='host')
+ args = parser.parse_args()
+ pin = json.loads((ROOT / 'tools/sap/upstream.json').read_text())
+ dependency = pin['module'] + '@' + pin['version']
+ module = json.loads(subprocess.check_output([args.go, 'mod', 'download', '-json', dependency]))
+ if module.get('Error') or module.get('Sum') != pin['sum']:
+ raise SystemExit('Pinned ipatool source checksum verification failed')
+ if args.target == 'host':
+ goos, goarch = subprocess.check_output([args.go, 'env', 'GOOS', 'GOARCH'], text=True).split()
+ targets = [goos + '-' + goarch]
+ else:
+ targets = TARGETS if args.target == 'all' else [args.target]
+ out = ROOT / 'mdast_cli/distribution_systems/appstore_client/bin'
+ out.mkdir(exist_ok=True)
+ with tempfile.TemporaryDirectory(prefix='mdast-sap-build-') as temporary:
+ source = Path(temporary) / 'ipatool'
+ shutil.copytree(module['Dir'], source)
+ # Go module cache files are read-only; leave the cache untouched.
+ for path in source.rglob('*'):
+ path.chmod(0o755 if path.is_dir() else 0o644)
+ command = source / 'cmd/mdast-sap'
+ command.mkdir()
+ shutil.copy2(ROOT / 'tools/sap/main.go', command / 'main.go')
+ for target in targets:
+ goos, goarch = target.split('-')
+ binary = out / ('mdast-sap-' + target + ('.exe' if goos == 'windows' else ''))
+ env = dict(os.environ, GOOS=goos, GOARCH=goarch, CGO_ENABLED='0')
+ subprocess.run([args.go, 'build', '-mod=readonly', '-trimpath', '-buildvcs=false',
+ '-ldflags=-s -w', '-o', str(binary), './cmd/mdast-sap'],
+ cwd=source, env=env, check=True)
+ binary.chmod(0o755)
+ print(json.dumps({'target': target, 'sha256': hashlib.sha256(binary.read_bytes()).hexdigest()}), flush=True)
+ shutil.copy2(source / 'LICENSE', ROOT / 'mdast_cli/distribution_systems/appstore_client/IPATOOL-LICENSE')
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/sap/main.go b/tools/sap/main.go
new file mode 100644
index 0000000..f4d770f
--- /dev/null
+++ b/tools/sap/main.go
@@ -0,0 +1,95 @@
+// Built inside pinned ipatool sources to use its internal SAP implementation.
+package main
+
+import (
+ "bufio"
+ "context"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/majd/ipatool/v2/internal/sap"
+)
+
+type request struct {
+ Operation string `json:"op"`
+ Version uint32 `json:"version"`
+ SetupURL string `json:"setup_url"`
+ CertificateURL string `json:"certificate_url"`
+ HardwareID string `json:"hardware_id"`
+ Payload []byte `json:"payload"`
+}
+
+type response struct {
+ OK bool `json:"ok"`
+ Signature []byte `json:"signature,omitempty"`
+ Error string `json:"error,omitempty"`
+ Detail string `json:"detail,omitempty"`
+}
+
+func run() error {
+ scanner := bufio.NewScanner(os.Stdin)
+ scanner.Buffer(make([]byte, 4096), 1<<20)
+ encoder := json.NewEncoder(os.Stdout)
+ var signer sap.ActionSigner
+ defer func() {
+ if signer != nil {
+ _ = signer.Close()
+ }
+ }()
+ deadline := time.AfterFunc(35*time.Minute, func() { os.Exit(124) })
+ defer deadline.Stop()
+ for scanner.Scan() {
+ var req request
+ if err := json.Unmarshal(scanner.Bytes(), &req); err != nil {
+ return encoder.Encode(response{Error: "invalid_request"})
+ }
+ switch req.Operation {
+ case "init":
+ if signer != nil {
+ return encoder.Encode(response{Error: "already_initialized"})
+ }
+ hardware, err := hex.DecodeString(req.HardwareID)
+ if err != nil {
+ return encoder.Encode(response{Error: "invalid_hardware_id"})
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+ signer, err = sap.NewSigner(ctx, sap.Config{
+ SetupURL: req.SetupURL, CertificateURL: req.CertificateURL,
+ Version: req.Version, HardwareID: hardware,
+ })
+ cancel()
+ if err != nil {
+ return encoder.Encode(response{Error: "initialization_failed", Detail: err.Error()})
+ }
+ if err := encoder.Encode(response{OK: true}); err != nil {
+ return err
+ }
+ case "sign":
+ if signer == nil || len(req.Payload) == 0 {
+ return encoder.Encode(response{Error: "invalid_sign_state"})
+ }
+ signature, err := signer.Sign(req.Payload)
+ if err != nil {
+ return encoder.Encode(response{Error: "signing_failed"})
+ }
+ if err := encoder.Encode(response{OK: true, Signature: signature}); err != nil {
+ return err
+ }
+ case "close":
+ return nil
+ default:
+ return encoder.Encode(response{Error: "unknown_operation"})
+ }
+ }
+ return scanner.Err()
+}
+
+func main() {
+ if err := run(); err != nil {
+ fmt.Fprintln(os.Stderr, "SAP helper protocol failed")
+ os.Exit(1)
+ }
+}
diff --git a/tools/sap/upstream.json b/tools/sap/upstream.json
new file mode 100644
index 0000000..5d76e08
--- /dev/null
+++ b/tools/sap/upstream.json
@@ -0,0 +1,6 @@
+{
+ "module": "github.com/majd/ipatool/v2",
+ "version": "v2.5.0",
+ "commit": "d5d0b56faf64e3fdef885d49e7928b390aadb6c7",
+ "sum": "h1:U8jBPKFNMKeBCBUOSo7dj1p/Ec+dTo4iR+N+/db8NjQ="
+}
diff --git a/tools/sap/verify_artifacts.py b/tools/sap/verify_artifacts.py
new file mode 100644
index 0000000..0692a0a
--- /dev/null
+++ b/tools/sap/verify_artifacts.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+"""Reject a wheel missing a release SAP helper or its executable mode."""
+import hashlib
+from pathlib import Path
+import zipfile
+
+from build import ROOT, TARGETS
+
+prefix = 'mdast_cli/distribution_systems/appstore_client/'
+wheels = list((ROOT / 'dist').glob('*.whl'))
+assert len(wheels) == 1, 'Expected exactly one release wheel'
+with zipfile.ZipFile(wheels[0]) as wheel:
+ for target in TARGETS:
+ name = prefix + 'bin/mdast-sap-' + target + ('.exe' if target.startswith('windows') else '')
+ member = wheel.getinfo(name)
+ assert member.external_attr >> 16 & 0o111, 'SAP helper is not executable'
+ assert hashlib.sha256(wheel.read(name)).digest() == hashlib.sha256((ROOT / name).read_bytes()).digest()
+ assert wheel.read(prefix + 'IPATOOL-LICENSE')
+print('Verified all six SAP helpers and upstream license in', wheels[0].name)