Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 28 additions & 1 deletion src/index_503/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from pathlib import Path
from shutil import rmtree
from tempfile import mkdtemp
from typing import Optional

from natsort import natsorted
from yarl import URL
Expand Down Expand Up @@ -52,6 +53,7 @@ def make_index(self) -> Path:
"""Generate a simple repository of Python wheels."""
target_path = self.target_path
old_index = target_path.readlink() if target_path.exists() else None
self._cleanup_orphan_build_dirs(old_index)
temp_dir = mkdtemp(None, None, str(self.target_path.parent))
try:
self.cache.load()
Expand All @@ -68,9 +70,34 @@ def make_index(self) -> Path:
return target_path
except Exception:
_LOGGER.exception("Error generating index")
rmtree(temp_dir)
# Use ignore_errors so a partial _atomic_replace_old_index (which
# already renamed temp_dir) doesn't mask the original exception.
# Any leftover dirs are cleaned up on the next run.
rmtree(temp_dir, ignore_errors=True)
raise

def _cleanup_orphan_build_dirs(self, current_target: Optional[Path]) -> None:
"""Remove build directories left behind by crashed previous runs.

The atomic-replace flow renames temp dirs to ``<target>-tmpXXXX`` and
briefly creates ``<target>-tmpXXXX-build`` symlinks. A crash between
steps leaves these in place forever; clean them up at startup while we
hold the lock.
"""
parent = self.target_path.parent
prefix = f"{self.target_path.name}-tmp"
keep_name = current_target.name if current_target is not None else None
for entry in parent.iterdir():
if not entry.name.startswith(prefix):
continue
if entry.name == keep_name:
continue
_LOGGER.warning("Removing orphan build artifact: %s", entry)
if entry.is_symlink() or not entry.is_dir():
entry.unlink()
else:
rmtree(entry, ignore_errors=True)

def _atomic_replace_old_index(self, temp_dir_path: Path, target_path: Path) -> None:
"""Atomically replace the old index with the new one."""
final_name = target_path.parent / (target_path.name + "-" + temp_dir_path.name)
Expand Down
56 changes: 55 additions & 1 deletion tests/test_index.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
import json
import os
from pathlib import Path
from shutil import copyfile
from unittest.mock import ANY, patch

import pytest

from index_503.index import make_index
from index_503.index import IndexMaker, make_index
from index_503.wheel_file import WHEEL_FILE_VERSION

from . import FIXTURES
Expand Down Expand Up @@ -60,6 +61,59 @@ def test_make_index_fails(tmp_path: Path) -> None:
assert parent_dir_contents == {"musllinux", ".musllinux.index_503.lock"}


def test_make_index_fails_partial_replace(tmp_path: Path) -> None:
"""If _atomic_replace_old_index renames the temp dir then crashes, the
original exception must propagate (no FileNotFoundError masking it) and
the orphan must be cleaned up on the next run.
"""
origin_path, target_path = setup_wheels(tmp_path, TEST_WHEELS)
parent_dir = origin_path.parent

def rename_then_fail(self: IndexMaker, temp_dir_path: Path, t: Path) -> None:
# Mimic a partial run: rename succeeds, then we blow up.
final_name = t.parent / (t.name + "-" + temp_dir_path.name)
os.rename(temp_dir_path, final_name)
raise ValueError("boom")

with pytest.raises(ValueError, match="boom"), patch.object(
IndexMaker, "_atomic_replace_old_index", rename_then_fail
):
make_index(origin_path)

# Orphan dir survives the failed run...
orphans = [
p for p in parent_dir.iterdir() if p.name.startswith("musllinux-index-tmp")
]
assert len(orphans) == 1

# ...but the next successful run cleans it up.
assert make_index(origin_path) == target_path
leftover = [
p for p in parent_dir.iterdir() if p.name.startswith("musllinux-index-tmp")
]
# Exactly one survives — the live target the symlink points at.
assert len(leftover) == 1
assert leftover[0].name == target_path.readlink().name


def test_make_index_cleans_orphan_build_symlink(tmp_path: Path) -> None:
"""An orphan -build symlink from a crash between symlink and replace gets removed."""
origin_path, target_path = setup_wheels(tmp_path, TEST_WHEELS)
parent_dir = origin_path.parent

# Seed orphans that look exactly like crash leftovers.
orphan_dir = parent_dir / "musllinux-index-tmpDEADBEEF"
orphan_dir.mkdir()
(orphan_dir / "junk").write_text("x")
orphan_symlink = parent_dir / "musllinux-index-tmpFEEDFACE-build"
orphan_symlink.symlink_to(orphan_dir)

assert make_index(origin_path) == target_path

assert not orphan_dir.exists()
assert not orphan_symlink.exists()


def test_make_index_end_to_end(tmp_path: Path) -> None:
"""Test make_index() end to end."""
origin_path, origin_path_index = setup_wheels(tmp_path, TEST_WHEELS)
Expand Down