diff --git a/.changelog/next/fixed-issue-4200.md b/.changelog/next/fixed-issue-4200.md new file mode 100644 index 0000000000..cc3a892bc9 --- /dev/null +++ b/.changelog/next/fixed-issue-4200.md @@ -0,0 +1 @@ +- Windows installers for local audio and image runtimes now use the correct virtual environment interpreter. diff --git a/scripts/setup-image-video.sh b/scripts/setup-image-video.sh index d582647391..f5976e7ec0 100755 --- a/scripts/setup-image-video.sh +++ b/scripts/setup-image-video.sh @@ -33,6 +33,20 @@ is_macos() { [[ "$(uname -s)" == "Darwin" ]]; } # in-app installer through git-bash on Windows, so match all three. is_windows() { case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) return 0;; *) return 1;; esac; } +# Resolve a venv interpreter from the layout its creating Python used. Git Bash +# keeps its POSIX shell paths on Windows, but Python still creates Scripts/. +venv_python() { + if [[ -x "$1/bin/python3" ]]; then + printf '%s\n' "$1/bin/python3" + else + printf '%s\n' "$1/Scripts/python.exe" + fi +} + +# True if a venv at $1 already has an interpreter under either layout — +# shared with venv_python so the two probes can't drift apart. +venv_exists() { [[ -x "$1/bin/python3" || -x "$1/Scripts/python.exe" ]]; } + # Check out a pin (a commit SHA, tag, or branch name) in an already-fetched # clone. A bare `git checkout ` lands on the *local* branch created at # clone time, which `git fetch origin` never advances — so re-running with a @@ -458,12 +472,11 @@ if [[ "$INSTALL_MINIMAX_H3_CUDA" == "1" ]]; then # A Windows venv puts the interpreter under Scripts/, a POSIX one under bin/. # Probe for either rather than branching on `uname`, so an MSYS/Cygwin bash on # Windows (which is what the in-app installer runs) resolves correctly. - MINIMAX_H3_CUDA_PY="${MINIMAX_H3_CUDA_VENV}/bin/python3" - if [[ ! -x "$MINIMAX_H3_CUDA_PY" && ! -x "${MINIMAX_H3_CUDA_VENV}/Scripts/python.exe" ]]; then + if ! venv_exists "$MINIMAX_H3_CUDA_VENV"; then echo "📦 Creating MiniMax H3 CUDA venv..." "$PYTHON_BIN" -m venv "$MINIMAX_H3_CUDA_VENV" fi - [[ -x "$MINIMAX_H3_CUDA_PY" ]] || MINIMAX_H3_CUDA_PY="${MINIMAX_H3_CUDA_VENV}/Scripts/python.exe" + MINIMAX_H3_CUDA_PY="$(venv_python "$MINIMAX_H3_CUDA_VENV")" "$MINIMAX_H3_CUDA_PY" -m pip install --disable-pip-version-check --upgrade pip wheel setuptools # torch comes from PyTorch's own CUDA index on Windows — the default PyPI @@ -626,19 +639,18 @@ if [[ "$INSTALL_AUDIOLDM2" == "1" ]]; then # diffusion text-to-audio model shipped in HuggingFace `diffusers` (a pip # package — no clone needed), so this is just a sibling torch venv. The sidecar # `scripts/generate_audioldm2.py` imports `AudioLDM2Pipeline` from diffusers; - # server/lib/pythonSetup.js (resolveAudioldm2Python) looks for python3 here. + # server/lib/pythonSetup.js (resolveAudioldm2Python) discovers this venv. # Runs on Apple-Silicon MPS, CUDA, or CPU — not gated to macOS like MusicGen. - # This is a bash installer, so the venv layout is the POSIX bin/python3 (same - # as the MusicGen block); the Windows Scripts/python.exe path is resolved on - # the JS side by pythonSetup's AUDIOLDM2_VENV_CANDIDATES. + # Git Bash on Windows creates Scripts/python.exe even though this is a bash + # installer; venv_python keeps the installer aligned with the JS resolver. AUDIOLDM2_VENV="${HOME}/.portos/venv-audioldm2" - AUDIOLDM2_PY="$AUDIOLDM2_VENV/bin/python3" mkdir -p "${HOME}/.portos" - if [[ ! -x "$AUDIOLDM2_PY" ]]; then + if ! venv_exists "$AUDIOLDM2_VENV"; then echo "📦 Creating AudioLDM2 venv at ${AUDIOLDM2_VENV}..." "$PYTHON_BIN" -m venv "$AUDIOLDM2_VENV" fi + AUDIOLDM2_PY="$(venv_python "$AUDIOLDM2_VENV")" echo "📦 Installing AudioLDM2 (diffusers) packages into ${AUDIOLDM2_VENV}..." "$AUDIOLDM2_PY" -m pip install --upgrade pip wheel setuptools >/dev/null # torch runs the model; diffusers provides AudioLDM2Pipeline; transformers + @@ -672,19 +684,19 @@ if [[ "$INSTALL_ACESTEP" == "1" ]]; then # vocals. It installs as the `acestep` pip package (from git — no clone to # import from), so this is a sibling torch venv. The sidecar # `scripts/generate_acestep.py` imports `ACEStepPipeline` from acestep; - # server/lib/pythonSetup.js (resolveAcestepPython) looks for python3 here. - # Runs on Apple-Silicon MPS, CUDA, or CPU. The Windows Scripts/python.exe path - # is resolved on the JS side by pythonSetup's ACESTEP_VENV_CANDIDATES. + # server/lib/pythonSetup.js (resolveAcestepPython) discovers this venv. + # Runs on Apple-Silicon MPS, CUDA, or CPU. Git Bash on Windows creates + # Scripts/python.exe, so resolve the venv path after it is created. # ACE-Step auto-downloads its 3.5B checkpoints to ~/.cache/ace-step on first # run, so this only builds the venv — no weights are fetched here. ACESTEP_VENV="${HOME}/.portos/venv-acestep" - ACESTEP_PY="$ACESTEP_VENV/bin/python3" mkdir -p "${HOME}/.portos" - if [[ ! -x "$ACESTEP_PY" ]]; then + if ! venv_exists "$ACESTEP_VENV"; then echo "📦 Creating ACE-Step venv at ${ACESTEP_VENV}..." "$PYTHON_BIN" -m venv "$ACESTEP_VENV" fi + ACESTEP_PY="$(venv_python "$ACESTEP_VENV")" echo "📦 Installing ACE-Step into ${ACESTEP_VENV} (this pulls torch + the acestep package)..." "$ACESTEP_PY" -m pip install --upgrade pip wheel setuptools >/dev/null # The acestep package declares its own deps (torch, diffusers, transformers, @@ -720,9 +732,10 @@ if [[ "$INSTALL_MINIMAX_MUSIC3" == "1" ]]; then echo "♻️ Existing MiniMax Music 3 venv was built from a conda base (torch can't load there) — rebuilding from ${PYTHON_BIN}." rm -rf "$MINIMAX_MUSIC3_VENV" fi - MINIMAX_MUSIC3_PY="$MINIMAX_MUSIC3_VENV/bin/python3" - [[ -x "$MINIMAX_MUSIC3_PY" || -x "$MINIMAX_MUSIC3_VENV/Scripts/python.exe" ]] || "$PYTHON_BIN" -m venv "$MINIMAX_MUSIC3_VENV" - [[ -x "$MINIMAX_MUSIC3_PY" ]] || MINIMAX_MUSIC3_PY="$MINIMAX_MUSIC3_VENV/Scripts/python.exe" + if ! venv_exists "$MINIMAX_MUSIC3_VENV"; then + "$PYTHON_BIN" -m venv "$MINIMAX_MUSIC3_VENV" + fi + MINIMAX_MUSIC3_PY="$(venv_python "$MINIMAX_MUSIC3_VENV")" "$MINIMAX_MUSIC3_PY" -m pip install --upgrade pip wheel setuptools # torch comes from PyTorch's own CUDA index on Windows — the default PyPI # Windows wheel is CPU-only, which fails the cuda assert below. Linux's PyPI @@ -772,20 +785,13 @@ if [[ "$INSTALL_MUSCRIPTOR" == "1" ]]; then # so this only builds the venv — no weights are fetched here. Weights are # CC BY-NC 4.0 (non-commercial). MUSCRIPTOR_VENV="${HOME}/.portos/venv-muscriptor" - MUSCRIPTOR_PY="$MUSCRIPTOR_VENV/bin/python3" mkdir -p "${HOME}/.portos" - if [[ ! -x "$MUSCRIPTOR_PY" && ! -x "$MUSCRIPTOR_VENV/Scripts/python.exe" ]]; then + if ! venv_exists "$MUSCRIPTOR_VENV"; then echo "📦 Creating MuScriptor venv at ${MUSCRIPTOR_VENV}..." "$PYTHON_BIN" -m venv "$MUSCRIPTOR_VENV" fi - # A venv created by Windows Python (this script run under git-bash) places the - # interpreter at Scripts/python.exe instead of bin/python3 — pick whichever - # layout the venv actually has so the pip steps below work on both. The JS - # side (pythonSetup's MUSCRIPTOR_VENV_CANDIDATES) already probes both. - if [[ ! -x "$MUSCRIPTOR_PY" ]]; then - MUSCRIPTOR_PY="$MUSCRIPTOR_VENV/Scripts/python.exe" - fi + MUSCRIPTOR_PY="$(venv_python "$MUSCRIPTOR_VENV")" echo "📦 Installing MuScriptor into ${MUSCRIPTOR_VENV} (pulls torch + audio deps)..." "$MUSCRIPTOR_PY" -m pip install --upgrade pip wheel setuptools >/dev/null # muscriptor declares its own deps (torch, soundfile, etc.), so installing the @@ -810,14 +816,14 @@ if [[ "$INSTALL_FLUX2" == "1" ]]; then # FLUX.2-klein needs torch>=2.5 + diffusers-from-git + sdnq + optimum-quanto. # Mixing those into the mflux pip --user pile (mflux pulls older torch) is # fragile, so we use a sibling venv. server/lib/pythonSetup.js looks for - # python3 here when the active model has runner=='flux2'. + # this venv's interpreter when the active model has runner=='flux2'. FLUX2_VENV="${HOME}/.portos/venv-flux2" - FLUX2_PY="$FLUX2_VENV/bin/python3" - if [[ ! -x "$FLUX2_PY" ]]; then + if ! venv_exists "$FLUX2_VENV"; then echo "📦 Creating FLUX.2 venv at ${FLUX2_VENV}..." mkdir -p "${HOME}/.portos" "$PYTHON_BIN" -m venv "$FLUX2_VENV" fi + FLUX2_PY="$(venv_python "$FLUX2_VENV")" # Skip the (slow, network-heavy) pip path when Flux2KleinPipeline already # imports — diffusers-from-git is a git clone every run otherwise. Use @@ -889,16 +895,16 @@ if [[ "$INSTALL_MUSICGEN" == "1" ]] && is_macos; then echo " MusicGen: ${HOME}/.portos/venv-musicgen/bin/python3 (separate venv, MLX runtime @ ${HOME}/.portos/mlx-examples/musicgen)" fi if [[ "$INSTALL_AUDIOLDM2" == "1" ]]; then - echo " AudioLDM2: ${HOME}/.portos/venv-audioldm2/bin/python3 (separate venv, diffusers — long-form audio)" + echo " AudioLDM2: ${AUDIOLDM2_PY} (separate venv, diffusers — long-form audio)" fi if [[ "$INSTALL_ACESTEP" == "1" ]]; then - echo " ACE-Step: ${HOME}/.portos/venv-acestep/bin/python3 (separate venv, acestep — full song + vocals)" + echo " ACE-Step: ${ACESTEP_PY} (separate venv, acestep — full song + vocals)" fi if [[ "$INSTALL_MUSCRIPTOR" == "1" ]]; then - echo " MuScriptor: ${HOME}/.portos/venv-muscriptor/bin/python3 (separate venv, muscriptor — audio → MIDI)" + echo " MuScriptor: ${MUSCRIPTOR_PY} (separate venv, muscriptor — audio → MIDI)" fi if [[ "$INSTALL_FLUX2" == "1" ]]; then - echo " FLUX.2: ${HOME}/.portos/venv-flux2/bin/python3 (separate venv)" + echo " FLUX.2: ${FLUX2_PY} (separate venv)" echo " Z-Image: reuses the FLUX.2 venv (Apache 2.0, no HF login needed)" echo " ERNIE: reuses the FLUX.2 venv (Apache 2.0, no HF login needed)" echo "" diff --git a/scripts/setup-image-video.test.js b/scripts/setup-image-video.test.js new file mode 100644 index 0000000000..8bb16dd291 --- /dev/null +++ b/scripts/setup-image-video.test.js @@ -0,0 +1,103 @@ +import { execFileSync } from 'child_process'; +import { chmodSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { dirname, join } from 'path'; +import { fileURLToPath } from 'url'; +import { afterEach, describe, expect, it } from 'vitest'; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); +const SETUP_SCRIPT = join(REPO_ROOT, 'scripts', 'setup-image-video.sh'); +const source = readFileSync(SETUP_SCRIPT, 'utf8'); +const helper = source.match(/^venv_python\(\) \{\n(?:.*\n)*?^\}\n/m)?.[0]; +const existsHelper = source.match(/^venv_exists\(\) \{.*\}\n/m)?.[0]; +const tempVenvs = []; + +afterEach(() => { + while (tempVenvs.length) rmSync(tempVenvs.pop(), { recursive: true, force: true }); +}); + +function createVenv(layout) { + const venv = mkdtempSync(join(tmpdir(), 'portos-venv-python-')); + const interpreter = join(venv, layout === 'posix' ? 'bin/python3' : 'Scripts/python.exe'); + mkdirSync(dirname(interpreter), { recursive: true }); + writeFileSync(interpreter, ''); + chmodSync(interpreter, 0o755); + tempVenvs.push(venv); + return { venv, interpreter }; +} + +function resolveVenvPython(venv) { + return execFileSync('bash', ['-c', `${helper}\nvenv_python "$1"`, 'bash', venv], { + encoding: 'utf8', + }).trim(); +} + +function venvExists(venv) { + const result = execFileSync( + 'bash', + ['-c', `${existsHelper}\nvenv_exists "$1" && echo yes || echo no`, 'bash', venv], + { encoding: 'utf8' } + ).trim(); + return result === 'yes'; +} + +describe('setup-image-video venv layout handling (issue #4200)', () => { + it('defines the shared venv interpreter resolver', () => { + expect(helper).toBeTruthy(); + }); + + it('defines the shared venv-exists predicate', () => { + expect(existsHelper).toBeTruthy(); + }); + + // Windows CI runs Node directly; these bash-execution checks are POSIX-only, + // while the portable structural guard below still covers every platform. + it.skipIf(process.platform === 'win32')('passes Bash syntax validation', () => { + expect(() => execFileSync('bash', ['-n', SETUP_SCRIPT])).not.toThrow(); + }); + + it.skipIf(process.platform === 'win32')('resolves a POSIX venv interpreter', () => { + const { venv, interpreter } = createVenv('posix'); + expect(resolveVenvPython(venv)).toBe(interpreter); + }); + + it.skipIf(process.platform === 'win32')('resolves a Windows venv interpreter', () => { + const { venv, interpreter } = createVenv('windows'); + expect(resolveVenvPython(venv)).toBe(interpreter); + }); + + it.skipIf(process.platform === 'win32')('venv_exists is true for either layout, false for neither', () => { + expect(venvExists(createVenv('posix').venv)).toBe(true); + expect(venvExists(createVenv('windows').venv)).toBe(true); + const empty = mkdtempSync(join(tmpdir(), 'portos-venv-python-')); + tempVenvs.push(empty); + expect(venvExists(empty)).toBe(false); + }); + + it.each([ + ['MiniMax H3 CUDA', 'MINIMAX_H3_CUDA_VENV', 'MINIMAX_H3_CUDA_PY'], + ['AudioLDM2', 'AUDIOLDM2_VENV', 'AUDIOLDM2_PY'], + ['ACE-Step', 'ACESTEP_VENV', 'ACESTEP_PY'], + ['MiniMax Music 3', 'MINIMAX_MUSIC3_VENV', 'MINIMAX_MUSIC3_PY'], + ['MuScriptor', 'MUSCRIPTOR_VENV', 'MUSCRIPTOR_PY'], + ['FLUX.2', 'FLUX2_VENV', 'FLUX2_PY'], + ])('%s reuses an existing Windows venv and resolves it with the helper', (_name, venv, python) => { + expect(source).toContain(`if ! venv_exists "$${venv}"; then`); + expect(source).toContain(`${python}="$(venv_python "$${venv}")"`); + }); + + it('has no call site that hardcodes a venv interpreter path outside the shared helpers', () => { + // A literal "$SOMETHING_VENV/bin/python3" assignment outside the helper + // definitions means a call site bypassed venv_python() and reintroduced + // the POSIX-only bug this file guards against — EXCEPT for venvs that are + // gated behind is_macos and can never be created by Windows Python, which + // legitimately hardcode the POSIX path: mflux (uv-managed, Apple Silicon + // only) and MusicGen (MLX runtime, is_macos-gated). + const macosOnlyVenvExemptions = ['MFLUX_VENV', 'MUSICGEN_VENV']; + const body = macosOnlyVenvExemptions + .reduce((text, name) => text.replaceAll(`\${${name}}`, '').replaceAll(`$${name}`, ''), source) + .replace(helper, '') + .replace(existsHelper, ''); + expect(body).not.toMatch(/\$\{?\w+_VENV\}?\/bin\/python3/); + }); +});