From a0169c32b572c88ab98e85c48a2c767b28f727c9 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Sat, 16 Nov 2024 19:19:21 -0800 Subject: [PATCH 01/20] fix the test case for cpu --- test-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-image.sh b/test-image.sh index 4182695..008f568 100644 --- a/test-image.sh +++ b/test-image.sh @@ -22,7 +22,7 @@ while ! curl -sf http://localhost:8000/v1/models; do done echo "Container is ready." -curl -v http://localhost:8000/v1/completions \ +curl -v --fail-with-body --show-error http://localhost:8000/v1/completions \ -H "Content-Type: application/json" \ -d '{ "model": "facebook/opt-125m", From 980ea60308de963fb082db7404d8c37ec196657b Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Sat, 16 Nov 2024 19:22:20 -0800 Subject: [PATCH 02/20] fix the CPU image with missing oneDNN --- Dockerfile.cpu | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Dockerfile.cpu b/Dockerfile.cpu index fb66390..cf77779 100644 --- a/Dockerfile.cpu +++ b/Dockerfile.cpu @@ -35,6 +35,19 @@ RUN --mount=type=cache,target=/root/.cache/pip \ pip install --upgrade pip && \ pip install -r requirements-build.txt +# install oneDNN +RUN git clone -b rls-v3.5 https://github.com/oneapi-src/oneDNN.git + +RUN --mount=type=cache,target=/root/.cache/ccache \ + cmake -B ./oneDNN/build -S ./oneDNN -G Ninja -DONEDNN_LIBRARY_TYPE=STATIC \ + -DONEDNN_BUILD_DOC=OFF \ + -DONEDNN_BUILD_EXAMPLES=OFF \ + -DONEDNN_BUILD_TESTS=OFF \ + -DONEDNN_BUILD_GRAPH=OFF \ + -DONEDNN_ENABLE_WORKLOAD=INFERENCE \ + -DONEDNN_ENABLE_PRIMITIVE=MATMUL && \ + cmake --build ./oneDNN/build --target install --config Release + FROM cpu-test-1 AS build WORKDIR /workspace/vllm From e7340938f69f9647eb1f3b10307eb74bfcc3c943 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Sat, 16 Nov 2024 20:09:44 -0800 Subject: [PATCH 03/20] don't exit on error in test-image.sh --- test-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-image.sh b/test-image.sh index 008f568..5f6e4c3 100644 --- a/test-image.sh +++ b/test-image.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -xe +set -x IMAGE_TAG="${IMAGE_TAG:-vllm/vllm-openai:latest}" MODEL_NAME="${MODEL_NAME:-facebook/opt-125m}" From cb73787f31165d1c3d24ddfc6d400e3a443ff870 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Sat, 16 Nov 2024 20:22:51 -0800 Subject: [PATCH 04/20] fix end of lines --- Dockerfile.cpu | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile.cpu b/Dockerfile.cpu index cf77779..5135250 100644 --- a/Dockerfile.cpu +++ b/Dockerfile.cpu @@ -39,12 +39,12 @@ RUN --mount=type=cache,target=/root/.cache/pip \ RUN git clone -b rls-v3.5 https://github.com/oneapi-src/oneDNN.git RUN --mount=type=cache,target=/root/.cache/ccache \ - cmake -B ./oneDNN/build -S ./oneDNN -G Ninja -DONEDNN_LIBRARY_TYPE=STATIC \ - -DONEDNN_BUILD_DOC=OFF \ - -DONEDNN_BUILD_EXAMPLES=OFF \ - -DONEDNN_BUILD_TESTS=OFF \ - -DONEDNN_BUILD_GRAPH=OFF \ - -DONEDNN_ENABLE_WORKLOAD=INFERENCE \ + cmake -B ./oneDNN/build -S ./oneDNN -G Ninja -DONEDNN_LIBRARY_TYPE=STATIC \ + -DONEDNN_BUILD_DOC=OFF \ + -DONEDNN_BUILD_EXAMPLES=OFF \ + -DONEDNN_BUILD_TESTS=OFF \ + -DONEDNN_BUILD_GRAPH=OFF \ + -DONEDNN_ENABLE_WORKLOAD=INFERENCE \ -DONEDNN_ENABLE_PRIMITIVE=MATMUL && \ cmake --build ./oneDNN/build --target install --config Release From 185f37b6d8b9bdc499b645baebf9cd0669d90f95 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Sat, 16 Nov 2024 20:41:45 -0800 Subject: [PATCH 05/20] add more flags to vllm for easy debugging --- test-image.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test-image.sh b/test-image.sh index 5f6e4c3..79f6fcd 100644 --- a/test-image.sh +++ b/test-image.sh @@ -5,8 +5,12 @@ set -x IMAGE_TAG="${IMAGE_TAG:-vllm/vllm-openai:latest}" MODEL_NAME="${MODEL_NAME:-facebook/opt-125m}" -docker run --rm -d --name vllm -p 8000:8000 ${IMAGE_TAG} \ - --model ${MODEL_NAME} ${ARGS} +docker run --rm -d --name vllm -p 8000:8000 \ + -e VLLM_WORKER_MULTIPROC_METHOD=spawn \ + ${IMAGE_TAG} \ + --model ${MODEL_NAME} \ + --disable-frontend-multiprocessing \ + ${ARGS} # Wait for up to 120 seconds for the Docker container to be ready echo "Waiting for the container to be ready..." From fdc9ec18d2539465ca095ffc38eb8027d6c6e854 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Sat, 16 Nov 2024 20:42:02 -0800 Subject: [PATCH 06/20] oneDNN use specific tag --- Dockerfile.cpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.cpu b/Dockerfile.cpu index 5135250..ffef60f 100644 --- a/Dockerfile.cpu +++ b/Dockerfile.cpu @@ -36,7 +36,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ pip install -r requirements-build.txt # install oneDNN -RUN git clone -b rls-v3.5 https://github.com/oneapi-src/oneDNN.git +RUN git clone --depth 1 -b v3.6.1 https://github.com/oneapi-src/oneDNN.git RUN --mount=type=cache,target=/root/.cache/ccache \ cmake -B ./oneDNN/build -S ./oneDNN -G Ninja -DONEDNN_LIBRARY_TYPE=STATIC \ From aca6a5d560af5b04c75df01736dde1032eafec37 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Sat, 16 Nov 2024 20:50:30 -0800 Subject: [PATCH 07/20] enable JIT for oneDNN --- Dockerfile.cpu | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/Dockerfile.cpu b/Dockerfile.cpu index ffef60f..b4fa5b1 100644 --- a/Dockerfile.cpu +++ b/Dockerfile.cpu @@ -29,6 +29,9 @@ WORKDIR /workspace RUN git clone https://github.com/vllm-project/vllm.git && cd vllm && git checkout v${VERSION} WORKDIR /workspace/vllm +# Fix needed for oneDNN, see: https://github.com/vllm-project/vllm/pull/9344#issuecomment-2480921672 +RUN sed -i 's/ONEDNN_ENABLE_JIT_PROFILING "OFF"/ONEDNN_ENABLE_JIT_PROFILING "ON"/' cmake/cpu_extension.cmake + ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} RUN --mount=type=cache,target=/root/.cache/pip \ @@ -36,17 +39,17 @@ RUN --mount=type=cache,target=/root/.cache/pip \ pip install -r requirements-build.txt # install oneDNN -RUN git clone --depth 1 -b v3.6.1 https://github.com/oneapi-src/oneDNN.git - -RUN --mount=type=cache,target=/root/.cache/ccache \ - cmake -B ./oneDNN/build -S ./oneDNN -G Ninja -DONEDNN_LIBRARY_TYPE=STATIC \ - -DONEDNN_BUILD_DOC=OFF \ - -DONEDNN_BUILD_EXAMPLES=OFF \ - -DONEDNN_BUILD_TESTS=OFF \ - -DONEDNN_BUILD_GRAPH=OFF \ - -DONEDNN_ENABLE_WORKLOAD=INFERENCE \ - -DONEDNN_ENABLE_PRIMITIVE=MATMUL && \ - cmake --build ./oneDNN/build --target install --config Release +# RUN git clone --depth 1 -b v3.6.1 https://github.com/oneapi-src/oneDNN.git +# +# RUN --mount=type=cache,target=/root/.cache/ccache \ +# cmake -B ./oneDNN/build -S ./oneDNN -G Ninja -DONEDNN_LIBRARY_TYPE=STATIC \ +# -DONEDNN_BUILD_DOC=OFF \ +# -DONEDNN_BUILD_EXAMPLES=OFF \ +# -DONEDNN_BUILD_TESTS=OFF \ +# -DONEDNN_BUILD_GRAPH=OFF \ +# -DONEDNN_ENABLE_WORKLOAD=INFERENCE \ +# -DONEDNN_ENABLE_PRIMITIVE=MATMUL && \ +# cmake --build ./oneDNN/build --target install --config Release FROM cpu-test-1 AS build WORKDIR /workspace/vllm @@ -58,6 +61,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ ARG VLLM_CPU_DISABLE_AVX512 ENV VLLM_CPU_DISABLE_AVX512=${VLLM_CPU_DISABLE_AVX512} + RUN --mount=type=cache,target=/root/.cache/pip \ --mount=type=cache,target=/root/.cache/ccache \ VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel && \ From f69be811fa8fb848a07fd341fc35c2bcd20904bb Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Sat, 16 Nov 2024 22:18:02 -0800 Subject: [PATCH 08/20] also set ONEDNN_ENABLE_ITT_TASKS to on --- Dockerfile.cpu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile.cpu b/Dockerfile.cpu index b4fa5b1..b46d7d2 100644 --- a/Dockerfile.cpu +++ b/Dockerfile.cpu @@ -31,6 +31,8 @@ WORKDIR /workspace/vllm # Fix needed for oneDNN, see: https://github.com/vllm-project/vllm/pull/9344#issuecomment-2480921672 RUN sed -i 's/ONEDNN_ENABLE_JIT_PROFILING "OFF"/ONEDNN_ENABLE_JIT_PROFILING "ON"/' cmake/cpu_extension.cmake +RUN sed -i 's/ONEDNN_ENABLE_ITT_TASKS "OFF"/ONEDNN_ENABLE_ITT_TASKS "ON"/' cmake/cpu_extension.cmake +RUN cat cmake/cpu_extension.cmake ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} From 8db0fc0b78f8510f0c7296e1c233be5edcc255f3 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Sun, 17 Nov 2024 22:28:00 -0800 Subject: [PATCH 09/20] revert to original flags --- Dockerfile.cpu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile.cpu b/Dockerfile.cpu index b46d7d2..8c37b1c 100644 --- a/Dockerfile.cpu +++ b/Dockerfile.cpu @@ -32,6 +32,8 @@ WORKDIR /workspace/vllm # Fix needed for oneDNN, see: https://github.com/vllm-project/vllm/pull/9344#issuecomment-2480921672 RUN sed -i 's/ONEDNN_ENABLE_JIT_PROFILING "OFF"/ONEDNN_ENABLE_JIT_PROFILING "ON"/' cmake/cpu_extension.cmake RUN sed -i 's/ONEDNN_ENABLE_ITT_TASKS "OFF"/ONEDNN_ENABLE_ITT_TASKS "ON"/' cmake/cpu_extension.cmake +RUN sed -i 's/ONEDNN_ENABLE_MAX_CPU_ISA "OFF"/ONEDNN_ENABLE_MAX_CPU_ISA "ON"/' cmake/cpu_extension.cmake +RUN sed -i 's/ONEDNN_ENABLE_CPU_ISA_HINTS "OFF"/ONEDNN_ENABLE_CPU_ISA_HINTS "ON"/' cmake/cpu_extension.cmake RUN cat cmake/cpu_extension.cmake ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" From e03fd7f94db5c69c6b3cb4b41dd2f102aef7d171 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Tue, 19 Nov 2024 08:02:46 -0800 Subject: [PATCH 10/20] revert to previous release logic for oneDNN inclusion --- Dockerfile.cpu | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile.cpu b/Dockerfile.cpu index 8c37b1c..7863a75 100644 --- a/Dockerfile.cpu +++ b/Dockerfile.cpu @@ -30,10 +30,11 @@ RUN git clone https://github.com/vllm-project/vllm.git && cd vllm && git checkou WORKDIR /workspace/vllm # Fix needed for oneDNN, see: https://github.com/vllm-project/vllm/pull/9344#issuecomment-2480921672 -RUN sed -i 's/ONEDNN_ENABLE_JIT_PROFILING "OFF"/ONEDNN_ENABLE_JIT_PROFILING "ON"/' cmake/cpu_extension.cmake -RUN sed -i 's/ONEDNN_ENABLE_ITT_TASKS "OFF"/ONEDNN_ENABLE_ITT_TASKS "ON"/' cmake/cpu_extension.cmake -RUN sed -i 's/ONEDNN_ENABLE_MAX_CPU_ISA "OFF"/ONEDNN_ENABLE_MAX_CPU_ISA "ON"/' cmake/cpu_extension.cmake -RUN sed -i 's/ONEDNN_ENABLE_CPU_ISA_HINTS "OFF"/ONEDNN_ENABLE_CPU_ISA_HINTS "ON"/' cmake/cpu_extension.cmake +# RUN sed -i 's/ONEDNN_ENABLE_JIT_PROFILING "OFF"/ONEDNN_ENABLE_JIT_PROFILING "ON"/' cmake/cpu_extension.cmake +# RUN sed -i 's/ONEDNN_ENABLE_ITT_TASKS "OFF"/ONEDNN_ENABLE_ITT_TASKS "ON"/' cmake/cpu_extension.cmake +# RUN sed -i 's/ONEDNN_ENABLE_MAX_CPU_ISA "OFF"/ONEDNN_ENABLE_MAX_CPU_ISA "ON"/' cmake/cpu_extension.cmake +# RUN sed -i 's/ONEDNN_ENABLE_CPU_ISA_HINTS "OFF"/ONEDNN_ENABLE_CPU_ISA_HINTS "ON"/' cmake/cpu_extension.cmake +RUN sed -i 's/if (AVX512_FOUND AND NOT AVX512_DISABLED)/if (AVX512_FOUND OR AVX2_FOUND)/' cmake/cpu_extension.cmake RUN cat cmake/cpu_extension.cmake ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" From 1610ca3613c5d57751cd42ebf34ab6ed865529af Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Tue, 19 Nov 2024 13:25:04 -0800 Subject: [PATCH 11/20] set timeout to 60 mins --- .github/workflows/build-push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index a10e54d..2f4d2ab 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -18,6 +18,7 @@ env: jobs: cpu: runs-on: ubuntu-latest + timeout-minutes: 60 # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. permissions: contents: read From 401ee37ef3a02caef4dfbeee40a8646ab318b622 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Tue, 19 Nov 2024 14:34:56 -0800 Subject: [PATCH 12/20] increase timeout to 120 mins --- .github/workflows/build-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 2f4d2ab..ce8b7cc 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -18,7 +18,7 @@ env: jobs: cpu: runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 120 # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. permissions: contents: read From 76f532d40c0bbc7edc9f804fcbe2db77707705f2 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Tue, 19 Nov 2024 15:06:42 -0800 Subject: [PATCH 13/20] increase disk space --- .github/workflows/build-push.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index ce8b7cc..2cd0241 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -24,6 +24,17 @@ jobs: contents: read packages: write steps: + - name: Maximize build space + uses: easimon/maximize-build-space@master + with: + build-mount-path: /var/lib/docker/ + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' + remove-codeql: 'true' + remove-docker-images: 'true' + - name: Restart docker + run: sudo service docker restart - name: Checkout repository uses: actions/checkout@v4 - name: Set up Docker Buildx From 346639a1e70a260ec040b88f8403f38f858ce7fe Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Tue, 19 Nov 2024 16:49:31 -0800 Subject: [PATCH 14/20] remove disk otpimizer --- .github/workflows/build-push.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 2cd0241..ce8b7cc 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -24,17 +24,6 @@ jobs: contents: read packages: write steps: - - name: Maximize build space - uses: easimon/maximize-build-space@master - with: - build-mount-path: /var/lib/docker/ - remove-dotnet: 'true' - remove-android: 'true' - remove-haskell: 'true' - remove-codeql: 'true' - remove-docker-images: 'true' - - name: Restart docker - run: sudo service docker restart - name: Checkout repository uses: actions/checkout@v4 - name: Set up Docker Buildx From b62737c1b7abc62b0054797027069f3f67e7257f Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Tue, 19 Nov 2024 16:54:04 -0800 Subject: [PATCH 15/20] use optimized build process --- Dockerfile.cpu | 101 ++++++++++++++++++------------------------------- 1 file changed, 37 insertions(+), 64 deletions(-) diff --git a/Dockerfile.cpu b/Dockerfile.cpu index 7863a75..6a69dfa 100644 --- a/Dockerfile.cpu +++ b/Dockerfile.cpu @@ -1,80 +1,53 @@ -# This vLLM Dockerfile is used to construct image that can build and run vLLM on x86 CPU platform. +# Base image +FROM ubuntu:22.04 AS base ARG VERSION=0.6.4.post1 -FROM ubuntu:22.04 AS cpu-test-1 -ARG VERSION - ENV CCACHE_DIR=/root/.cache/ccache - ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache - -RUN --mount=type=cache,target=/var/cache/apt \ - apt-get update -y \ - && apt-get install -y curl ccache git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 libnuma-dev \ - && apt-get install -y ffmpeg libsm6 libxext6 libgl1 \ - && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 - -# https://intel.github.io/intel-extension-for-pytorch/cpu/latest/tutorials/performance_tuning/tuning_guide.html -# intel-openmp provides additional performance improvement vs. openmp -# tcmalloc provides better memory allocation efficiency, e.g, holding memory in caches to speed up access of commonly-used objects. -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install intel-openmp - ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/usr/local/lib/libiomp5.so" -RUN echo 'ulimit -c 0' >> ~/.bashrc -RUN pip install intel_extension_for_pytorch==2.5.0 +# Python environment setup +FROM base AS python-env +RUN python3 -m pip install --upgrade pip && \ + pip install virtualenv && \ + virtualenv /opt/vllm-venv +# Activate virtual environment in Docker image +ENV PATH="/opt/vllm-venv/bin:$PATH" + +# Code preparation +FROM python-env AS code-prep WORKDIR /workspace RUN git clone https://github.com/vllm-project/vllm.git && cd vllm && git checkout v${VERSION} -WORKDIR /workspace/vllm -# Fix needed for oneDNN, see: https://github.com/vllm-project/vllm/pull/9344#issuecomment-2480921672 -# RUN sed -i 's/ONEDNN_ENABLE_JIT_PROFILING "OFF"/ONEDNN_ENABLE_JIT_PROFILING "ON"/' cmake/cpu_extension.cmake -# RUN sed -i 's/ONEDNN_ENABLE_ITT_TASKS "OFF"/ONEDNN_ENABLE_ITT_TASKS "ON"/' cmake/cpu_extension.cmake -# RUN sed -i 's/ONEDNN_ENABLE_MAX_CPU_ISA "OFF"/ONEDNN_ENABLE_MAX_CPU_ISA "ON"/' cmake/cpu_extension.cmake -# RUN sed -i 's/ONEDNN_ENABLE_CPU_ISA_HINTS "OFF"/ONEDNN_ENABLE_CPU_ISA_HINTS "ON"/' cmake/cpu_extension.cmake -RUN sed -i 's/if (AVX512_FOUND AND NOT AVX512_DISABLED)/if (AVX512_FOUND OR AVX2_FOUND)/' cmake/cpu_extension.cmake -RUN cat cmake/cpu_extension.cmake +# Code building stage +FROM code-prep AS builder +# Install build dependencies and clean up in one layer to reduce image size +RUN apt-get update -y \ + && apt-get install -y curl ccache git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 libnuma-dev ffmpeg libsm6 libxext6 libgl1 \ + && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && echo 'ulimit -c 0' >> ~/.bashrc -ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" -ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --upgrade pip && \ - pip install -r requirements-build.txt - -# install oneDNN -# RUN git clone --depth 1 -b v3.6.1 https://github.com/oneapi-src/oneDNN.git -# -# RUN --mount=type=cache,target=/root/.cache/ccache \ -# cmake -B ./oneDNN/build -S ./oneDNN -G Ninja -DONEDNN_LIBRARY_TYPE=STATIC \ -# -DONEDNN_BUILD_DOC=OFF \ -# -DONEDNN_BUILD_EXAMPLES=OFF \ -# -DONEDNN_BUILD_TESTS=OFF \ -# -DONEDNN_BUILD_GRAPH=OFF \ -# -DONEDNN_ENABLE_WORKLOAD=INFERENCE \ -# -DONEDNN_ENABLE_PRIMITIVE=MATMUL && \ -# cmake --build ./oneDNN/build --target install --config Release - -FROM cpu-test-1 AS build WORKDIR /workspace/vllm +COPY cmake/cpu_extension.cmake . +RUN sed -i 's/if (AVX512_FOUND AND NOT AVX512_DISABLED)/if (AVX512_FOUND OR AVX2_FOUND)/' cmake/cpu_extension.cmake \ + && cat cmake/cpu_extension.cmake -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install -v -r requirements-cpu.txt - -# Support for building with non-AVX512 vLLM: docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" ... -ARG VLLM_CPU_DISABLE_AVX512 -ENV VLLM_CPU_DISABLE_AVX512=${VLLM_CPU_DISABLE_AVX512} - - -RUN --mount=type=cache,target=/root/.cache/pip \ - --mount=type=cache,target=/root/.cache/ccache \ - VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel && \ - pip install dist/*.whl && \ - rm -rf dist +ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" +ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} +RUN pip install -r requirements-build.txt \ + && pip install -v -r requirements-cpu.txt \ + && VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel \ + && pip install dist/*.whl -WORKDIR /workspace/ +# Final image +FROM base AS final +COPY --from=builder /opt/vllm-venv /opt/vllm-venv -RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks +# Activate virtual environment in Docker image +ENV PATH="/opt/vllm-venv/bin:$PATH" -ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"] +WORKDIR /workspace +ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"] \ No newline at end of file From 2e01f5dd7324cc6822134bbb25a0d74838a634ac Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Tue, 19 Nov 2024 17:37:35 -0800 Subject: [PATCH 16/20] remove incorrect copy --- Dockerfile.cpu | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile.cpu b/Dockerfile.cpu index 6a69dfa..f249c7d 100644 --- a/Dockerfile.cpu +++ b/Dockerfile.cpu @@ -31,7 +31,6 @@ RUN apt-get update -y \ && echo 'ulimit -c 0' >> ~/.bashrc WORKDIR /workspace/vllm -COPY cmake/cpu_extension.cmake . RUN sed -i 's/if (AVX512_FOUND AND NOT AVX512_DISABLED)/if (AVX512_FOUND OR AVX2_FOUND)/' cmake/cpu_extension.cmake \ && cat cmake/cpu_extension.cmake From bba412b23eff5463ba98198332a7201d2db0f18e Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Tue, 19 Nov 2024 17:41:32 -0800 Subject: [PATCH 17/20] split build dependencies --- Dockerfile.cpu | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Dockerfile.cpu b/Dockerfile.cpu index f249c7d..537e35b 100644 --- a/Dockerfile.cpu +++ b/Dockerfile.cpu @@ -5,6 +5,13 @@ ENV CCACHE_DIR=/root/.cache/ccache ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/usr/local/lib/libiomp5.so" +RUN apt-get update -y \ + && apt-get install -y curl git wget numactl python3 python3-pip libtcmalloc-minimal4 ffmpeg libsm6 libxext6 libgl1 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && echo 'ulimit -c 0' >> ~/.bashrc + + # Python environment setup FROM base AS python-env @@ -22,13 +29,13 @@ RUN git clone https://github.com/vllm-project/vllm.git && cd vllm && git checkou # Code building stage FROM code-prep AS builder + # Install build dependencies and clean up in one layer to reduce image size RUN apt-get update -y \ - && apt-get install -y curl ccache git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 libnuma-dev ffmpeg libsm6 libxext6 libgl1 \ + && apt-get install -y gcc-12 g++-12 libnuma-dev \ && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && echo 'ulimit -c 0' >> ~/.bashrc + && rm -rf /var/lib/apt/lists/* WORKDIR /workspace/vllm RUN sed -i 's/if (AVX512_FOUND AND NOT AVX512_DISABLED)/if (AVX512_FOUND OR AVX2_FOUND)/' cmake/cpu_extension.cmake \ From 9815ebceee6c2d795ab58f8e1ee471ec45a76158 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Tue, 19 Nov 2024 18:17:53 -0800 Subject: [PATCH 18/20] simplify --- Dockerfile.cpu | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/Dockerfile.cpu b/Dockerfile.cpu index 537e35b..e350a44 100644 --- a/Dockerfile.cpu +++ b/Dockerfile.cpu @@ -3,15 +3,14 @@ FROM ubuntu:22.04 AS base ARG VERSION=0.6.4.post1 ENV CCACHE_DIR=/root/.cache/ccache ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache -ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/usr/local/lib/libiomp5.so" - -RUN apt-get update -y \ - && apt-get install -y curl git wget numactl python3 python3-pip libtcmalloc-minimal4 ffmpeg libsm6 libxext6 libgl1 \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && echo 'ulimit -c 0' >> ~/.bashrc +RUN --mount=type=cache,target=/var/cache/apt \ + apt-get update -y \ + && apt-get install -y curl ccache git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 libnuma-dev \ + && apt-get install -y ffmpeg libsm6 libxext6 libgl1 \ + && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 +ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/usr/local/lib/libiomp5.so" # Python environment setup FROM base AS python-env @@ -30,12 +29,6 @@ RUN git clone https://github.com/vllm-project/vllm.git && cd vllm && git checkou # Code building stage FROM code-prep AS builder -# Install build dependencies and clean up in one layer to reduce image size -RUN apt-get update -y \ - && apt-get install -y gcc-12 g++-12 libnuma-dev \ - && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* WORKDIR /workspace/vllm RUN sed -i 's/if (AVX512_FOUND AND NOT AVX512_DISABLED)/if (AVX512_FOUND OR AVX2_FOUND)/' cmake/cpu_extension.cmake \ @@ -45,6 +38,7 @@ ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} RUN pip install -r requirements-build.txt \ && pip install -v -r requirements-cpu.txt \ + && pip install intel-openmp \ && VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel \ && pip install dist/*.whl From 86b57f6ff939d86d48f8815e858234b656684cf6 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Tue, 19 Nov 2024 19:24:56 -0800 Subject: [PATCH 19/20] don't remove container --- test-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-image.sh b/test-image.sh index 79f6fcd..28d49b3 100644 --- a/test-image.sh +++ b/test-image.sh @@ -5,7 +5,7 @@ set -x IMAGE_TAG="${IMAGE_TAG:-vllm/vllm-openai:latest}" MODEL_NAME="${MODEL_NAME:-facebook/opt-125m}" -docker run --rm -d --name vllm -p 8000:8000 \ +docker run -d --name vllm -p 8000:8000 \ -e VLLM_WORKER_MULTIPROC_METHOD=spawn \ ${IMAGE_TAG} \ --model ${MODEL_NAME} \ From 9262e3c57d14d243b901cdc917f41ef684e3dc49 Mon Sep 17 00:00:00 2001 From: Sam Stoelinga Date: Tue, 19 Nov 2024 21:49:43 -0800 Subject: [PATCH 20/20] fix path using venv --- Dockerfile.cpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.cpu b/Dockerfile.cpu index e350a44..00fc685 100644 --- a/Dockerfile.cpu +++ b/Dockerfile.cpu @@ -10,7 +10,7 @@ RUN --mount=type=cache,target=/var/cache/apt \ && apt-get install -y ffmpeg libsm6 libxext6 libgl1 \ && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 -ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/usr/local/lib/libiomp5.so" +ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/opt/vllm-venv/lib/libiomp5.so" # Python environment setup FROM base AS python-env