From 4736e3e8c5df90255bfe951d2c597142c54bc784 Mon Sep 17 00:00:00 2001 From: Lukas Bommes Date: Tue, 29 Oct 2024 20:37:55 +0000 Subject: [PATCH 1/3] add aarch64 build to CI --- .github/workflows/ci.yml | 164 +++++++++++++++++++++++++++++---------- Dockerfile | 6 +- 2 files changed, 126 insertions(+), 44 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 970dd15..9644592 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ on: workflow_dispatch: jobs: - build_docker: + build_docker_x86_64: name: Build Docker image runs-on: ubuntu-latest @@ -33,52 +33,92 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and export + - name: Build and export x86_64 uses: docker/build-push-action@v6 with: context: . - tags: mv-extractor:local - outputs: type=docker,dest=/tmp/image.tar - cache-from: type=registry,ref=lubo1994/mv-extractor:buildcache - cache-to: type=registry,ref=lubo1994/mv-extractor:buildcache,mode=max + platforms: linux/amd64 + build-args: "ARCH=x86_64" + tags: mv-extractor:local-x86_64 + outputs: type=docker,dest=/tmp/image-x86_64.tar + cache-from: type=registry,ref=lubo1994/mv-extractor:buildcache-x86_64 + cache-to: type=registry,ref=lubo1994/mv-extractor:buildcache-x86_64,mode=max - - name: Upload Docker image as artifact + - name: Upload x86_64 Docker image as artifact uses: actions/upload-artifact@v4 with: - name: mv-extractor-docker-image - path: /tmp/image.tar + name: mv-extractor-image-x86_64 + path: /tmp/image-x86_64.tar - test_docker: + build_docker_aarch64: + name: Build Docker image + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and export aarch64 + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/arm64 + build-args: "ARCH=aarch64" + tags: mv-extractor:local-aarch64 + outputs: type=docker,dest=/tmp/image-aarch64.tar + cache-from: type=registry,ref=lubo1994/mv-extractor:buildcache-aarch64 + cache-to: type=registry,ref=lubo1994/mv-extractor:buildcache-aarch64,mode=max + + - name: Upload aarch64 Docker image as artifact + uses: actions/upload-artifact@v4 + with: + name: mv-extractor-image-aarch64 + path: /tmp/image-aarch64.tar + + test_docker_x86_64: name: Run unit tests in Docker container (only for the Python version used in the Dockerfile command) runs-on: ubuntu-latest needs: - - build_docker + - build_docker_x86_64 steps: - name: Checkout uses: actions/checkout@v4 - - name: Download artifact containing Docker image + - name: Download x86_64 Docker image uses: actions/download-artifact@v4 with: - name: mv-extractor-docker-image + name: mv-extractor-image-x86_64 path: /tmp - name: Load Docker image run: | - docker load --input /tmp/image.tar + docker load --input /tmp/image-x86_64.tar - name: Run unit tests run: | docker run -v ${{ github.workspace }}:/home/video_cap \ - mv-extractor:local \ + mv-extractor:local-x86_64 \ python3.12 -m unittest discover -s tests -p "*tests.py" build_and_test_wheels: name: Build wheels for cp${{ matrix.python }}-${{ matrix.platform_id }} runs-on: ${{ matrix.os }} needs: - - build_docker + - build_docker_x86_64 + - build_docker_aarch64 strategy: # Ensure that a wheel builder finishes even if another fails fail-fast: false @@ -87,37 +127,42 @@ jobs: - os: ubuntu-latest python: 39 bitness: 64 - platform_id: manylinux_x86_64 - manylinux_image: mv-extractor:local + platform_id: manylinux + manylinux_x86_64_image: mv-extractor:local-x86_64 + manylinux_aarch64_image: mv-extractor:local-aarch64 numpy_min_version: "numpy==1.19.3" opencv_min_version: "opencv-python==4.4.0.46" - os: ubuntu-latest python: 310 bitness: 64 - platform_id: manylinux_x86_64 - manylinux_image: mv-extractor:local + platform_id: manylinux + manylinux_x86_64_image: mv-extractor:local-x86_64 + manylinux_aarch64_image: mv-extractor:local-aarch64 numpy_min_version: "numpy==1.21.2" opencv_min_version: "opencv-python==4.5.4.60" - os: ubuntu-latest python: 311 bitness: 64 - platform_id: manylinux_x86_64 - manylinux_image: mv-extractor:local + platform_id: manylinux + manylinux_x86_64_image: mv-extractor:local-x86_64 + manylinux_aarch64_image: mv-extractor:local-aarch64 numpy_min_version: "numpy==1.23.3" opencv_min_version: "opencv-python==4.7.0.72" - os: ubuntu-latest python: 312 bitness: 64 - platform_id: manylinux_x86_64 - manylinux_image: mv-extractor:local + platform_id: manylinux + manylinux_x86_64_image: mv-extractor:local-x86_64 + manylinux_aarch64_image: mv-extractor:local-aarch64 numpy_min_version: "numpy==1.26.0" opencv_min_version: "opencv-python==4.9.0.80" # opencv-python does not yet support Python 3.13 as of Oct 2024 # - os: ubuntu-latest # python: 313 # bitness: 64 - # platform_id: manylinux_x86_64 - # manylinux_image: mv-extractor:local + # platform_id: manylinux + # manylinux_x86_64_image: mv-extractor:local-x86_64 + # manylinux_aarch64_image: mv-extractor:local-aarch64 # numpy_min_version: "numpy==2.1.0" # opencv_min_version: "opencv-python==" @@ -125,25 +170,33 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Download artifact containing Docker image + - name: Download x86_64 Docker image + uses: actions/download-artifact@v4 + with: + name: mv-extractor-image-x86_64 + path: /tmp + + - name: Download aarch64 Docker image uses: actions/download-artifact@v4 with: - name: mv-extractor-docker-image + name: mv-extractor-image-aarch64 path: /tmp - - name: Load Docker image + - name: Load Docker images run: | - docker load --input /tmp/image.tar + docker load --input /tmp/image-x86_64.tar + docker load --input /tmp/image-aarch64.tar - name: Build and test wheels uses: pypa/cibuildwheel@v2.21.3 env: CIBW_PLATFORM: linux - CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }} + CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }}_* # Disable building PyPy wheels on all platforms CIBW_SKIP: pp* - CIBW_ARCHS: x86_64 - CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }} + CIBW_ARCHS: x86_64,aarch64 + CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_x86_64_image }} + CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.manylinux_aarch64_image }} #CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }} CIBW_BUILD_FRONTEND: build CIBW_TEST_COMMAND: | @@ -159,20 +212,47 @@ jobs: name: python-wheel-${{ matrix.python }} path: ./wheelhouse/*.whl - push_docker: - name: Push Docker image to DockerHub + push_docker_x86_64: + name: Push Docker images to DockerHub if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest needs: - - build_docker + - build_docker_x86_64 - test_docker - build_and_test_wheels steps: - - name: Download artifact containing Docker image + - name: Download x86_64 Docker image + uses: actions/download-artifact@v4 + with: + name: mv-extractor-image-x86_64 + path: /tmp + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Load and push x86_64 Docker image + run: | + docker load --input /tmp/image-x86_64.tar + docker tag mv-extractor:local-x86_64 lubo1994/mv-extractor:dev-x86_64 + docker push lubo1994/mv-extractor:dev-x86_64 + + push_docker_aarch64: + name: Push Docker images to DockerHub + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + needs: + - build_docker_aarch64 + - build_and_test_wheels + + steps: + - name: Download aarch64 Docker image uses: actions/download-artifact@v4 with: - name: mv-extractor-docker-image + name: mv-extractor-image-aarch64 path: /tmp - name: Login to DockerHub @@ -181,8 +261,8 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Load and push Docker image + - name: Load and push aarch64 Docker image run: | - docker load --input /tmp/image.tar - docker tag mv-extractor:local lubo1994/mv-extractor:dev - docker push lubo1994/mv-extractor:dev \ No newline at end of file + docker load --input /tmp/image-aarch64.tar + docker tag mv-extractor:local-aarch64 lubo1994/mv-extractor:dev-aarch64 + docker push lubo1994/mv-extractor:dev-aarch64 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 4f6122b..e0b61ce 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ -FROM quay.io/pypa/manylinux_2_28_x86_64 AS builder +ARG ARCH=x86_64 + +FROM quay.io/pypa/manylinux_2_28_${ARCH} AS builder WORKDIR /home/video_cap @@ -38,7 +40,7 @@ RUN mkdir -p /home/video_cap && \ chmod +x install_opencv.sh && \ ./install_opencv.sh -FROM quay.io/pypa/manylinux_2_28_x86_64 +FROM quay.io/pypa/manylinux_2_28_${ARCH} RUN yum update -y && \ yum install -y \ From 50af30e5ac1f0bec97d2f846adcc83f17606fc58 Mon Sep 17 00:00:00 2001 From: Lukas Bommes Date: Tue, 29 Oct 2024 20:41:49 +0000 Subject: [PATCH 2/3] add aarch64 build to CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9644592..57124d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -218,7 +218,7 @@ jobs: runs-on: ubuntu-latest needs: - build_docker_x86_64 - - test_docker + - test_docker_x86_64 - build_and_test_wheels steps: From 548758990c25c4251384dd55000443f9727b9905 Mon Sep 17 00:00:00 2001 From: Lukas Bommes Date: Tue, 29 Oct 2024 20:53:59 +0000 Subject: [PATCH 3/3] add aarch64 build to CI --- .github/workflows/ci.yml | 178 ++++++++++++++------------------------- 1 file changed, 63 insertions(+), 115 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57124d4..b1fc9b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,46 +13,17 @@ on: workflow_dispatch: jobs: - build_docker_x86_64: - name: Build Docker image - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and export x86_64 - uses: docker/build-push-action@v6 - with: - context: . - platforms: linux/amd64 - build-args: "ARCH=x86_64" - tags: mv-extractor:local-x86_64 - outputs: type=docker,dest=/tmp/image-x86_64.tar - cache-from: type=registry,ref=lubo1994/mv-extractor:buildcache-x86_64 - cache-to: type=registry,ref=lubo1994/mv-extractor:buildcache-x86_64,mode=max - - - name: Upload x86_64 Docker image as artifact - uses: actions/upload-artifact@v4 - with: - name: mv-extractor-image-x86_64 - path: /tmp/image-x86_64.tar - - build_docker_aarch64: - name: Build Docker image + build_docker: + name: Build Docker image ${{ matrix.arch }} runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - arch: x86_64 + platform_tag: linux/amd64 + - arch: aarch64 + platform_tag: linux/arm64 steps: - name: Checkout @@ -70,55 +41,54 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and export aarch64 + - name: Build and export ${{ matrix.arch }} uses: docker/build-push-action@v6 with: context: . - platforms: linux/arm64 - build-args: "ARCH=aarch64" - tags: mv-extractor:local-aarch64 - outputs: type=docker,dest=/tmp/image-aarch64.tar - cache-from: type=registry,ref=lubo1994/mv-extractor:buildcache-aarch64 - cache-to: type=registry,ref=lubo1994/mv-extractor:buildcache-aarch64,mode=max - - - name: Upload aarch64 Docker image as artifact + platforms: ${{ matrix.platform_tag }} + build-args: "ARCH=${{ matrix.arch }}" + tags: mv-extractor:local-${{ matrix.arch }} + outputs: type=docker,dest=/tmp/image-${{ matrix.arch }}.tar + cache-from: type=registry,ref=lubo1994/mv-extractor:buildcache-${{ matrix.arch }} + cache-to: type=registry,ref=lubo1994/mv-extractor:buildcache-${{ matrix.arch }},mode=max + + - name: Upload ${{ matrix.arch }} Docker image as artifact uses: actions/upload-artifact@v4 with: - name: mv-extractor-image-aarch64 - path: /tmp/image-aarch64.tar - - test_docker_x86_64: - name: Run unit tests in Docker container (only for the Python version used in the Dockerfile command) - runs-on: ubuntu-latest - needs: - - build_docker_x86_64 - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Download x86_64 Docker image - uses: actions/download-artifact@v4 - with: - name: mv-extractor-image-x86_64 - path: /tmp - - - name: Load Docker image - run: | - docker load --input /tmp/image-x86_64.tar - - - name: Run unit tests - run: | - docker run -v ${{ github.workspace }}:/home/video_cap \ - mv-extractor:local-x86_64 \ - python3.12 -m unittest discover -s tests -p "*tests.py" + name: mv-extractor-image-${{ matrix.arch }} + path: /tmp/image-${{ matrix.arch }}.tar + + # test_docker_x86_64: + # name: Run unit tests in x86_64 Docker container (only for the Python version used in the Dockerfile command) + # runs-on: ubuntu-latest + # needs: + # - build_docker_x86_64 + + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Download x86_64 Docker image + # uses: actions/download-artifact@v4 + # with: + # name: mv-extractor-image-x86_64 + # path: /tmp + + # - name: Load Docker image + # run: | + # docker load --input /tmp/image-x86_64.tar + + # - name: Run unit tests + # run: | + # docker run -v ${{ github.workspace }}:/home/video_cap \ + # mv-extractor:local-x86_64 \ + # python3.12 -m unittest discover -s tests -p "*tests.py" build_and_test_wheels: name: Build wheels for cp${{ matrix.python }}-${{ matrix.platform_id }} runs-on: ${{ matrix.os }} needs: - - build_docker_x86_64 - - build_docker_aarch64 + - build_docker strategy: # Ensure that a wheel builder finishes even if another fails fail-fast: false @@ -212,47 +182,25 @@ jobs: name: python-wheel-${{ matrix.python }} path: ./wheelhouse/*.whl - push_docker_x86_64: - name: Push Docker images to DockerHub - if: github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - needs: - - build_docker_x86_64 - - test_docker_x86_64 - - build_and_test_wheels - - steps: - - name: Download x86_64 Docker image - uses: actions/download-artifact@v4 - with: - name: mv-extractor-image-x86_64 - path: /tmp - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Load and push x86_64 Docker image - run: | - docker load --input /tmp/image-x86_64.tar - docker tag mv-extractor:local-x86_64 lubo1994/mv-extractor:dev-x86_64 - docker push lubo1994/mv-extractor:dev-x86_64 - - push_docker_aarch64: - name: Push Docker images to DockerHub + push_docker: + name: Push ${{ matrix.arch }} Docker image to DockerHub if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest needs: - - build_docker_aarch64 + - build_docker - build_and_test_wheels + strategy: + fail-fast: false + matrix: + include: + - arch: x86_64 + - arch: aarch64 steps: - - name: Download aarch64 Docker image + - name: Download ${{ matrix.arch }} Docker image uses: actions/download-artifact@v4 with: - name: mv-extractor-image-aarch64 + name: mv-extractor-image-${{ matrix.arch }} path: /tmp - name: Login to DockerHub @@ -261,8 +209,8 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Load and push aarch64 Docker image + - name: Load and push ${{ matrix.arch }} Docker image run: | - docker load --input /tmp/image-aarch64.tar - docker tag mv-extractor:local-aarch64 lubo1994/mv-extractor:dev-aarch64 - docker push lubo1994/mv-extractor:dev-aarch64 \ No newline at end of file + docker load --input /tmp/image-${{ matrix.arch }}.tar + docker tag mv-extractor:local-${{ matrix.arch }} lubo1994/mv-extractor:dev-${{ matrix.arch }} + docker push lubo1994/mv-extractor:dev-${{ matrix.arch }} \ No newline at end of file