diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index de5891c8f..739ede301 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - backend: [nvidia, metax] + backend: [nvidia, metax, ascend] steps: - name: Checkout repository @@ -27,6 +27,9 @@ jobs: with: submodules: recursive + - name: Sync changelog + run: python3 packaging/sync-changelog.py + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/build-rpm.yml b/.github/workflows/build-rpm.yml new file mode 100644 index 000000000..ec698fa8d --- /dev/null +++ b/.github/workflows/build-rpm.yml @@ -0,0 +1,45 @@ +name: Build RPM Packages + +on: + push: + tags: + - 'v*' + pull_request: + branches: [ main ] + paths: + - 'flagcx/**' + - 'packaging/rpm/**' + - '.github/workflows/build-rpm.yml' + workflow_dispatch: + +jobs: + build-rpm-packages: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + backend: [nvidia, ascend] + # metax requires custom base image setup + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Sync changelog + run: python3 packaging/sync-changelog.py + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build ${{ matrix.backend }} RPM packages + run: ./packaging/rpm/build-flagcx-rpm.sh ${{ matrix.backend }} + + - name: Upload ${{ matrix.backend }} RPM packages + uses: actions/upload-artifact@v4 + with: + name: flagcx-${{ matrix.backend }}-rpm-packages + path: rpm-packages/${{ matrix.backend }}/**/*.rpm + retention-days: 7 diff --git a/.gitignore b/.gitignore index 4703bb0cd..5d7c4d58b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ build plugin/*/build test/*/build debian-packages +rpm-packages # Ignore compiled Python files and shared object files plugin/*/*.so diff --git a/packaging/CHANGELOG-MANAGEMENT.md b/packaging/CHANGELOG-MANAGEMENT.md new file mode 100644 index 000000000..0dd1db93c --- /dev/null +++ b/packaging/CHANGELOG-MANAGEMENT.md @@ -0,0 +1,74 @@ +# Changelog Management + +This document describes how to manage changelogs for FlagCX packages. + +## Overview + +FlagCX uses `docs/CHANGELOG.md` as the single source of truth for release history. This changelog is automatically converted to both Debian and RPM formats during the build process. + +## Files + +- `docs/CHANGELOG.md` - Single source of truth for project release history +- `packaging/sync-changelog.py` - Converts docs/CHANGELOG.md to Debian and RPM formats + +## Workflow + +### Updating Changelogs + +1. Edit `docs/CHANGELOG.md` following the existing format +2. Run the sync script to update package-specific changelogs: + ```bash + python3 packaging/sync-changelog.py + ``` +3. The script will update: + - `packaging/debian/changelog` (Debian format) + - `packaging/rpm/specs/flagcx.spec` (%changelog section) + +## Build Integration + +The changelog sync is automatically run during package builds: + +- **CI/CD**: Both `.github/workflows/build-deb.yml` and `build-rpm.yml` run the sync script +- **Local builds**: Both `build-flagcx.sh` and `build-flagcx-rpm.sh` run the sync script + +## Changelog Format + +### docs/CHANGELOG.md (Source Format) + +```markdown +## Release History + +- **[2025/11]** Released [v0.7](https://github.com/flagos-ai/FlagCX/releases/tag/v0.7.0): + + - Added support to TsingMicro, including device adaptor `tsmicroAdaptor` and CCL adaptor `tcclAdaptor`. + - Implemented an experimental kernel-free non-reduce collective communication + (*SendRecv*, *AlltoAll*, *AlltoAllv*, *Broadcast*, *Gather*, *Scatter*, *AllGather*) + using device-buffer IPC/RDMA. +``` + +### Debian Format (auto-generated) + +``` +flagcx (0.7-1) unstable; urgency=medium + + * Added support to TsingMicro, including device adaptor tsmicroAdaptor and CCL adaptor tcclAdaptor. + * Implemented an experimental kernel-free non-reduce collective communication (SendRecv, AlltoAll, AlltoAllv, Broadcast, Gather, Scatter, AllGather) using device-buffer IPC/RDMA. + + -- FlagOS Contributors Sat, 01 Nov 2025 10:00:00 +0800 +``` + +### RPM Format (auto-generated) + +``` +* Sat Nov 01 2025 FlagOS Contributors - 0.7-1 +- Added support to TsingMicro, including device adaptor tsmicroAdaptor and CCL adaptor tcclAdaptor. +- Implemented an experimental kernel-free non-reduce collective communication (SendRecv, AlltoAll, AlltoAllv, Broadcast, Gather, Scatter, AllGather) using device-buffer IPC/RDMA. +``` + +## Best Practices + +1. Always edit `docs/CHANGELOG.md` directly, never edit the generated files +2. Run `sync-changelog.py` after updating `docs/CHANGELOG.md` +3. Follow the existing format for consistency +4. Use clear, descriptive changelog entries +5. Multi-line entries are supported - just indent continuation lines with 4 spaces diff --git a/packaging/README.md b/packaging/README.md index b30c2f724..e16660e09 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -9,13 +9,23 @@ packaging/ ├── debian/ # Debian/Ubuntu packaging │ ├── control # Package metadata (with build profiles) │ ├── rules # Build rules -│ ├── changelog # Version history +│ ├── changelog # Version history (auto-generated) │ ├── copyright # License information │ └── build-helpers/ # Build scripts and Dockerfiles │ ├── build-flagcx.sh # Unified build script │ ├── Dockerfile.deb # Unified build configuration │ └── test-nexus-upload.sh # Local Nexus upload test script -└── rpm/ # Future: RPM packaging for RHEL/Fedora/etc. +├── rpm/ # RPM packaging for RHEL/Rocky/OpenEuler +│ ├── specs/ +│ │ └── flagcx.spec # RPM spec file +│ ├── dockerfiles/ +│ │ ├── Dockerfile.nvidia # NVIDIA backend build environment +│ │ ├── Dockerfile.metax # MetaX backend build environment +│ │ └── Dockerfile.ascend # Ascend backend build environment +│ └── build-flagcx-rpm.sh # Build script +├── sync-changelog.py # Changelog sync (docs/CHANGELOG.md -> deb/rpm) +├── CHANGELOG-MANAGEMENT.md +└── README.md # This file ``` ## Why `packaging/` Instead of Top-Level `/debian`? @@ -25,158 +35,171 @@ Following [Debian UpstreamGuide](https://wiki.debian.org/UpstreamGuide) recommen > Upstream projects should NOT include a top-level `/debian` directory. > Use `contrib/debian/` or `packaging/debian/` instead. -**Benefits:** +Benefits: - Avoids conflicts with distribution maintainers' packaging - Clearly indicates upstream-maintained packaging - Allows multi-format support (Debian + RPM + others) - Industry standard (see [Miniflux](https://github.com/miniflux/v2/tree/main/packaging), etc.) -## Building Debian Packages +## Supported Backends and Architectures -Use the unified build script to build packages for any vendor/backend: +| Backend | DEB (Debian/Ubuntu) | RPM (RHEL/Rocky/OpenEuler) | +|---------|--------------------|-----------------------------| +| NVIDIA | amd64 | x86_64 (Rocky Linux 8/9) | +| MetaX | amd64 | x86_64 (TBD) | +| Ascend | amd64, arm64 | x86_64, aarch64 (OpenEuler 24.03) | -### Usage +## Building Packages -```bash -./packaging/debian/build-helpers/build-flagcx.sh [base_image_version] -``` - -**Parameters:** -- `` - Hardware vendor/backend (e.g., `nvidia`, `metax`) -- `[base_image_version]` - Optional base image version tag (default: `latest`) +### Prerequisites -**Output:** `debian-packages//*.deb` +- Docker +- Docker Buildx (for multi-architecture builds) +- Python 3 (for changelog sync) -### Examples +### Debian Packages -**Build for NVIDIA:** ```bash -./packaging/debian/build-helpers/build-flagcx.sh nvidia -# Output: debian-packages/nvidia/*.deb -``` +./packaging/debian/build-helpers/build-flagcx.sh [base_image_version] -**Build for MetaX:** -```bash +# Examples +./packaging/debian/build-helpers/build-flagcx.sh nvidia ./packaging/debian/build-helpers/build-flagcx.sh metax -# Output: debian-packages/metax/*.deb -``` - -**Specify custom base image version:** -```bash ./packaging/debian/build-helpers/build-flagcx.sh nvidia v1.2.3 -./packaging/debian/build-helpers/build-flagcx.sh metax latest ``` -### Base Images +Output: `debian-packages//*.deb` -The build script uses upstream base images from `harbor.baai.ac.cn/flagbase/`: +Base images are from `harbor.baai.ac.cn/flagbase/`: - NVIDIA: `flagbase-nvidia:` - MetaX: `flagbase-metax:` -To add support for a new vendor, ensure a corresponding base image exists at: -`harbor.baai.ac.cn/flagbase/flagbase-:` - -### Quality Checks +The build script automatically runs `lintian` for quality checks if available. -The build script automatically runs `lintian` to validate the generated packages if available: +### RPM Packages ```bash -# Install lintian (optional but recommended) -sudo apt-get install lintian +./packaging/rpm/build-flagcx-rpm.sh [base_image_version] -# Build packages - lintian runs automatically -./packaging/debian/build-helpers/build-flagcx.sh nvidia +# Examples +./packaging/rpm/build-flagcx-rpm.sh nvidia +./packaging/rpm/build-flagcx-rpm.sh ascend +./packaging/rpm/build-flagcx-rpm.sh ascend 8.5.0-910-openeuler24.03-py3.11 ``` -Lintian checks are non-fatal and won't stop the build if issues are found. +Output: `rpm-packages//RPMS//*.rpm` + +Default base images: +- NVIDIA: `nvcr.io/nvidia/cuda:12.4.1-devel-rockylinux8` +- Ascend: `ascendai/cann:8.5.0-910-openeuler24.03-py3.11` ## Installation -Install packages for your hardware vendor: +### Debian/Ubuntu ```bash -# General syntax -sudo dpkg -i debian-packages//*.deb +# From local packages +sudo dpkg -i debian-packages//*.deb + +# From APT repository +echo "deb https://resource.flagos.net/repository/flagos-apt-hosted/ flagos-apt-hosted main" | \ + sudo tee /etc/apt/sources.list.d/flagcx.list +sudo apt-get update +sudo apt-get install libflagcx- libflagcx--dev +``` -# Example: NVIDIA -sudo dpkg -i debian-packages/nvidia/*.deb +### RHEL/Rocky/OpenEuler -# Example: MetaX -sudo dpkg -i debian-packages/metax/*.deb +```bash +sudo yum install libflagcx-nvidia-0.8.0-1.el8.x86_64.rpm +sudo yum install libflagcx-nvidia-devel-0.8.0-1.el8.x86_64.rpm + +# Or on OpenEuler +sudo dnf install libflagcx-ascend-0.8.0-1.oe2403.aarch64.rpm ``` -## CI/CD +## Package Contents -Automated builds are triggered by: -- Push to `main` branch (when packaging files change) -- Pull requests to `main` -- Manual workflow dispatch +### Runtime Package (libflagcx-{backend}) +- DEB: `/usr/lib//libflagcx.so.*` +- RPM: `/usr/lib64/libflagcx.so.*` -See `.github/workflows/build-deb.yml` for details. +### Development Package (libflagcx-{backend}-dev / -devel) +- Headers: `/usr/include/flagcx/` +- DEB: `/usr/lib//libflagcx.so` +- RPM: `/usr/lib64/libflagcx.so` -### Publishing to Nexus APT Repository +## Changelog Management -Packages are uploaded to Nexus when: -- A version tag is pushed: `git tag v1.0.0 && git push origin v1.0.0` -- Manual workflow dispatch via GitHub Actions +Changelogs are managed from a single source: `docs/CHANGELOG.md`. -After upload, users can install packages from the APT repository: +The sync script automatically converts it to both Debian and RPM formats: ```bash -# Add the FlagOS APT repository -echo "deb https://resource.flagos.net/repository/flagos-apt-hosted/ flagos-apt-hosted main" | \ - sudo tee /etc/apt/sources.list.d/flagcx.list +python3 packaging/sync-changelog.py +``` -# Update package list -sudo apt-get update +See [CHANGELOG-MANAGEMENT.md](CHANGELOG-MANAGEMENT.md) for details. -# Install packages for your vendor -sudo apt-get install libflagcx- # Runtime library -sudo apt-get install libflagcx--dev # Development files +## CI/CD -# Examples: -sudo apt-get install libflagcx-nvidia libflagcx-nvidia-dev -sudo apt-get install libflagcx-metax libflagcx-metax-dev -``` +Automated builds are triggered by: +- Version tag push (`v*`) +- Pull requests to `main` (when packaging files change) +- Manual workflow dispatch + +Workflows: +- `.github/workflows/build-deb.yml` - Debian packages +- `.github/workflows/build-rpm.yml` - RPM packages ## Architecture -The build process uses a **unified multi-stage Dockerfile** with build profiles: +### DEB Build + +Uses a unified multi-stage Dockerfile with build profiles: +- `pkg.flagcx.nvidia-only` / `pkg.flagcx.metax-only` / `pkg.flagcx.ascend-only` +- Single `Dockerfile.deb` for all backends via build arguments +- Builder stage (flagbase image) -> Output stage (Alpine, .deb files only) + +### RPM Build + +Uses per-backend Dockerfiles with native RPM distributions: +- NVIDIA: Rocky Linux 8 with CUDA toolkit +- Ascend: OpenEuler 24.03 with CANN toolkit +- Builds via `rpmbuild` with `--define 'backend '` -### Build Profiles Support +### Why Separate Build Environments? -The `debian/control` file defines build profiles to support multiple backends: -- `pkg.flagcx.nvidia-only` - Build only NVIDIA packages -- `pkg.flagcx.metax-only` - Build only MetaX packages +RPM packages must be built on RPM-based distributions: +- Different file system layouts (`/usr/lib64` vs `/usr/lib/`) +- Different dependency resolution (yum/dnf vs apt) +- System library version mismatches -### Unified Dockerfile +## Key Differences: DEB vs RPM -A single `Dockerfile.deb` builds packages for all backends using build arguments: -- `BASE_IMAGE` - Upstream base image (e.g., `flagbase-nvidia`, `flagbase-metax`) -- `BASE_IMAGE_VERSION` - Image version tag (default: `latest`) -- `VENDOR` - Backend vendor name (used for build profile selection) +| Aspect | Debian/Ubuntu | RPM (RHEL/Rocky/OpenEuler) | +|--------|---------------|----------------------------| +| Package Tool | dpkg-buildpackage | rpmbuild | +| Lib Directory | /usr/lib/x86_64-linux-gnu/ | /usr/lib64/ | +| Build Deps | Build-Depends in control | BuildRequires in spec | +| Runtime Deps | Depends in control | Requires in spec | +| Profiles | Build-Profiles | RPM macros (--define) | +| Quality Check | lintian | rpmlint | -### Build Stages +## Troubleshooting -1. **Builder stage**: Based on upstream flagbase images - - Contains all necessary build dependencies (CUDA/NCCL or MACA SDK) - - Installs Debian packaging tools (`debhelper`, `dpkg-dev`, etc.) - - Runs `dpkg-buildpackage` with `DEB_BUILD_PROFILES=pkg.flagcx.${VENDOR}-only` - - Only builds packages for the specified vendor +### Missing dependencies during build +Ensure the base image includes all required SDKs (CUDA, MACA, CANN). -2. **Output stage**: Minimal Alpine image - - Only contains the built `.deb` files - - Used to extract packages to the host +### Wrong library path (RPM) +RPM uses `/usr/lib64` on x86_64, not `/usr/lib/x86_64-linux-gnu`. -This approach ensures: -- ✓ Reproducible builds using official base images -- ✓ Single Dockerfile for all backends (DRY principle) -- ✓ Backend selection via build profiles -- ✓ No custom Docker images to maintain -- ✓ Clean separation of build environment and outputs +### SONAME conflicts (RPM) +Use `patchelf --set-soname` to fix SONAME in the spec file. -## Future Plans +## References -- [ ] Add RPM packaging in `packaging/rpm/` -- [ ] Add Arch Linux packaging -- [x] Add APT repository hosting (Nexus) +- [Debian UpstreamGuide](https://wiki.debian.org/UpstreamGuide) +- [RPM Packaging Guide](https://rpm-packaging-guide.github.io/) +- [Fedora Packaging Guidelines](https://docs.fedoraproject.org/en-US/packaging-guidelines/) +- [OpenEuler Packaging](https://docs.openeuler.org/en/docs/22.03_LTS/docs/ApplicationDev/packaging-software.html) diff --git a/packaging/debian/build-helpers/build-flagcx.sh b/packaging/debian/build-helpers/build-flagcx.sh index 14a7b4389..a40afeafb 100755 --- a/packaging/debian/build-helpers/build-flagcx.sh +++ b/packaging/debian/build-helpers/build-flagcx.sh @@ -62,6 +62,14 @@ esac log_info "Building FlagCX Debian packages for $BACKEND backend" log_info "Using base image: ${BASE_IMAGE}:${BASE_IMAGE_VERSION}" +# Sync changelog from CHANGELOG.md +log_step "Synchronizing changelog..." +if [ -f "${PROJECT_DIR}/packaging/sync-changelog.py" ]; then + python3 "${PROJECT_DIR}/packaging/sync-changelog.py" || log_warn "Failed to sync changelog" +else + log_warn "sync-changelog.py not found, skipping changelog sync" +fi + DOCKERFILE="${SCRIPT_DIR}/Dockerfile.deb" # Check if Dockerfile exists diff --git a/packaging/debian/changelog b/packaging/debian/changelog index 625686a00..e133c423d 100644 --- a/packaging/debian/changelog +++ b/packaging/debian/changelog @@ -1,9 +1,61 @@ -flagcx (0.8.0-1) unstable; urgency=medium +flagcx (0.7-1) unstable; urgency=medium - * New upstream release 0.8.0 - * Support for MetaX and NVIDIA hardware backends - * Split packages for runtime and development files - * Improved build process with APT-based MetaX SDK + * Added support to TsingMicro, including device adaptor tsmicroAdaptor and CCL adaptor tcclAdaptor. + * Implemented an experimental kernel-free non-reduce collective communication (SendRecv, AlltoAll, AlltoAllv, Broadcast, Gather, Scatter, AllGather) using device-buffer IPC/RDMA. + * Enabled auto-tuning on NVIDIA, MetaX, and Hygon platforms, achieving 1.02×–1.26× speedups for AllReduce, AllGather, ReduceScatter, and AlltoAll. + * Enhanced flagcxNetAdaptor with one-sided primitives (put, putSignal, waitValue) and added retransmission support for reliability improvement. - -- FlagOS Contributors Sat, 17 Jan 2026 17:00:00 +0800 + -- FlagOS Contributors Sat, 01 Nov 2025 10:00:00 +0800 +flagcx (0.6-1) unstable; urgency=medium + + * Implemented device-buffer IPC communication to support intra-node SendRecv operations. + * Introduced device-initiated, host-launched device-side primitives, enabling kernel-based communication directly from devices. + * Enhanced auto-tuning with 50% performance improvement on MetaX platforms for the AllReduce operations. + + -- FlagOS Contributors Wed, 01 Oct 2025 10:00:00 +0800 + +flagcx (0.5-1) unstable; urgency=medium + + * Added support for AMD GPUs, including a device adaptor hipAdaptor and a CCL adaptor rcclAdaptor. + * Introduced flagcxNetAdaptor to unify network backends, currently supporting socket, IBRC, UCX and IBUC (experimental). + * Enabled zero-copy device-buffer RDMA (user-buffer RDMA) to boost performance for small messages. + * Supported auto-tuning in homogeneous scenarios via flagcxTuner. + * Added test automation in CI/CD for PyTorch APIs. + + -- FlagOS Contributors Mon, 01 Sep 2025 10:00:00 +0800 + +flagcx (0.4-1) unstable; urgency=medium + + * Supported heterogeneous training of ERNIE4.5 (Baidu) on NVIDIA and Iluvatar GPUs with Paddle + FlagCX. + * Improved heterogeneous communication across arbitrary NIC configurations, with more robust and flexible deployments. + * Introduced an experimental network plugin interface with extended supports for IBRC and SOCKET. Device buffer registration now can be done via DMA-BUF. + * Added an InterOp-level DSL to enable customized C2C algorithm design. + * Provided user documentation under docs/. + + -- FlagOS Contributors Fri, 01 Aug 2025 10:00:00 +0800 + +flagcx (0.3-1) unstable; urgency=medium + + * Integrated three additional native communication libraries: HCCL (Huawei), MUSACCL (Moore Threads) and MPI. + * Enhanced heterogeneous collective communication operations with pipeline optimizations. + * Introduced device-side functions to enable device-buffer RDMA, complementing the existing host-side functions. + * Delivered a full-stack open-source solution, FlagScale + FlagCX, for efficient heterogeneous prefilling-decoding disaggregation. + + -- FlagOS Contributors Tue, 01 Jul 2025 10:00:00 +0800 + +flagcx (0.2-1) unstable; urgency=medium + + * Integrated 3 additional native communications libraries, including MCCL (Moore Threads), XCCL (Mellanox) and DUCCL (BAAI). + * Improved 11 heterogeneous collective communication operations with automatic topology detection and full support to single-NIC and multi-NIC environments. + + -- FlagOS Contributors Thu, 01 May 2025 10:00:00 +0800 + +flagcx (0.1-1) unstable; urgency=medium + + * Added 5 native communications libraries including CCL adaptors for NCCL (NVIDIA), IXCCL (Iluvatar), and CNCL (Cambricon), and Host CCL adaptors GLOO and Bootstrap. + * Supported 11 heterogeneous collective communication operations using the C2C (Cluster-to-Cluster) algorithm. + * Provided a full-stack open-source solution, FlagScale + FlagCX, for efficient heterogeneous training. + * Natively integrated into PaddlePaddle [v3.0.0](https://github.com/PaddlePaddle/Paddle/tree/v3.0.0), with support for both dynamic and static graphs. + + -- FlagOS Contributors Tue, 01 Apr 2025 10:00:00 +0800 diff --git a/packaging/debian/control b/packaging/debian/control index 26dc89c17..2fed7819f 100644 --- a/packaging/debian/control +++ b/packaging/debian/control @@ -20,12 +20,15 @@ X-Build-Environment: This package is designed to be built in backend-specific . - MetaX backend: Requires maca_sdk (from repos.metax-tech.com) Built in Ubuntu 22.04 with MetaX APT repository configured + . + - Ascend backend: Requires Ascend CANN toolkit + Built in Ubuntu 22.04 with Ascend SDK . The Build-Depends listed above are for the Debian packaging tools only. Backend-specific dependencies are provided by the container environment. Package: libflagcx-metax -Build-Profiles: +Build-Profiles: Architecture: amd64 Depends: ${shlibs:Depends}, ${misc:Depends} Description: FlagCX communication library for MetaX hardware @@ -37,7 +40,7 @@ Description: FlagCX communication library for MetaX hardware MCCL backend support. Package: libflagcx-metax-dev -Build-Profiles: +Build-Profiles: Section: libdevel Architecture: amd64 Depends: libflagcx-metax (= ${binary:Version}), ${misc:Depends} @@ -48,7 +51,7 @@ Description: FlagCX development files for MetaX hardware building applications against libflagcx-metax. Package: libflagcx-nvidia -Build-Profiles: +Build-Profiles: Architecture: amd64 Depends: ${shlibs:Depends}, ${misc:Depends} Description: FlagCX communication library for NVIDIA hardware @@ -60,7 +63,7 @@ Description: FlagCX communication library for NVIDIA hardware NCCL backend support. Package: libflagcx-nvidia-dev -Build-Profiles: +Build-Profiles: Section: libdevel Architecture: amd64 Depends: libflagcx-nvidia (= ${binary:Version}), ${misc:Depends} @@ -69,3 +72,26 @@ Description: FlagCX development files for NVIDIA hardware . This package contains the header files and development files for building applications against libflagcx-nvidia. + +Package: libflagcx-ascend +Build-Profiles: +Architecture: amd64 arm64 +Depends: ${shlibs:Depends}, ${misc:Depends} +Description: FlagCX communication library for Ascend NPUs + FlagCX is a scalable and adaptive cross-chip communication library. + It serves as a platform where developers, researchers, and AI engineers + can collaborate on various projects. + . + This package contains the shared library built for Huawei Ascend NPUs + with HCCL backend support. + +Package: libflagcx-ascend-dev +Build-Profiles: +Section: libdevel +Architecture: amd64 arm64 +Depends: libflagcx-ascend (= ${binary:Version}), ${misc:Depends} +Description: FlagCX development files for Ascend NPUs + FlagCX is a scalable and adaptive cross-chip communication library. + . + This package contains the header files and development files for + building applications against libflagcx-ascend. diff --git a/packaging/debian/rules b/packaging/debian/rules index a80459541..1fcd229d4 100755 --- a/packaging/debian/rules +++ b/packaging/debian/rules @@ -4,19 +4,20 @@ export DH_VERBOSE = 1 # Allow building only specific backend via environment variable # Usage: FLAGCX_BUILD_BACKEND=metax dpkg-buildpackage ... -# Valid values: metax, nvidia, all (default) +# Valid values: metax, nvidia, ascend, all (default) FLAGCX_BUILD_BACKEND ?= all # Build directories for different backends BUILD_DIR_METAX = $(CURDIR)/build-metax BUILD_DIR_NVIDIA = $(CURDIR)/build-nvidia +BUILD_DIR_ASCEND = $(CURDIR)/build-ascend %: dh $@ override_dh_auto_clean: dh_auto_clean - rm -rf $(BUILD_DIR_METAX) $(BUILD_DIR_NVIDIA) + rm -rf $(BUILD_DIR_METAX) $(BUILD_DIR_NVIDIA) $(BUILD_DIR_ASCEND) $(MAKE) clean || true override_dh_auto_build: @@ -38,6 +39,15 @@ else ifeq ($(FLAGCX_BUILD_BACKEND),metax) mkdir -p $(BUILD_DIR_METAX)/lib $(BUILD_DIR_METAX)/include cp -a build/lib/libflagcx.so* $(BUILD_DIR_METAX)/lib/ || cp -a build/libflagcx.so* $(BUILD_DIR_METAX)/lib/ cp -r flagcx/include $(BUILD_DIR_METAX)/include/flagcx +else ifeq ($(FLAGCX_BUILD_BACKEND),ascend) + @echo "Building Ascend variant only" + # Build Ascend variant + mkdir -p $(BUILD_DIR_ASCEND) + $(MAKE) USE_ASCEND=1 PREFIX=/usr + # Copy built files to ascend build dir + mkdir -p $(BUILD_DIR_ASCEND)/lib $(BUILD_DIR_ASCEND)/include + cp -a build/lib/libflagcx.so* $(BUILD_DIR_ASCEND)/lib/ || cp -a build/libflagcx.so* $(BUILD_DIR_ASCEND)/lib/ + cp -r flagcx/include $(BUILD_DIR_ASCEND)/include/flagcx else @echo "Building both variants" # Build MetaX variant @@ -87,6 +97,19 @@ else ifeq ($(FLAGCX_BUILD_BACKEND),metax) ln -sf libflagcx.so libflagcx.so.0 && \ ln -sf libflagcx.so libflagcx.so.0.1.0 cp -r $(BUILD_DIR_METAX)/include/* debian/libflagcx-metax-dev/usr/include/flagcx/ +else ifeq ($(FLAGCX_BUILD_BACKEND),ascend) + @echo "Installing Ascend variant only" + # Install Ascend variant + mkdir -p debian/libflagcx-ascend/usr/lib + mkdir -p debian/libflagcx-ascend-dev/usr/include/flagcx + cp -a $(BUILD_DIR_ASCEND)/lib/libflagcx.so* debian/libflagcx-ascend/usr/lib/ + # Fix SONAME and RPATH + cd debian/libflagcx-ascend/usr/lib && \ + patchelf --set-soname libflagcx.so.0 libflagcx.so && \ + patchelf --remove-rpath libflagcx.so && \ + ln -sf libflagcx.so libflagcx.so.0 && \ + ln -sf libflagcx.so libflagcx.so.0.1.0 + cp -r $(BUILD_DIR_ASCEND)/include/* debian/libflagcx-ascend-dev/usr/include/flagcx/ else @echo "Installing both variants" # Install MetaX variant diff --git a/packaging/rpm/build-flagcx-rpm.sh b/packaging/rpm/build-flagcx-rpm.sh new file mode 100755 index 000000000..3aef9ad5b --- /dev/null +++ b/packaging/rpm/build-flagcx-rpm.sh @@ -0,0 +1,98 @@ +#!/bin/bash +set -e + +# FlagCX RPM package build script +# Usage: ./build-flagcx-rpm.sh [base_image_version] +# Supported backends: nvidia, metax, ascend + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" +BACKEND="${1:-}" +BASE_IMAGE_VERSION="${2:-}" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } +log_step() { echo -e "${BLUE}[STEP]${NC} $1"; } + +# Show usage +if [ -z "$BACKEND" ]; then + log_error "No backend specified" + echo "" + echo "Usage: $0 [base_image_version]" + echo "" + echo "Supported backends:" + echo " nvidia - Build RPM packages for NVIDIA GPUs" + echo " metax - Build RPM packages for MetaX accelerators" + echo " ascend - Build RPM packages for Ascend NPUs" + echo "" + echo "Examples:" + echo " $0 nvidia" + echo " $0 ascend 8.5.0-910-openeuler24.03-py3.11" + exit 1 +fi + +# Validate backend and set base image +case "$BACKEND" in + nvidia) + BASE_IMAGE="nvcr.io/nvidia/cuda" + [ -z "$BASE_IMAGE_VERSION" ] && BASE_IMAGE_VERSION="12.4.1-devel-rockylinux8" + ;; + metax) + BASE_IMAGE="harbor.baai.ac.cn/flagbase/flagbase-metax" + [ -z "$BASE_IMAGE_VERSION" ] && BASE_IMAGE_VERSION="latest" + log_warn "MetaX RPM build may require custom base image with RPM tools" + ;; + ascend) + BASE_IMAGE="ascendai/cann" + [ -z "$BASE_IMAGE_VERSION" ] && BASE_IMAGE_VERSION="8.5.0-910-openeuler24.03-py3.11" + ;; + *) + log_error "Invalid backend: $BACKEND" + echo "Supported backends: nvidia, metax, ascend" + exit 1 + ;; +esac + +log_info "Building FlagCX RPM packages for $BACKEND backend" +log_info "Using base image: ${BASE_IMAGE}:${BASE_IMAGE_VERSION}" + +# Sync changelog from CHANGELOG.md +log_step "Synchronizing changelog..." +if [ -f "${PROJECT_DIR}/packaging/sync-changelog.py" ]; then + python3 "${PROJECT_DIR}/packaging/sync-changelog.py" || log_warn "Failed to sync changelog" +else + log_warn "sync-changelog.py not found, skipping changelog sync" +fi + +# Build Docker image +log_step "Building Docker image..." +docker build \ + --build-arg BASE_IMAGE="${BASE_IMAGE}" \ + --build-arg BASE_IMAGE_VERSION="${BASE_IMAGE_VERSION}" \ + -f "${SCRIPT_DIR}/dockerfiles/Dockerfile.${BACKEND}" \ + -t "flagcx-rpm-${BACKEND}:${BASE_IMAGE_VERSION}" \ + "${PROJECT_DIR}" + +# Extract RPM packages +log_step "Extracting RPM packages..." +OUTPUT_DIR="${PROJECT_DIR}/rpm-packages/${BACKEND}" +mkdir -p "${OUTPUT_DIR}" + +CONTAINER_ID=$(docker create "flagcx-rpm-${BACKEND}:${BASE_IMAGE_VERSION}") +docker cp "${CONTAINER_ID}:/root/rpmbuild/RPMS/" "${OUTPUT_DIR}/" 2>/dev/null || true +docker cp "${CONTAINER_ID}:/root/rpmbuild/SRPMS/" "${OUTPUT_DIR}/" 2>/dev/null || true +docker rm "${CONTAINER_ID}" + +log_info "✓ Packages built successfully for ${BACKEND}:" +echo "" +find "${OUTPUT_DIR}" -name "*.rpm" -exec ls -lh {} \; + +log_info "Build complete! Packages in: ${OUTPUT_DIR}" diff --git a/packaging/rpm/dockerfiles/Dockerfile.ascend b/packaging/rpm/dockerfiles/Dockerfile.ascend new file mode 100644 index 000000000..338c0e914 --- /dev/null +++ b/packaging/rpm/dockerfiles/Dockerfile.ascend @@ -0,0 +1,41 @@ +ARG BASE_IMAGE=ascendai/cann +ARG BASE_IMAGE_VERSION=8.5.0-910-openeuler24.03-py3.11 + +FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION} + +# Install RPM build tools +# OpenEuler uses different package names +RUN yum install -y \ + rpm-build \ + rpmdevtools \ + gcc-c++ \ + make \ + cmake \ + patchelf \ + nlohmann-json-devel \ + && yum clean all + +# Setup RPM build environment +RUN rpmdev-setuptree + +# Copy source code +WORKDIR /workspace +COPY . /workspace/ + +# Create source tarball +RUN tar czf /root/rpmbuild/SOURCES/flagcx-0.8.0.tar.gz \ + --transform 's,^\.,flagcx-0.8.0,' \ + --exclude='.git' \ + --exclude='build' \ + --exclude='debian-packages' \ + . + +# Build RPM with Ascend backend +RUN rpmbuild -ba \ + --define 'backend ascend' \ + /workspace/packaging/rpm/specs/flagcx.spec + +# List built packages +RUN ls -lh /root/rpmbuild/RPMS/*/*.rpm + +CMD ["/bin/bash"] diff --git a/packaging/rpm/dockerfiles/Dockerfile.nvidia b/packaging/rpm/dockerfiles/Dockerfile.nvidia new file mode 100644 index 000000000..6462100fd --- /dev/null +++ b/packaging/rpm/dockerfiles/Dockerfile.nvidia @@ -0,0 +1,41 @@ +ARG BASE_IMAGE=nvcr.io/nvidia/cuda +ARG BASE_IMAGE_VERSION=12.4.1-devel-rockylinux8 + +FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION} + +# Install EPEL and RPM build tools +RUN yum install -y epel-release && \ + yum install -y \ + rpm-build \ + rpmdevtools \ + gcc-c++ \ + make \ + cmake \ + patchelf \ + json-devel \ + && yum clean all + +# Setup RPM build environment +RUN rpmdev-setuptree + +# Copy source code +WORKDIR /workspace +COPY . /workspace/ + +# Create source tarball +RUN tar czf /root/rpmbuild/SOURCES/flagcx-0.8.0.tar.gz \ + --transform 's,^\.,flagcx-0.8.0,' \ + --exclude='.git' \ + --exclude='build' \ + --exclude='debian-packages' \ + . + +# Build RPM with NVIDIA backend +RUN rpmbuild -ba \ + --define 'backend nvidia' \ + /workspace/packaging/rpm/specs/flagcx.spec + +# List built packages +RUN ls -lh /root/rpmbuild/RPMS/*/*.rpm + +CMD ["/bin/bash"] diff --git a/packaging/rpm/specs/flagcx.spec b/packaging/rpm/specs/flagcx.spec new file mode 100644 index 000000000..0012125b8 --- /dev/null +++ b/packaging/rpm/specs/flagcx.spec @@ -0,0 +1,221 @@ +%global debug_package %{nil} +%global _build_id_links none + +Name: flagcx +Version: 0.8.0 +Release: 1%{?dist} +Summary: FlagCX scalable cross-chip communication library + +License: ASL 2.0 +URL: https://github.com/flagos-ai/FlagCX +Source0: %{name}-%{version}.tar.gz + +BuildRequires: gcc-c++ +BuildRequires: make +BuildRequires: cmake +BuildRequires: patchelf +# nlohmann-json package name varies by distro +%if 0%{?rhel} == 8 +BuildRequires: json-devel +%else +BuildRequires: nlohmann-json-devel +%endif + +# Backend-specific packages will be built with different profiles +# This is the base spec, actual builds use --define 'backend nvidia|metax|ascend' + +%description +FlagCX is a scalable and adaptive cross-chip communication library. +It serves as a platform where developers, researchers, and AI engineers +can collaborate on various projects. + +%package -n libflagcx-nvidia +Summary: FlagCX library for NVIDIA GPUs +Requires: libnccl >= 2.0 + +%description -n libflagcx-nvidia +FlagCX communication library built for NVIDIA hardware with NCCL backend support. + +%package -n libflagcx-nvidia-devel +Summary: Development files for libflagcx-nvidia +Requires: libflagcx-nvidia = %{version}-%{release} + +%description -n libflagcx-nvidia-devel +Development files (headers and libraries) for libflagcx-nvidia. + +%package -n libflagcx-metax +Summary: FlagCX library for MetaX accelerators + +%description -n libflagcx-metax +FlagCX communication library built for MetaX hardware with MCCL backend support. + +%package -n libflagcx-metax-devel +Summary: Development files for libflagcx-metax +Requires: libflagcx-metax = %{version}-%{release} + +%description -n libflagcx-metax-devel +Development files (headers and libraries) for libflagcx-metax. + +%package -n libflagcx-ascend +Summary: FlagCX library for Ascend NPUs + +%description -n libflagcx-ascend +FlagCX communication library built for Huawei Ascend NPUs with HCCL backend support. + +%package -n libflagcx-ascend-devel +Summary: Development files for libflagcx-ascend +Requires: libflagcx-ascend = %{version}-%{release} + +%description -n libflagcx-ascend-devel +Development files (headers and libraries) for libflagcx-ascend. + +%prep +%setup -q + +%build +# Determine which backend to build based on RPM macro +%if "%{?backend}" == "nvidia" + make USE_NVIDIA=1 PREFIX=%{_prefix} +%endif + +%if "%{?backend}" == "metax" + make USE_METAX=1 PREFIX=%{_prefix} +%endif + +%if "%{?backend}" == "ascend" + make USE_ASCEND=1 PREFIX=%{_prefix} +%endif + +%install +rm -rf %{buildroot} + +%if "%{?backend}" == "nvidia" + # Install NVIDIA variant + install -d %{buildroot}%{_libdir} + + # Install library + install -m 755 build/lib/libflagcx.so %{buildroot}%{_libdir}/libflagcx.so.0 + + # Create symlinks + ln -s libflagcx.so.0 %{buildroot}%{_libdir}/libflagcx.so + + install -d %{buildroot}%{_includedir}/flagcx + cp -r flagcx/include/* %{buildroot}%{_includedir}/flagcx/ + + # Fix RPATH and set SONAME + patchelf --remove-rpath %{buildroot}%{_libdir}/libflagcx.so.0 || true + patchelf --set-soname libflagcx.so.0 %{buildroot}%{_libdir}/libflagcx.so.0 || true +%endif + +%if "%{?backend}" == "metax" + # Install MetaX variant + install -d %{buildroot}%{_libdir} + + # Install library + install -m 755 build/lib/libflagcx.so %{buildroot}%{_libdir}/libflagcx.so.0 + + # Create symlinks + ln -s libflagcx.so.0 %{buildroot}%{_libdir}/libflagcx.so + + install -d %{buildroot}%{_includedir}/flagcx + cp -r flagcx/include/* %{buildroot}%{_includedir}/flagcx/ + + patchelf --remove-rpath %{buildroot}%{_libdir}/libflagcx.so.0 || true + patchelf --set-soname libflagcx.so.0 %{buildroot}%{_libdir}/libflagcx.so.0 || true +%endif + +%if "%{?backend}" == "ascend" + # Install Ascend variant + install -d %{buildroot}%{_libdir} + + # Install library + install -m 755 build/lib/libflagcx.so %{buildroot}%{_libdir}/libflagcx.so.0 + + # Create symlinks + ln -s libflagcx.so.0 %{buildroot}%{_libdir}/libflagcx.so + + install -d %{buildroot}%{_includedir}/flagcx + cp -r flagcx/include/* %{buildroot}%{_includedir}/flagcx/ + + patchelf --remove-rpath %{buildroot}%{_libdir}/libflagcx.so.0 || true + patchelf --set-soname libflagcx.so.0 %{buildroot}%{_libdir}/libflagcx.so.0 || true +%endif + +%files -n libflagcx-nvidia +%if "%{?backend}" == "nvidia" +%license LICENSE +%{_libdir}/libflagcx.so.0 +%endif + +%files -n libflagcx-nvidia-devel +%if "%{?backend}" == "nvidia" +%{_includedir}/flagcx/ +%{_libdir}/libflagcx.so +%endif + +%files -n libflagcx-metax +%if "%{?backend}" == "metax" +%license LICENSE +%{_libdir}/libflagcx.so.0 +%endif + +%files -n libflagcx-metax-devel +%if "%{?backend}" == "metax" +%{_includedir}/flagcx/ +%{_libdir}/libflagcx.so +%endif + +%files -n libflagcx-ascend +%if "%{?backend}" == "ascend" +%license LICENSE +%{_libdir}/libflagcx.so.0 +%{_libdir}/libflagcx.so.%{version} +%endif + +%files -n libflagcx-ascend-devel +%if "%{?backend}" == "ascend" +%{_includedir}/flagcx/ +%{_libdir}/libflagcx.so +%endif + +%changelog +* Sat Nov 01 2025 FlagOS Contributors - 0.7-1 +- Added support to TsingMicro, including device adaptor tsmicroAdaptor and CCL adaptor tcclAdaptor. +- Implemented an experimental kernel-free non-reduce collective communication (SendRecv, AlltoAll, AlltoAllv, Broadcast, Gather, Scatter, AllGather) using device-buffer IPC/RDMA. +- Enabled auto-tuning on NVIDIA, MetaX, and Hygon platforms, achieving 1.02×–1.26× speedups for AllReduce, AllGather, ReduceScatter, and AlltoAll. +- Enhanced flagcxNetAdaptor with one-sided primitives (put, putSignal, waitValue) and added retransmission support for reliability improvement. + +* Wed Oct 01 2025 FlagOS Contributors - 0.6-1 +- Implemented device-buffer IPC communication to support intra-node SendRecv operations. +- Introduced device-initiated, host-launched device-side primitives, enabling kernel-based communication directly from devices. +- Enhanced auto-tuning with 50% performance improvement on MetaX platforms for the AllReduce operations. + +* Mon Sep 01 2025 FlagOS Contributors - 0.5-1 +- Added support for AMD GPUs, including a device adaptor hipAdaptor and a CCL adaptor rcclAdaptor. +- Introduced flagcxNetAdaptor to unify network backends, currently supporting socket, IBRC, UCX and IBUC (experimental). +- Enabled zero-copy device-buffer RDMA (user-buffer RDMA) to boost performance for small messages. +- Supported auto-tuning in homogeneous scenarios via flagcxTuner. +- Added test automation in CI/CD for PyTorch APIs. + +* Fri Aug 01 2025 FlagOS Contributors - 0.4-1 +- Supported heterogeneous training of ERNIE4.5 (Baidu) on NVIDIA and Iluvatar GPUs with Paddle + FlagCX. +- Improved heterogeneous communication across arbitrary NIC configurations, with more robust and flexible deployments. +- Introduced an experimental network plugin interface with extended supports for IBRC and SOCKET. Device buffer registration now can be done via DMA-BUF. +- Added an InterOp-level DSL to enable customized C2C algorithm design. +- Provided user documentation under docs/. + +* Tue Jul 01 2025 FlagOS Contributors - 0.3-1 +- Integrated three additional native communication libraries: HCCL (Huawei), MUSACCL (Moore Threads) and MPI. +- Enhanced heterogeneous collective communication operations with pipeline optimizations. +- Introduced device-side functions to enable device-buffer RDMA, complementing the existing host-side functions. +- Delivered a full-stack open-source solution, FlagScale + FlagCX, for efficient heterogeneous prefilling-decoding disaggregation. + +* Thu May 01 2025 FlagOS Contributors - 0.2-1 +- Integrated 3 additional native communications libraries, including MCCL (Moore Threads), XCCL (Mellanox) and DUCCL (BAAI). +- Improved 11 heterogeneous collective communication operations with automatic topology detection and full support to single-NIC and multi-NIC environments. + +* Tue Apr 01 2025 FlagOS Contributors - 0.1-1 +- Added 5 native communications libraries including CCL adaptors for NCCL (NVIDIA), IXCCL (Iluvatar), and CNCL (Cambricon), and Host CCL adaptors GLOO and Bootstrap. +- Supported 11 heterogeneous collective communication operations using the C2C (Cluster-to-Cluster) algorithm. +- Provided a full-stack open-source solution, FlagScale + FlagCX, for efficient heterogeneous training. +- Natively integrated into PaddlePaddle [v3.0.0](https://github.com/PaddlePaddle/Paddle/tree/v3.0.0), with support for both dynamic and static graphs. diff --git a/packaging/sync-changelog.py b/packaging/sync-changelog.py new file mode 100755 index 000000000..ca112bc8b --- /dev/null +++ b/packaging/sync-changelog.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +""" +Convert docs/CHANGELOG.md to Debian changelog and RPM %changelog format +""" + +import re +import sys +from datetime import datetime +from pathlib import Path + +def parse_changelog(changelog_path): + """Parse docs/CHANGELOG.md and extract version entries""" + with open(changelog_path, 'r') as f: + content = f.read() + + # Pattern: - **[YYYY/MM]** Released [vX.Y.Z](link): + version_pattern = r'- \*\*\[(\d{4})/(\d{2})\]\*\* Released \[v([^\]]+)\]' + versions = [] + + lines = content.split('\n') + i = 0 + + while i < len(lines): + line = lines[i] + match = re.search(version_pattern, line) + + if match: + year = match.group(1) + month = match.group(2) + version = match.group(3) + + # Construct date (use first day of month) + date = f"{year}-{month}-01" + + # Collect change entries (lines starting with " - ") + entries = [] + i += 1 + + # Skip empty lines after version header + while i < len(lines) and lines[i].strip() == '': + i += 1 + + # Collect bullet points + while i < len(lines): + line = lines[i] + + # Stop if we hit another version entry + if re.search(version_pattern, line): + break + + # Collect bullet points (lines starting with " - ") + if line.strip().startswith('- '): + # Clean up the entry + entry = line.strip()[2:].strip() + + # Continue collecting if next lines are continuations (indented but not bullet points) + i += 1 + while i < len(lines): + next_line = lines[i] + # If it's a continuation (starts with spaces but not a bullet) + if next_line.startswith(' ') and not next_line.strip().startswith('-'): + entry += ' ' + next_line.strip() + i += 1 + else: + break + + # Remove markdown formatting + entry = re.sub(r'\*([^*]+)\*', r'\1', entry) # Remove *italic* + entry = re.sub(r'_([^_]+)_', r'\1', entry) # Remove _italic_ + entry = re.sub(r'`([^`]+)`', r'\1', entry) # Remove `code` + entries.append(entry) + elif line.strip() == '': + # Empty line might indicate end of this version's entries + i += 1 + # Check if next non-empty line is a new version + peek = i + while peek < len(lines) and lines[peek].strip() == '': + peek += 1 + if peek < len(lines) and re.search(version_pattern, lines[peek]): + break + else: + i += 1 + + versions.append({ + 'version': version, + 'date': date, + 'entries': entries + }) + else: + i += 1 + + return versions + +def generate_debian_changelog(versions, output_path): + """Generate Debian changelog format""" + lines = [] + + for v in versions: + # Parse date + date_obj = datetime.strptime(v['date'], '%Y-%m-%d') + # Debian date format: Mon, 10 Feb 2025 10:00:00 +0800 + deb_date = date_obj.strftime('%a, %d %b %Y 10:00:00 +0800') + + # Header + lines.append(f"flagcx ({v['version']}-1) unstable; urgency=medium") + lines.append("") + + # Entries + for entry in v['entries']: + lines.append(f" * {entry}") + + lines.append("") + # Footer + lines.append(f" -- FlagOS Contributors {deb_date}") + lines.append("") + + with open(output_path, 'w') as f: + f.write('\n'.join(lines)) + + print(f"✓ Generated Debian changelog: {output_path}") + +def generate_rpm_changelog(versions): + """Generate RPM %changelog format""" + lines = [] + + for v in versions: + # Parse date + date_obj = datetime.strptime(v['date'], '%Y-%m-%d') + # RPM date format: Mon Feb 10 2025 + rpm_date = date_obj.strftime('%a %b %d %Y') + + # Header + lines.append(f"* {rpm_date} FlagOS Contributors - {v['version']}-1") + + # Entries + for entry in v['entries']: + lines.append(f"- {entry}") + + lines.append("") + + return '\n'.join(lines) + +def update_rpm_spec(spec_path, changelog_content): + """Update %changelog section in RPM spec file""" + with open(spec_path, 'r') as f: + spec_content = f.read() + + # Replace %changelog section + changelog_pattern = r'%changelog.*$' + new_spec = re.sub(changelog_pattern, f'%changelog\n{changelog_content}', + spec_content, flags=re.DOTALL) + + with open(spec_path, 'w') as f: + f.write(new_spec) + + print(f"✓ Updated RPM spec changelog: {spec_path}") + +def main(): + # Paths + project_root = Path(__file__).parent.parent + changelog_md = project_root / 'docs' / 'CHANGELOG.md' + debian_changelog = project_root / 'packaging' / 'debian' / 'changelog' + rpm_spec = project_root / 'packaging' / 'rpm' / 'specs' / 'flagcx.spec' + + if not changelog_md.exists(): + print(f"Error: {changelog_md} not found") + sys.exit(1) + + # Parse CHANGELOG.md + print(f"Parsing {changelog_md}...") + versions = parse_changelog(changelog_md) + print(f"Found {len(versions)} version(s)") + + if not versions: + print("Error: No versions found in changelog") + sys.exit(1) + + # Generate Debian changelog + generate_debian_changelog(versions, debian_changelog) + + # Generate and update RPM changelog + rpm_changelog = generate_rpm_changelog(versions) + update_rpm_spec(rpm_spec, rpm_changelog) + + print("\n✓ Changelog sync complete!") + print(f"\nTo update changelogs:") + print(f" 1. Edit {changelog_md}") + print(f" 2. Run: {Path(__file__).name}") + +if __name__ == '__main__': + main()