diff --git a/bits_helpers/args.py b/bits_helpers/args.py index 1d3243a..e1e47c5 100644 --- a/bits_helpers/args.py +++ b/bits_helpers/args.py @@ -123,7 +123,9 @@ def doParseArgs(): build_parser.add_argument("--only-deps", dest="onlyDeps", default=False, action="store_true", help="Only build dependencies, not the main package (e.g. for caching)") build_parser.add_argument("--gcc-toolchain", dest="gccToolchain", default=None, metavar="PACKAGE", action="append", - help=("Override gcc toolchain version tag")) + help=("Override gcc toolchain version tag")) + build_parser.add_argument("--generate-rpm", dest="generate_rpm", action="store_true", + help="Generate RPM spec file.") build_docker = build_parser.add_argument_group(title="Build inside a container", description="""\ Builds can be done inside a Docker container, to make it easier to get a diff --git a/bits_helpers/build.py b/bits_helpers/build.py index 636dc21..b806b64 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -18,6 +18,7 @@ from bits_helpers.scm import SCMError from bits_helpers.sync import remote_from_url from bits_helpers.workarea import logged_scm, updateReferenceRepoSpec, checkout_sources +from bits_helpers.dependency import RPMPackageManager try: from bits_helpers.resource_monitor import run_monitor_on_command except: @@ -649,7 +650,8 @@ def runBuildCommand(scheduler, p, specs, args, build_command, cachedTarball, scr warning("Failed to gather build info: %s", exc) dieOnError(err, buildErrMsg.strip()) - + if args.generate_rpm and not spec["package"].startswith("defaults-"): + RPMPackageManager.check_dependencies(os.path.join(args.workDir, args.architecture, spec['package'], spec['version']+"-"+spec['revision'], "etc")) doFinalSync(spec, specs, args, syncHelper) @@ -1008,6 +1010,8 @@ def performPreferCheckWithTempDir(pkg, cmd): from bits_helpers.log import logger scheduler = Scheduler(args.builders, logDelegate=logger, buildStats=args.resources) + order=buildOrder.copy() + while buildOrder: p = buildOrder.pop(0) spec = specs[p] @@ -1314,6 +1318,13 @@ def performPreferCheckWithTempDir(pkg, cmd): "build_requires": " ".join(spec["build_requires"]), "runtime_requires": " ".join(spec["runtime_requires"]), }) + if args.generate_rpm: + from bits_helpers.utilities import getConfigPaths + configPath=getConfigPaths(args.configDir) + for f in configPath: + f = f + "/system-provides.spec" + if os.path.exists(f): + shutil.copyfile(f, scriptDir + "/system-provides.spec") # Define the environment so that it can be passed up to the # actual build script @@ -1362,6 +1373,12 @@ def performPreferCheckWithTempDir(pkg, cmd): # Add the computed track_env environment buildEnvironment += [(key, value) for key, value in spec.get("track_env", {}).items()] + # Build the spec file which will be used to generate rpms. + if not spec['package'].startswith('defaults-') and args.generate_rpm: + specFile = os.path.join(scriptDir, f"{spec['package']}.spec") + shutil.copyfile(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'template.spec'), specFile) + shutil.copyfile(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'spec_template.sh'), os.path.join(scriptDir,spec["package"]+"_spec.sh")) + # In case the --docker options is passed, we setup a docker container which # will perform the actual build. Otherwise build as usual using bash. if args.docker: @@ -1551,5 +1568,6 @@ def performPreferCheckWithTempDir(pkg, cmd): if untrackedFilesDirectories: banner("Untracked files in the following directories resulted in a rebuild of " "the associated package and its dependencies:\n%s\n\nPlease commit or remove them to avoid useless rebuilds.", "\n".join(untrackedFilesDirectories)) + debug("Everything done") diff --git a/bits_helpers/build_template.sh b/bits_helpers/build_template.sh index ce29286..5f3b6d7 100644 --- a/bits_helpers/build_template.sh +++ b/bits_helpers/build_template.sh @@ -292,7 +292,9 @@ cat "$INSTALLROOT/relocate-me.sh" cat "$INSTALLROOT/.original-unrelocated" | xargs -n1 -I{} cp '{}' '{}'.unrelocated fi cd "$WORK_DIR/INSTALLROOT/$PKGHASH" - +if [[ -f "$WORK_DIR/SPECS/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/${PKGNAME}_spec.sh" ]]; then + bash -ex "$WORK_DIR/SPECS/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/${PKGNAME}_spec.sh" +fi # Archive creation HASHPREFIX=`echo $PKGHASH | cut -b1,2` HASH_PATH=$ARCHITECTURE/store/$HASHPREFIX/$PKGHASH @@ -329,6 +331,10 @@ fi # Last package built gets a "latest" mark. ln -snf $PKGVERSION-$PKGREVISION $ARCHITECTURE/$PKGNAME/latest +if [ "$PKGNAME" != defaults-* ] && [ -f "$WORK_DIR/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/etc/profile.d/post-relocate.sh" ]; then + bash -ex "$WORK_DIR/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/etc/profile.d/post-relocate.sh" +fi + # Latest package built for a given devel prefix gets latest-$BUILD_FAMILY if [[ $BUILD_FAMILY ]]; then ln -snf $PKGVERSION-$PKGREVISION $ARCHITECTURE/$PKGNAME/latest-$BUILD_FAMILY diff --git a/bits_helpers/dependency.py b/bits_helpers/dependency.py new file mode 100644 index 0000000..a32358c --- /dev/null +++ b/bits_helpers/dependency.py @@ -0,0 +1,84 @@ +import os +import subprocess +import shutil +import glob +import json +from typing import Dict, List, Optional, Set +from bits_helpers.log import debug, info, banner, warning + + +class RPMPackageManager: + + def load_json(filepath: str) -> Dict[str, List[str]]: + """Load JSON file and return as dictionary.""" + with open(filepath, 'r') as f: + return json.load(f) + + def build_provides_set(provides: Dict[str, List[str]]) -> Set[str]: + """Build a set of all provided dependencies.""" + all_provides = set() + for package, dependencies in provides.items(): + all_provides.update(dependencies) + return all_provides + + def check_dependencies(file_path: str) -> Dict: + """ + Check if all required dependencies are satisfied. + Returns a dictionary with results. + """ + requires_path = os.path.join(file_path, "requires.json") + provides_path = os.path.join(file_path, "provides.json") + + requires = RPMPackageManager.load_json(requires_path) + provides = RPMPackageManager.load_json(provides_path) + provides_set = RPMPackageManager.build_provides_set(provides) + + results = { + 'satisfied': [], + 'missing': [], + 'packages_with_missing': {} + } + + for package, dependencies in requires.items(): + package_missing = [] + debug(f"Checking dependencies for package: {package}") + + for dep in dependencies: + if dep.startswith('rpmlib('): + continue + + if dep not in provides_set: + debug(f" [MISSING] {dep}") + package_missing.append(dep) + results['missing'].append({ + 'package': package, + 'dependency': dep + }) + else: + debug(f" [OK] {dep}") + results['satisfied'].append({ + 'package': package, + 'dependency': dep + }) + + if package_missing: + results['packages_with_missing'][package] = package_missing + + if results['missing']: + warning(f"Dependency Check Failed: {len(results['missing'])} missing dependencies found.") + warning("\n" + "="*60) + warning("❌ MISSING DEPENDENCIES") + warning("="*60) + for i, dep in enumerate(results['missing'], 1): + if isinstance(dep, dict): + package = dep.get('package', 'Unknown') + dependency = dep.get('dependency', 'Unknown') + warning(f"\n{i}. 📦 {package}") + warning(f" └─ Requires: {dependency}") + else: + warning(f"\n{i}. ❌ {dep}") + warning("="*60 + "\n") + else: + banner("Dependency Check Passed: All dependencies satisfied.") + + return results \ No newline at end of file diff --git a/bits_helpers/spec_template.sh b/bits_helpers/spec_template.sh new file mode 100644 index 0000000..948677b --- /dev/null +++ b/bits_helpers/spec_template.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# We don't build RPMs if we have requires.json and provides.json. We can just proceed with checking dependencies. +if [ -f "$INSTALLROOT/etc/requires.json" ] && [ -f "$INSTALLROOT/etc/provides.json" ]; then + exit 0 +fi + +# Build system-provides RPM and extract provides.json and store in $WORK_DIR/provides.json from where it will be copied by into each package provides.json +if [ ! -f "$WORK_DIR/rpmbuild/RPMS/$(uname -m)/system-provides-1-1.$(uname -m).rpm" ]; then + mkdir -p "$WORK_DIR/rpmbuild"/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} + cp $WORK_DIR/SPECS/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/system-provides.spec "$WORK_DIR/rpmbuild/SPECS/system-provides.spec" + rpmbuild -bb \ + --define "_topdir $WORK_DIR/rpmbuild" \ + --define "_buildarch $(uname -m)" \ + "$WORK_DIR/rpmbuild/SPECS/system-provides.spec" + + rpm -qp --queryformat "%{NAME}\n[%{PROVIDES}\n]" "$WORK_DIR/rpmbuild/RPMS/$(uname -m)/system-provides-1-1.$(uname -m).rpm" | \ + jq -R -s ' + split("\n") + | map(select(length > 0)) + | { (.[0]): .[1:] } + ' > "$WORK_DIR/provides.json" +fi + +if [ "$PKGNAME" != defaults-* ] && [ -f "$WORK_DIR/SPECS/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/${PKGNAME}.spec" ]; then + mkdir -p "$WORK_DIR/rpmbuild"/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} + mkdir -p "$WORK_DIR/rpmbuild/BUILDROOT/${PKGNAME}" + chmod -R u+w "$WORK_DIR/rpmbuild" + source "$WORK_DIR/$ARCHITECTURE/rpm/latest/etc/profile.d/init.sh" || true + cp "$WORK_DIR/SPECS/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/${PKGNAME}.spec" "$WORK_DIR/rpmbuild/SPECS/" + requires=() + for f in $REQUIRES; do + if [[ "$f" == "defaults-"* ]]; then + continue + fi + F=${f^^} + F=${F//-/_} + hash="${F}_HASH" + ver="${F}_VERSION" + rev="${F}_REVISION" + requires+=("${f}_${!ver}_${!rev}_${!hash}") + done + + if [ ${#requires[@]} -eq 0 ]; then + requires_str="%{nil}" + else + printf -v requires_str '%s ' "${requires[@]}" + requires_str="${requires_str% }" + fi + + rpmbuild -bb \ + --define "name ${PKGNAME}_${PKGVERSION}_${PKGREVISION}_${PKGHASH}" \ + --define "pkgname ${PKGNAME}" \ + --define "arch $(uname -m)" \ + --define "installroot $INSTALLROOT" \ + --define "requires $requires_str" \ + --define "_topdir $WORK_DIR/rpmbuild" \ + --define "buildroot $WORK_DIR/rpmbuild/BUILDROOT/${PKGNAME}" \ + "$WORK_DIR/rpmbuild/SPECS/${PKGNAME}.spec" + + rpm -qp --queryformat "%{NAME}\n[%{REQUIRES}\n]" "$WORK_DIR/rpmbuild/RPMS/$(uname -m)/${PKGNAME}_${PKGVERSION}_${PKGREVISION}_${PKGHASH}-1-1.$(uname -m).rpm" | \ + jq -R -s ' + split("\n") + | map(select(length > 0)) + | { (.[0]): .[1:] } + ' > "$INSTALLROOT/etc/requires.json" + + rpm -qp --queryformat "%{NAME}\n[%{PROVIDES}\n]" "$WORK_DIR/rpmbuild/RPMS/$(uname -m)/${PKGNAME}_${PKGVERSION}_${PKGREVISION}_${PKGHASH}-1-1.$(uname -m).rpm" | \ + jq -R -s ' + split("\n") + | map(select(length > 0)) + | { (.[0]): .[1:] } + ' > "$INSTALLROOT/etc/provides.json" + + # Collect all provides.json for all the package mentioned in $REQUIRES and also system-provides.json and create a single provides.json file from where we can see if each + # REQUIRES is satisfied or not. + provides_files=("$INSTALLROOT/etc/provides.json") + provides_files+=("$WORK_DIR/provides.json") + + for f in $REQUIRES; do + if [[ "$f" == "defaults-"* ]]; then + continue + fi + F=${f^^} + F=${F//-/_} + hash="${F}_HASH" + ver="${F}_VERSION" + rev="${F}_REVISION" + + provides_file="${WORK_DIR}/$ARCHITECTURE/$f/${!ver}-${!rev}/etc/provides.json" + + if [ -f "$provides_file" ]; then + provides_files+=("$provides_file") + fi + done + + # Merge all provides.json files into a single file and store it in $INSTALLROOT/etc/provides.json + jq -s 'reduce .[] as $item ({}; . * $item)' "${provides_files[@]}" > "$INSTALLROOT/provides.json" + rm $INSTALLROOT/etc/provides.json + mv $INSTALLROOT/provides.json $INSTALLROOT/etc/provides.json +fi \ No newline at end of file diff --git a/bits_helpers/sync.py b/bits_helpers/sync.py index 2e0c263..eb8c69a 100644 --- a/bits_helpers/sync.py +++ b/bits_helpers/sync.py @@ -713,8 +713,7 @@ def _upload_single_symlink(link_key, hash_path): self.s3.put_object(Bucket=self.writeStore, Key=link_key, Body=os.fsencode(hash_path), - ACL="public-read", - WebsiteRedirectLocation=hash_path) + WebsiteRedirectLocation="/"+hash_path) return link_key with ThreadPoolExecutor(max_workers=max_workers) as executor: diff --git a/bits_helpers/template.spec b/bits_helpers/template.spec new file mode 100644 index 0000000..3c00490 --- /dev/null +++ b/bits_helpers/template.spec @@ -0,0 +1,31 @@ +%define __os_install_post %{nil} +%define __spec_install_post %{nil} +%define __spec_install_pre %{___build_pre} +%define _empty_manifest_terminate_build 0 +%define _use_internal_dependency_generator 0 +%define _source_payload w9.gzdio +%define _binary_payload w9.gzdio + +Name: %{name} +Version: 1 +Release: 1 +Summary: Package %{name} built using bits. +License: Public Domain +BuildArch: %{arch} +Vendor: CERN + +%if "%{?requires}" != "" +Requires: %{requires} +%endif + +%description + +%prep + +%build + +%install +cp -a %{installroot}/* %{buildroot} + +%files +/* \ No newline at end of file diff --git a/tests/test_build.py b/tests/test_build.py index 5f30cd3..63e3d42 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -299,6 +299,7 @@ def test_coverDoBuild(self, mock_debug, mock_listdir, mock_warning, mock_git_git resources=None, resourceMonitoring=False, makeflow=False, + generate_rpm=False, ) def mkcall(args):