IDEA 25.11.0 Release

CHANGELOG.md

@@ -4,6 +4,35 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Calendar Versioning](https://calver.org/).
 
+## [25.11.0] - 2025-11-TBD
+
+**Upgrade Instructions:**
+* It's recommended to perform a full cluster upgrade as Base AMIs and other settings have been updated
+```bash
+./idea-admin.sh upgrade-cluster --aws-region $IDEA_AWS_REGION --cluster-name $IDEA_CLUSTER_NAME
+```
+([Upgrade Documentation](https://docs.idea-hpc.com/first-time-users/cluster-operations/update-idea-cluster/upgrade-cluster))
+
+### **🔧 Improvements**
+* **Dependencies**: Updated AWS CDK (2.1031.2), Python (3.13.9), Node (22.21.1), and all Python packages
+* **NVIDIA GPU Drivers**: Updated driver versions
+  * LTSB from `470.239.06` to `470.256.02`
+  * Production from `580.65.06` to `580.105.08`
+* **AMI Updates**: Refreshed base and software stack AMI IDs across all regions
+  * Amazon Linux 2023 updated to kernel 6.12
+* **HPC Job Provisioning**: Added configurable job reconciler interval (`scheduler.job_provisioning.job_reconciler_interval_seconds`, default: 60 seconds)
+* **FSx Lustre Support**: Added FSx Lustre support for modern kernels
+
+### **🐛 Bug Fixes**
+* **HPC Job Monitor**: Fixed job monitor thread crash from unhandled exceptions in finished job processor
+* **HPC Job Provisioning**: Fixed jobs getting stuck when PBS hooks don't fire immediately
+  * Added job reconciler fallback to catch missed jobs
+* **EC2 Instance Type Cache**: Fixed cache overflow preventing job provisioning for new instance types
+  * Enhanced with larger size and on-demand fetching
+* **Lustre Client Repo**: Fixed repo family on Ubuntu
+* **CDK Template**: Removed invalid TTL on Alias record
+* **GovCloud VDC Deployment**: Fixed EventBridge scheduler rule deployment failure in AWS GovCloud
+
 ## [25.09.1] - 2025-09-22
 
 **Upgrade Instructions:**

IDEA_VERSION.txt

@@ -1 +1 @@
-25.09.1
+25.11.0

deployment/ecr/idea-administrator/Dockerfile

@@ -1,11 +1,11 @@
 # Build arguments for software versions
-ARG PYTHON_VERSION=3.13.7
+ARG PYTHON_VERSION=3.13.9
 
 FROM public.ecr.aws/docker/library/python:${PYTHON_VERSION}-slim
 ARG TARGETARCH
-ARG NODE_VERSION=22.18.0
+ARG NODE_VERSION=22.21.1
 ARG NVM_VERSION=0.40.3
-ARG AWS_CDK_VERSION=2.1026.0
+ARG AWS_CDK_VERSION=2.1031.2
 
 ENV OS_ARCH="${TARGETARCH:-amd64}"
 

idea-admin-windows.ps1

@@ -38,7 +38,7 @@ function Verify-Command($type,$message,$command) {
 $IDEADevMode = if ($Env:IDEA_DEV_MODE) {$Env:IDEA_DEV_MODE} else {""}
 $VirtualEnv = if ($Env:VIRTUAL_ENV) {$Env:VIRTUAL_ENV} else {""}
 $ScriptDir = $PSScriptRoot
-$IDEARevision = if ($Env:IDEA_REVISION) {$Env:IDEA_REVISION} else {"v25.09.1"}
+$IDEARevision = if ($Env:IDEA_REVISION) {$Env:IDEA_REVISION} else {"v25.11.0"}
 $IDEADockerRepo = "public.ecr.aws/s5o2b4m0"
 $DocumentationError = "https://docs.idea-hpc.com"
 $AWSProfile = if ($Env:AWS_PROFILE) {$Env:AWS_PROFILE} else {"default"}

idea-admin.sh

@@ -28,7 +28,7 @@
 # * IDEA_DEV_MODE - Set to "true" if you are working with IDEA sources
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-IDEA_REVISION=${IDEA_REVISION:-"v25.09.1"}
+IDEA_REVISION=${IDEA_REVISION:-"v25.11.0"}
 IDEA_DOCKER_REPO=${IDEA_DOCKER_REPO:-"public.ecr.aws/s5o2b4m0/idea-administrator"}
 IDEA_ECR_CREDS_RESET=${IDEA_ECR_CREDS_RESET:-"true"}
 IDEA_ADMIN_AWS_CREDENTIAL_PROVIDER=${IDEA_ADMIN_AWS_CREDENTIAL_PROVIDER:=""}

requirements/dev.txt

@@ -1,133 +1,133 @@
-aiofiles==24.1.0
-alembic==1.16.5
+aiofiles==25.1.0
+alembic==1.17.2
 annotated-types==0.7.0
-arrow==1.3.0
-astroid==3.3.11
-attrs==25.3.0
+arrow==1.4.0
+astroid==4.0.2
+attrs==25.4.0
 aws-cdk-asset-awscli-v1==2.2.242
 aws-cdk-asset-node-proxy-agent-v6==2.1.0
-aws-cdk-cloud-assembly-schema==48.6.0
-aws-cdk-lib==2.213.0
+aws-cdk-cloud-assembly-schema==48.18.0
+aws-cdk-lib==2.224.0
 babel==2.17.0
-backrefs==5.9
+backrefs==6.0.1
 banal==1.0.6
 blinker==1.9.0
-boto3==1.40.21
-botocore==1.40.21
+boto3==1.40.74
+botocore==1.40.74
 build==1.3.0
 cacheout==0.16.0
-cachetools==6.2.0
-cattrs==25.1.1
-cdk-nag==2.37.13
-certifi==2025.8.3
-cffi==1.17.1
+cachetools==6.2.2
+cattrs==25.3.0
+cdk-nag==2.37.55
+certifi==2025.11.12
+cffi==2.0.0
 cfn-flip==1.3.0
-charset-normalizer==3.4.3
-click==8.2.1
+charset-normalizer==3.4.4
+click==8.3.0
 colorama==0.4.6
 colored==2.3.1
-constructs==10.4.2
-coverage[toml]==7.10.6
-cryptography==45.0.7
+constructs==10.4.3
+coverage[toml]==7.11.3
+cryptography==46.0.3
 dataset==1.6.2
 dill==0.4.0
 events==0.5
 fastcounter==1.1.0
 ghp-import==2.1.0
 greenlet==3.2.4
 html5tagger==1.3.0
-httptools==0.6.4
-idna==3.10
+httptools==0.7.1
+idna==3.11
 importlib-resources==6.5.2
-iniconfig==2.1.0
-invoke==2.2.0
+iniconfig==2.3.0
+invoke==2.2.1
 ipaddress==1.0.23
-isort==6.0.1
+isort==7.0.0
 jinja2==3.1.6
 jmespath==1.0.1
-jsii==1.113.0
+jsii==1.119.0
 ldappool==3.0.0
 mako==1.3.10
-markdown==3.8.2
+markdown==3.10
 markdown-it-py==4.0.0
-markupsafe==3.0.2
+markupsafe==3.0.3
 mccabe==0.7.0
 mdurl==0.1.2
 memory-profiler==0.61.0
 mergedeep==1.3.4
 mkdocs==1.6.1
 mkdocs-get-deps==0.2.0
-mkdocs-material==9.6.18
+mkdocs-material==9.7.0
 mkdocs-material-extensions==1.3.1
-multidict==6.6.4
-mypy==1.17.1
+multidict==6.7.0
+mypy==1.18.2
 mypy-extensions==1.1.0
 openapi-pydantic==0.5.1
 opensearch-py==3.0.0
-orjson==3.11.3
+orjson==3.11.4
 packaging==25.0
 paginate==0.5.7
 passlib==1.7.4
 pathspec==0.12.1
-pip-tools==7.5.0
-platformdirs==4.4.0
+pip-tools==7.5.2
+platformdirs==4.5.0
 pluggy==1.6.0
-prettytable==3.16.0
-prometheus-client==0.22.1
+prettytable==3.17.0
+prometheus-client==0.23.1
 prompt-toolkit==3.0.52
-psutil==7.0.0
+psutil==7.1.3
 publication==0.0.3
 pyasn1==0.6.1
 pyasn1-modules==0.4.2
-pycparser==2.22
-pydantic==2.11.7
-pydantic-core==2.33.2
+pycparser==2.23
+pydantic==2.12.4
+pydantic-core==2.41.5
 pyfiglet==1.0.4
 pygments==2.19.2
 pyhocon==0.3.61
 pyjwt==2.10.1
-pylint==3.3.8
-pymdown-extensions==10.16.1
-pyparsing==3.2.3
+pylint==4.0.3
+pymdown-extensions==10.17.1
+pyparsing==3.2.5
 pyproject-hooks==1.2.0
-pytest==8.4.1
-pytest-cov==6.2.1
-pytest-mock==3.14.1
+pytest==9.0.1
+pytest-cov==7.0.0
+pytest-mock==3.15.1
 python-dateutil==2.9.0.post0
 python-dynamodb-lock==0.9.1
-python-ldap==3.4.4
+python-ldap==3.4.5
 pytz==2025.2
-pyyaml==6.0.2
+pyyaml==6.0.3
 pyyaml-env-tag==1.1
 questionary==2.1.1
 random-password-generator==2.2.0
 requests==2.32.5
 requests-aws4auth==1.3.1
 requests-unixsocket==0.4.1
-rich==14.1.0
-s3transfer==0.13.1
+rich==14.2.0
+s3transfer==0.14.0
 sanic==25.3.0
 sanic-routing==23.12.0
 semver==3.0.4
 sh==2.2.2
 shortuuid==1.0.13
 six==1.17.0
 sqlalchemy==1.4.54
-supervisor @ git+https://github.com/Supervisor/supervisor.git@main
+supervisor==4.3.0
 tomlkit==0.13.3
 tracerite==1.1.3
 troposphere==4.9.4
-typeguard==2.13.3
-types-python-dateutil==2.9.0.20250822
+typeguard==4.2.1
 typing-extensions==4.15.0
-typing-inspection==0.4.1
+typing-inspection==0.4.2
+tzdata==2025.2
 tzlocal==5.3.1
 ujson==5.11.0
 urllib3==2.5.0
-uvloop==0.21.0
+uvloop==0.22.1
 validators==0.35.0
 watchdog==6.0.0
-wcwidth==0.2.13
+wcwidth==0.2.14
 websockets==15.0.1
 wheel==0.45.1
 

requirements/doc.txt

@@ -1,27 +1,27 @@
 babel==2.17.0
-backrefs==5.9
-certifi==2025.8.3
-charset-normalizer==3.4.3
-click==8.2.1
+backrefs==6.0.1
+certifi==2025.11.12
+charset-normalizer==3.4.4
+click==8.3.0
 colorama==0.4.6
 ghp-import==2.1.0
-idna==3.10
+idna==3.11
 jinja2==3.1.6
-markdown==3.8.2
-markupsafe==3.0.2
+markdown==3.10
+markupsafe==3.0.3
 mergedeep==1.3.4
 mkdocs==1.6.1
 mkdocs-get-deps==0.2.0
-mkdocs-material==9.6.18
+mkdocs-material==9.7.0
 mkdocs-material-extensions==1.3.1
 packaging==25.0
 paginate==0.5.7
 pathspec==0.12.1
-platformdirs==4.4.0
+platformdirs==4.5.0
 pygments==2.19.2
-pymdown-extensions==10.16.1
+pymdown-extensions==10.17.1
 python-dateutil==2.9.0.post0
-pyyaml==6.0.2
+pyyaml==6.0.3
 pyyaml-env-tag==1.1
 requests==2.32.5
 six==1.17.0