diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 3e71b0f..a142c21 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -85,6 +85,43 @@ provider "registry.opentofu.org/hashicorp/kubernetes" { ] } +provider "registry.opentofu.org/hashicorp/random" { + version = "3.9.0" + constraints = "~> 3.0" + hashes = [ + "h1:8EQU5KSxezcjo/phRSe69rDOI0lk4pSaggj7FsskYp8=", + "h1:Lw9im2VBBJQ3RyAbHPQ0rcvcmmcZWm3x+kIOpN+Tv9s=", + "h1:U8KXqGCoNI9/guYbTvzgdtVk3fRthoG0UXwm1JoEpIs=", + "h1:YXaVd4p6qXPPVaxIBaIDNXmBwT02ZqDn0qD+tYpw8sA=", + "h1:cOpc03fphEt/G9Rfc4jLL/fW0D7tgvlXqiDKPF4vuww=", + "h1:g09RR7T1xWkeGrZwWvWMT9ncJrFGr1k3CBD585UmO7w=", + "h1:gGDdPPibmw2EWROx+sh1RGLjR5+nPwZyrf6/N9jXfeM=", + "h1:haE7/nXCOhXKP4oXeEnER3t5CaVQWqujz4nBnpeTUv4=", + "h1:ieSVpfZS2lKuMr05ph0QsOVpCzg7uk3cgKBaXR+Ikug=", + "h1:ig2s1IS9IzehorRjvVAnKIsUUj8fkgyxct1L/kswcc4=", + "h1:j3lS+ZEERFnoab8t1ppDrScGVP/cgWbzlCrEYKTCXYw=", + "h1:lxezrKmOiQIySHAM+os8qLVq7hqufDr8h3Hpzvsk+78=", + "h1:lzRqBJAG+NETxHbEZUJ/YP3RMEjZBinTX7VmgH3lw60=", + "h1:tdSNWK5ApqUsgbdYieyeYLTu6nIZUV3hR1oFqUfAuGo=", + "h1:xedet8yH/zI2CfdxsGlK0nlFWc/Bp61yrWsEa3fHB8g=", + "zh:03f1114cc20b8913523735ab76e0f0a2b16ce13c92923a53304bf85f07fc0dbc", + "zh:105b678ee72322a3067f105d7e05e940f6143238f377f6e87ff4ec909246ac2a", + "zh:55f3bbf13ea18cbace61a706566a80f25f33fe2b1780b6f3d7b582af2a05b6d2", + "zh:63adf996db48f082f7a6351eb485e219cd88795fc71e6ec60a837263ab0d2cb1", + "zh:7e99550738a4e3cc68b8a467714b0d69371025fe95e3326d5323d026d55653e9", + "zh:8342b54af3a18a37e075eeae61be57f4de2ba71b35d95c5075d402dd2c1f289d", + "zh:83ee18e32ac9dd5fc91298554b7c4cfa4c3a1db50f4c797945637cc93c0844ae", + "zh:993ecc0adbf6bd535a59fbc9b735d8c33950e6f6eb5e621d750da9b71d65d80a", + "zh:ad722bc59d4edbf1415e827fc007c0efe6e0e9462d5568bae20b34be1058a261", + "zh:ae9448e1f87b2f9a6c5197a0e9862162ec6b137cb3a3835e11522995d8939e7c", + "zh:bc9cdd3aac784f759125c6627f6f6416e8726a1c184eb9cf3e55b9edbc94c627", + "zh:c8e35b89572ba1c40a9b20022e033a3395fb8d42e7604d50c900f193ba10382e", + "zh:e2deaa8a9975ef81d9f62baed12c41286918b0a10908e0e031f13f69a3b730a1", + "zh:ee39707557210a0ab1098aa357d2cdfe502e5a312d0dbdffb09d08facc4d3fc5", + "zh:f81afe4eb63e8aa9e0ea71be6c990f0dc69cb360e7191c0742a991f4a5081b64", + ] +} + provider "registry.opentofu.org/hashicorp/tls" { version = "4.2.1" constraints = "~> 4.0" @@ -102,3 +139,34 @@ provider "registry.opentofu.org/hashicorp/tls" { "zh:f92baceb82d3a1e5b6a34a29c605b54cae8c6b09ea1fffb0af4d036337036a8f", ] } + +provider "registry.opentofu.org/loafoe/htpasswd" { + version = "1.6.0" + constraints = "~> 1.2" + hashes = [ + "h1:29RFQ+IAhh5vRpfYSpJ3drv8Z3PgP2i/QbHE8N4c21U=", + "h1:6wfVdoRCwu6lAHSMACA4rzMXitt/dbvDyxHvop4yddo=", + "h1:KydcstenXBOHUry9D9Ze0zjTcMDhby91f5vnzCNRcv4=", + "h1:PGg92r9ijapscD5+clsw24ADe6Fzl58JWiilaosxTmY=", + "h1:V5Kl96JdWfTAhnb1foASBsUlFWlKvtbZWK6a5CS38wA=", + "h1:iYtneJuaYYXSCjOqSUX6GZDo2gyG4ynNU8Tb0/Tf4S0=", + "h1:jr5xoW1F0fsR5TqwQDnmezT8GgLkipqkDfG3IqMcG5o=", + "h1:kTF4QwxdWU9BxMln+helXxtg0g9xEhwp67O+LW+bJU4=", + "h1:lQUxFpImg0lLntjtQeuYUxJbqeNjexGij4/PTICiUEw=", + "h1:wg1XKvfXEQduRklptPo/WroTh3fYmpb7sIx8ZO/ymog=", + "h1:wtjm2YTiR5EYigIuE6zN5kIWpzLrdEW5bLBSvVtU8W8=", + "h1:xWXUD45dAI6o2TB2AGZJppjG/674FPSIVQFGXSXRops=", + "zh:0050ca190d3f905668ec6c2bcaef44b251357cb1a1fa46c72059d7ee9a3dd62c", + "zh:014cd65e585d9f38e55250d9f52a933cd84aada3e763f27fe85e59244d8260fa", + "zh:03c4b24da1dd85b2c33f5649dd8c6f509e76ab2e8c023f01576354c74cda3c56", + "zh:1e2eeb9f7e88335503ed469218002ba6a477fd538ab0b23ac4d02d834eaecdc9", + "zh:386b173176ff2d04a038413edd4842bec0f4ccede63eb90e4603ec369c033363", + "zh:65bce4aa385a20b9294d50b7e17b5feff4d447e29dbc8e8a3e8823fc10b7de48", + "zh:b534794eeb7909890fca59d9a08dec1168e65d867f12f3187fdbbc8c710a0af8", + "zh:cb399ec66e2490e10ddeca0c5678b265c8826f7b0cdfb7259d2de85371e41358", + "zh:cfb1a4c703ca28ca13f79313ec5cd70ed9f310e9f7e109a955fb112a78d0ba01", + "zh:cff1c25002fc6d2e76e126f5bed678d9ad4902d08730011ce1e1f4fe37cc3526", + "zh:e85e8ca2cb1a4f512199c16907a08ff9753a067ab109cb8137466069281808aa", + "zh:eb2e20a2ee9430c2cd8240eaab6d2c61737c67478518970c6e3d9d332b6a73cc", + ] +} diff --git a/acm.tf b/acm.tf index b6c501b..e0e599a 100644 --- a/acm.tf +++ b/acm.tf @@ -5,8 +5,16 @@ # steps are required; tofu apply blocks until the cert reaches ISSUED status. # ───────────────────────────────────────────────────────────────────────────── +locals { + # Determine if any service needs a certificate (for Route 53 zone data source) + needs_route53_zone = ( + var.argocd_hostname != null || + var.skypilot_api_server_hostname != null + ) +} + data "aws_route53_zone" "main" { - count = var.argocd_hostname != null ? 1 : 0 + count = local.needs_route53_zone ? 1 : 0 name = var.route53_zone_name private_zone = false } @@ -56,3 +64,52 @@ resource "aws_route53_record" "argocd" { ttl = 60 records = [module.eks_addons.argocd_lb_hostname] } + +# ───────────────────────────────────────────────────────────────────────────── +# SkyPilot API Server Certificate — auto-created when hostname is configured +# ───────────────────────────────────────────────────────────────────────────── + +resource "aws_acm_certificate" "skypilot" { + count = var.skypilot_api_server_hostname != null ? 1 : 0 + domain_name = var.skypilot_api_server_hostname + validation_method = "DNS" + + lifecycle { + create_before_destroy = true + } + + tags = var.tags +} + +resource "aws_route53_record" "skypilot_cert_validation" { + for_each = var.skypilot_api_server_hostname != null ? { + for dvo in aws_acm_certificate.skypilot[0].domain_validation_options : dvo.domain_name => { + name = dvo.resource_record_name + record = dvo.resource_record_value + type = dvo.resource_record_type + } + } : {} + + allow_overwrite = true + name = each.value.name + records = [each.value.record] + ttl = 60 + type = each.value.type + zone_id = data.aws_route53_zone.main[0].zone_id +} + +resource "aws_acm_certificate_validation" "skypilot" { + count = var.skypilot_api_server_hostname != null ? 1 : 0 + certificate_arn = aws_acm_certificate.skypilot[0].arn + validation_record_fqdns = [for r in aws_route53_record.skypilot_cert_validation : r.fqdn] +} + +# Route 53 record — points the SkyPilot hostname at the ALB created by AWS LBC. +resource "aws_route53_record" "skypilot" { + count = var.skypilot_api_server_hostname != null ? 1 : 0 + zone_id = data.aws_route53_zone.main[0].zone_id + name = var.skypilot_api_server_hostname + type = "CNAME" + ttl = 60 + records = [module.eks_addons.skypilot_api_server_lb_hostname] +} diff --git a/main.tf b/main.tf index f5bc7f8..db4c5f3 100644 --- a/main.tf +++ b/main.tf @@ -22,6 +22,14 @@ terraform { source = "hashicorp/kubernetes" version = "~> 2.0" } + htpasswd = { + source = "loafoe/htpasswd" + version = "~> 1.2" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } } } @@ -207,6 +215,14 @@ module "eks_addons" { tailscale_oauth_client_secret = var.tailscale_oauth_client_secret tailscale_chart_version = var.tailscale_chart_version + # SkyPilot API Server + skypilot_api_server_enabled = var.skypilot_api_server_enabled + skypilot_api_server_web_password = var.skypilot_api_server_web_password + skypilot_api_server_hostname = var.skypilot_api_server_hostname + skypilot_api_server_certificate_arn = var.skypilot_api_server_hostname != null ? aws_acm_certificate_validation.skypilot[0].certificate_arn : null + skypilot_api_server_chart_version = var.skypilot_api_server_chart_version + skypilot_api_server_resources = var.skypilot_api_server_resources + tags = var.tags depends_on = [module.eks] diff --git a/modules/aws/eks-addons/outputs.tf b/modules/aws/eks-addons/outputs.tf index 3d914a5..e216948 100644 --- a/modules/aws/eks-addons/outputs.tf +++ b/modules/aws/eks-addons/outputs.tf @@ -21,3 +21,18 @@ output "argocd_lb_hostname" { description = "ALB hostname assigned to the ArgoCD ingress by AWS LBC. Null when ArgoCD is not exposed." value = local.expose_argocd ? data.kubernetes_ingress_v1.argocd[0].status[0].load_balancer[0].ingress[0].hostname : null } + +output "skypilot_api_server_installed" { + description = "Whether SkyPilot API server was installed." + value = local.install_skypilot_api_server +} + +output "skypilot_api_server_namespace" { + description = "Kubernetes namespace where SkyPilot API server is deployed." + value = local.install_skypilot_api_server ? var.skypilot_api_server_namespace : null +} + +output "skypilot_api_server_lb_hostname" { + description = "ALB hostname assigned to the SkyPilot API server ingress by AWS LBC. Null when SkyPilot is not exposed." + value = local.expose_skypilot_api_server ? data.kubernetes_ingress_v1.skypilot_api_server[0].status[0].load_balancer[0].ingress[0].hostname : null +} diff --git a/modules/aws/eks-addons/skypilot-api-server.tf b/modules/aws/eks-addons/skypilot-api-server.tf new file mode 100644 index 0000000..1e93ff4 --- /dev/null +++ b/modules/aws/eks-addons/skypilot-api-server.tf @@ -0,0 +1,177 @@ +# ───────────────────────────────────────────────────────────────────────────── +# SkyPilot API Server — Helm deployment for remote SkyPilot job submission +# Docs: https://docs.skypilot.co/en/latest/reference/api-server/api-server-admin-deploy.html +# ───────────────────────────────────────────────────────────────────────────── + +locals { + install_skypilot_api_server = var.skypilot_api_server_enabled + expose_skypilot_api_server = ( + local.install_skypilot_api_server && + var.skypilot_api_server_hostname != null + ) +} + +# Generate basic auth credentials for SkyPilot API server +resource "htpasswd_password" "skypilot_basic_auth" { + count = local.install_skypilot_api_server ? 1 : 0 + + password = var.skypilot_api_server_web_password + # Use a random salt + salt = random_password.skypilot_auth_salt[0].result +} + +resource "random_password" "skypilot_auth_salt" { + count = local.install_skypilot_api_server ? 1 : 0 + + length = 8 + special = true + override_special = "./" +} + +resource "helm_release" "skypilot_api_server" { + count = local.install_skypilot_api_server ? 1 : 0 + + name = var.skypilot_api_server_release_name + repository = "https://helm.skypilot.co" + chart = "skypilot-nightly" + version = var.skypilot_api_server_chart_version + namespace = var.skypilot_api_server_namespace + create_namespace = true + + wait = true + wait_for_jobs = true + timeout = 600 + + set { + name = "ingress.authCredentials" + value = "${var.skypilot_api_server_web_username}:${htpasswd_password.skypilot_basic_auth[0].apr1}" + } + + values = [yamlencode({ + # Use system nodes for API server components + apiService = { + nodeSelector = { "node-role" = "system" } + resources = { + requests = { + cpu = var.skypilot_api_server_resources.requests.cpu + memory = var.skypilot_api_server_resources.requests.memory + } + limits = { + cpu = var.skypilot_api_server_resources.limits.cpu + memory = var.skypilot_api_server_resources.limits.memory + } + } + } + + # Storage configuration for state persistence + storage = { + enabled = true + storageClassName = "gp3" + size = "20Gi" + accessMode = "ReadWriteOnce" + } + + # Ingress configuration + ingress = { + enabled = local.expose_skypilot_api_server + path = "/" + } + + # Disable bundled ingress-nginx if using custom/alb + ingress-nginx = { + enabled = false + } + + # AWS credentials (optional) - uses IRSA if enabled + awsCredentials = { + enabled = var.skypilot_api_server_aws_credentials_enabled + } + + # Enable using the hosting EKS cluster for tasks + kubernetesCredentials = { + useApiServerCluster = true + } + })] + + depends_on = [ + aws_eks_addon.coredns, + aws_eks_addon.ebs_csi, + helm_release.karpenter, + ] +} + +# ───────────────────────────────────────────────────────────────────────────── +# SkyPilot API Server Ingress — ALB with WAF +# Created only when skypilot_api_server_hostname is set. +# ───────────────────────────────────────────────────────────────────────────── + +resource "kubectl_manifest" "skypilot_api_server_ingress" { + count = local.expose_skypilot_api_server ? 1 : 0 + + wait = true + wait_for { + field { + key = "status.loadBalancer.ingress.[0].hostname" + value = ".+" + value_type = "regex" + } + } + + yaml_body = yamlencode({ + apiVersion = "networking.k8s.io/v1" + kind = "Ingress" + metadata = { + name = "${var.skypilot_api_server_release_name}-ingress" + namespace = var.skypilot_api_server_namespace + annotations = { + "kubernetes.io/ingress.class" = "alb" + "alb.ingress.kubernetes.io/scheme" = "internet-facing" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/ip-address-type" = "dualstack" + "alb.ingress.kubernetes.io/backend-protocol" = "HTTP" + "alb.ingress.kubernetes.io/listen-ports" = jsonencode([{ HTTPS = 443 }]) + "alb.ingress.kubernetes.io/ssl-redirect" = "443" + "alb.ingress.kubernetes.io/ssl-policy" = "ELBSecurityPolicy-TLS13-1-2-2021-06" + "alb.ingress.kubernetes.io/certificate-arn" = var.skypilot_api_server_certificate_arn + "alb.ingress.kubernetes.io/wafv2-acl-arn" = var.waf_web_acl_arn + "alb.ingress.kubernetes.io/healthcheck-path" = "/api/health" + "alb.ingress.kubernetes.io/healthcheck-protocol" = "HTTP" + "alb.ingress.kubernetes.io/success-codes" = "200" + # Basic auth is handled by the SkyPilot API server itself + } + } + spec = { + rules = [{ + host = var.skypilot_api_server_hostname + http = { + paths = [{ + path = "/" + pathType = "Prefix" + backend = { + service = { + name = "${var.skypilot_api_server_release_name}-api" + port = { number = 80 } + } + } + }] + } + }] + } + }) + + depends_on = [ + helm_release.skypilot_api_server, + helm_release.aws_lbc, + ] +} + +data "kubernetes_ingress_v1" "skypilot_api_server" { + count = local.expose_skypilot_api_server ? 1 : 0 + + metadata { + name = "${var.skypilot_api_server_release_name}-ingress" + namespace = var.skypilot_api_server_namespace + } + + depends_on = [kubectl_manifest.skypilot_api_server_ingress] +} diff --git a/modules/aws/eks-addons/variables.tf b/modules/aws/eks-addons/variables.tf index ea4c367..2e6d273 100644 --- a/modules/aws/eks-addons/variables.tf +++ b/modules/aws/eks-addons/variables.tf @@ -190,3 +190,84 @@ variable "tailscale_chart_version" { default = "1.78.3" } +# ── SkyPilot API Server Configuration ───────────────────────────────────────── + +variable "skypilot_api_server_enabled" { + description = "Deploy SkyPilot API server for remote job submission. Requires EBS CSI driver for persistent storage." + type = bool + default = false +} + +variable "skypilot_api_server_namespace" { + description = "Namespace for SkyPilot API server deployment." + type = string + default = "skypilot" +} + +variable "skypilot_api_server_release_name" { + description = "Helm release name for SkyPilot API server." + type = string + default = "skypilot" +} + +variable "skypilot_api_server_chart_version" { + description = "Version of the SkyPilot Helm chart to install. Use --devel for nightly builds." + type = string + default = "0.1.0-devel" +} + +variable "skypilot_api_server_web_username" { + description = "Basic auth username for SkyPilot API server." + type = string + default = "skypilot" +} + +variable "skypilot_api_server_web_password" { + description = "Basic auth password for SkyPilot API server. Required when skypilot_api_server_enabled is true." + type = string + sensitive = true + default = null +} + +variable "skypilot_api_server_hostname" { + description = "Public hostname for SkyPilot API server (e.g. skypilot.example.com). When set alongside skypilot_api_server_certificate_arn, an internet-facing ALB Ingress is created." + type = string + default = null +} + +variable "skypilot_api_server_certificate_arn" { + description = "ACM certificate ARN for the SkyPilot API server HTTPS listener. Must cover skypilot_api_server_hostname. If null and skypilot_api_server_hostname is set, certificate must be auto-created at root level (see acm.tf)." + type = string + default = null +} + +variable "skypilot_api_server_aws_credentials_enabled" { + description = "Enable AWS credentials for SkyPilot API server to launch jobs on AWS. Uses IRSA if enabled." + type = bool + default = true +} + +variable "skypilot_api_server_resources" { + description = "Resource requests and limits for SkyPilot API server. Defaults per SkyPilot docs: 4 CPU / 8Gi memory." + type = object({ + requests = object({ + cpu = string + memory = string + }) + limits = object({ + cpu = string + memory = string + }) + }) + default = { + requests = { + cpu = "4" + memory = "8Gi" + } + limits = { + cpu = "4" + memory = "8Gi" + } + } +} + diff --git a/modules/aws/eks-addons/versions.tf b/modules/aws/eks-addons/versions.tf index 3eb8cad..80d5ed7 100644 --- a/modules/aws/eks-addons/versions.tf +++ b/modules/aws/eks-addons/versions.tf @@ -12,5 +12,13 @@ terraform { source = "alekc/kubectl" version = "~> 2.1" } + htpasswd = { + source = "loafoe/htpasswd" + version = "~> 1.2" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } } } diff --git a/variables.tf b/variables.tf index 419f5e4..63ff490 100644 --- a/variables.tf +++ b/variables.tf @@ -291,3 +291,54 @@ variable "tailscale_chart_version" { default = "1.78.3" } +# ── SkyPilot API Server Configuration ───────────────────────────────────────── + +variable "skypilot_api_server_enabled" { + description = "Deploy SkyPilot API server for remote job submission. Requires EBS CSI driver for persistent storage." + type = bool + default = false +} + +variable "skypilot_api_server_web_password" { + description = "Basic auth password for SkyPilot API server. Required when skypilot_api_server_enabled is true." + type = string + sensitive = true + default = null +} + +variable "skypilot_api_server_hostname" { + description = "Public hostname for SkyPilot API server (e.g. skypilot.example.com). When set alongside route53_zone_name, an ACM certificate is auto-created and DNS-validated." + type = string + default = null +} + +variable "skypilot_api_server_chart_version" { + description = "Version of the SkyPilot Helm chart to install." + type = string + default = "0.1.0-devel" +} + +variable "skypilot_api_server_resources" { + description = "Resource requests and limits for SkyPilot API server. Defaults per SkyPilot docs: 4 CPU / 8Gi memory." + type = object({ + requests = object({ + cpu = string + memory = string + }) + limits = object({ + cpu = string + memory = string + }) + }) + default = { + requests = { + cpu = "4" + memory = "8Gi" + } + limits = { + cpu = "4" + memory = "8Gi" + } + } +} +