From ff70a5287854f607f1e892f61e887800bb67bf6e Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Wed, 17 Apr 2024 09:27:53 +0200 Subject: [PATCH 01/25] add getlogs method --- .../wave/service/builder/BuildStrategy.groovy | 2 ++ .../service/builder/DockerBuildStrategy.groovy | 18 +++++++++++++++++- .../service/builder/KubeBuildStrategy.groovy | 13 +++++++++---- .../wave/service/k8s/K8sServiceImpl.groovy | 2 +- .../service/logs/BuildLogServiceImpl.groovy | 14 +++++++++++++- .../FutureContainerBuildServiceTest.groovy | 5 +++++ .../builder/KubeBuildStrategyTest.groovy | 2 +- 7 files changed, 48 insertions(+), 8 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy index 0d8c7dc5b9..4514a3dafd 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy @@ -37,6 +37,8 @@ abstract class BuildStrategy { abstract BuildResult build(BuildRequest req) + abstract String getLogs(String buildId) + void cleanup(BuildRequest req) { req.workDir?.deleteDir() } diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index a4ed1e7455..f40980f3c4 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -113,7 +113,7 @@ class DockerBuildStrategy extends BuildStrategy { ? cmdForKaniko( req.workDir, credsFile, spack, req.platform) : cmdForSingularity( req.workDir, credsFile, spack, req.platform) - return dockerCmd + launchCmd(req) + return dockerCmd + builderName(req.buildId) +launchCmd(req) } protected List cmdForKaniko(Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform ) { @@ -173,4 +173,20 @@ class DockerBuildStrategy extends BuildStrategy { wrapper.add(buildConfig.singularityImage(platform)) return wrapper } + + protected String builderName(String buildId) { + def name = "build-${buildId}".toString().replace('_', '-') + + return ['--name', name] + } + + @Override + String getLogs(String buildId) { + def logCmd = ['docker', 'logs', builderName(buildId)] + final proc = new ProcessBuilder() + .command(logCmd) + .redirectErrorStream(true) + .start() + return proc.inputStream.text + } } diff --git a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy index be7fdc1787..7efd842bf4 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy @@ -71,8 +71,8 @@ class KubeBuildStrategy extends BuildStrategy { @Inject private RegistryProxyService proxyService - protected String podName(BuildRequest req) { - return "build-${req.buildId}".toString().replace('_', '-') + protected String podName(String buildId) { + return "build-${buildId}".toString().replace('_', '-') } @Override @@ -96,7 +96,7 @@ class KubeBuildStrategy extends BuildStrategy { try { final buildImage = getBuildImage(req) final buildCmd = launchCmd(req) - final name = podName(req) + final name = podName(req.buildId) final selector= getSelectorLabel(req.platform, nodeSelectorMap) final spackCfg0 = req.isSpackBuild ? spackConfig : null final pod = k8sService.buildContainer(name, buildImage, buildCmd, req.workDir, configFile, spackCfg0, selector) @@ -130,7 +130,7 @@ class KubeBuildStrategy extends BuildStrategy { @Override void cleanup(BuildRequest req) { super.cleanup(req) - final name = podName(req) + final name = podName(req.buildId) try { k8sService.deletePod(name) } @@ -139,4 +139,9 @@ class KubeBuildStrategy extends BuildStrategy { } } + @Override + String getLogs(String buildId) { + return k8sService.logsPod(podName(buildId)) + } + } diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index df64fc33bb..8a16015a3b 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -442,7 +442,7 @@ class K8sServiceImpl implements K8sService { } } - /** +/** * Fetch the logs of a pod * * @param name The pod name diff --git a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy index f8e0782ad1..f72de7438f 100644 --- a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy @@ -32,6 +32,9 @@ import io.micronaut.objectstorage.request.UploadRequest import io.micronaut.runtime.event.annotation.EventListener import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.builder.BuildStrategy +import io.seqera.wave.service.builder.KubeBuildStrategy +import io.seqera.wave.service.k8s.K8sService import io.seqera.wave.service.persistence.PersistenceService import jakarta.annotation.PostConstruct import jakarta.inject.Inject @@ -56,6 +59,9 @@ class BuildLogServiceImpl implements BuildLogService { @Inject private PersistenceService persistenceService + @Inject + private BuildStrategy buildStrategy + @Nullable @Value('${wave.build.logs.prefix}') private String prefix @@ -107,7 +113,13 @@ class BuildLogServiceImpl implements BuildLogService { private StreamedFile fetchLogStream0(String buildId) { if( !buildId ) return null final Optional> result = objectStorageOperations.retrieve(logKey(buildId)) - return result.isPresent() ? result.get().toStreamedFile() : null + return result.isPresent() ? result.get().toStreamedFile() : fetchLogStream1(buildId) + } + + private StreamedFile fetchLogStream1(String buildId) { + StreamedFile result = null + buildStrategy.getLogs(buildId) + return result } @Override diff --git a/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy index f2a1a40f8e..f685a53302 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy @@ -53,6 +53,11 @@ class FutureContainerBuildServiceTest extends Specification { BuildResult build(BuildRequest req) { new BuildResult("", exitCode, "a fake build result in a test", Instant.now(), Duration.ofSeconds(3), 'abc') } + + @Override + String getLogs(String buildId) { + return "fake build logs" + } } } diff --git a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy index 588b2f773a..a9cfbf0d43 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy @@ -122,7 +122,7 @@ class KubeBuildStrategyTest extends Specification { req = req.withBuildId('1') when: - def podName = strategy.podName(req) + def podName = strategy.podName(req.buildId) then: req.buildId == '143ee73bcdac45b1_1' From 6f3cd8f8376326c26b7d8a798b764839bb0449e8 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Wed, 17 Apr 2024 21:55:26 +0200 Subject: [PATCH 02/25] fixed builderName --- .../seqera/wave/service/builder/DockerBuildStrategy.groovy | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index f40980f3c4..296e0b3c6a 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -174,15 +174,14 @@ class DockerBuildStrategy extends BuildStrategy { return wrapper } - protected String builderName(String buildId) { + protected List builderName(String buildId) { def name = "build-${buildId}".toString().replace('_', '-') - - return ['--name', name] + return List.of('--name', name) } @Override String getLogs(String buildId) { - def logCmd = ['docker', 'logs', builderName(buildId)] + def logCmd = ['docker', 'logs'] + builderName(buildId) final proc = new ProcessBuilder() .command(logCmd) .redirectErrorStream(true) From 8af97ef34b79cabed31eb6d13d66bf4627d87f9d Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 18 Apr 2024 12:56:26 +0200 Subject: [PATCH 03/25] fixed DockerBuildStrategyTest --- .../builder/DockerBuildStrategy.groovy | 19 +++++++++++++------ .../builder/DockerBuildStrategyTest.groovy | 16 +++++++++++++--- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index 296e0b3c6a..6f7b7d3154 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -110,13 +110,13 @@ class DockerBuildStrategy extends BuildStrategy { final spack = req.isSpackBuild ? spackConfig : null final dockerCmd = req.formatDocker() - ? cmdForKaniko( req.workDir, credsFile, spack, req.platform) - : cmdForSingularity( req.workDir, credsFile, spack, req.platform) + ? cmdForKaniko( req.workDir, credsFile, spack, req.platform, req.buildId) + : cmdForSingularity( req.workDir, credsFile, spack, req.platform, req.buildId) - return dockerCmd + builderName(req.buildId) +launchCmd(req) + return dockerCmd + launchCmd(req) } - protected List cmdForKaniko(Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform ) { + protected List cmdForKaniko(Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform, String buildId) { final wrapper = ['docker', 'run', '--rm', @@ -137,13 +137,17 @@ class DockerBuildStrategy extends BuildStrategy { wrapper.add('--platform') wrapper.add(platform.toString()) } + + //add builder name + wrapper.addAll(builderName(buildId)) + // the container image to be used t wrapper.add( buildConfig.kanikoImage ) // return it return wrapper } - protected List cmdForSingularity(Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform) { + protected List cmdForSingularity(Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform, String buildId) { final wrapper = ['docker', 'run', '--rm', @@ -170,11 +174,14 @@ class DockerBuildStrategy extends BuildStrategy { wrapper.add(platform.toString()) } + //add builder name + wrapper.addAll(builderName(buildId)) + wrapper.add(buildConfig.singularityImage(platform)) return wrapper } - protected List builderName(String buildId) { + protected static List builderName(String buildId) { def name = "build-${buildId}".toString().replace('_', '-') return List.of('--name', name) } diff --git a/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy index d9140bd103..4409862ba8 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy @@ -45,16 +45,17 @@ class DockerBuildStrategyTest extends Specification { and: def work = Path.of('/work/foo') when: - def cmd = service.cmdForKaniko(work, null, null, null) + def cmd = service.cmdForKaniko(work, null, null, null, '1234') then: cmd == ['docker', 'run', '--rm', '-v', '/work/foo:/work/foo', + '--name', 'build-1234', 'gcr.io/kaniko-project/executor:v1.19.2'] when: - cmd = service.cmdForKaniko(work, Path.of('/foo/creds.json'), null, ContainerPlatform.of('arm64')) + cmd = service.cmdForKaniko(work, Path.of('/foo/creds.json'), null, ContainerPlatform.of('arm64'), '1234') then: cmd == ['docker', 'run', @@ -62,10 +63,11 @@ class DockerBuildStrategyTest extends Specification { '-v', '/work/foo:/work/foo', '-v', '/foo/creds.json:/kaniko/.docker/config.json:ro', '--platform', 'linux/arm64', + '--name', 'build-1234', 'gcr.io/kaniko-project/executor:v1.19.2'] when: - cmd = service.cmdForKaniko(work, Path.of('/foo/creds.json'), spackConfig, null) + cmd = service.cmdForKaniko(work, Path.of('/foo/creds.json'), spackConfig, null, '1234') then: cmd == ['docker', 'run', @@ -73,6 +75,7 @@ class DockerBuildStrategyTest extends Specification { '-v', '/work/foo:/work/foo', '-v', '/foo/creds.json:/kaniko/.docker/config.json:ro', '-v', '/host/spack/key:/opt/spack/key:ro', + '--name', 'build-1234', 'gcr.io/kaniko-project/executor:v1.19.2'] cleanup: @@ -87,6 +90,7 @@ class DockerBuildStrategyTest extends Specification { def creds = Path.of('/work/creds.json') and: def req = new BuildRequest( + buildId: '1234', workDir: Path.of('/work/foo/89fb83ce6ec8627b'), platform: ContainerPlatform.of('linux/amd64'), targetImage: 'repo:89fb83ce6ec8627b', @@ -100,6 +104,7 @@ class DockerBuildStrategyTest extends Specification { '-v', '/work/foo/89fb83ce6ec8627b:/work/foo/89fb83ce6ec8627b', '-v', '/work/creds.json:/kaniko/.docker/config.json:ro', '--platform', 'linux/amd64', + '--name', 'build-1234', 'gcr.io/kaniko-project/executor:v1.19.2', '--dockerfile', '/work/foo/89fb83ce6ec8627b/Containerfile', '--context', '/work/foo/89fb83ce6ec8627b/context', @@ -118,6 +123,7 @@ class DockerBuildStrategyTest extends Specification { def service = ctx.getBean(DockerBuildStrategy) and: def req = new BuildRequest( + buildId: '1234', workDir: Path.of('/work/foo/89fb83ce6ec8627b'), platform: ContainerPlatform.of('linux/amd64'), targetImage: 'repo:89fb83ce6ec8627b', @@ -151,6 +157,7 @@ class DockerBuildStrategyTest extends Specification { def creds = Path.of('/work/creds.json') and: def req = new BuildRequest( + buildId: '1234', workDir: Path.of('/work/foo/d4869cc39b8d7d55'), platform: ContainerPlatform.of('linux/amd64'), targetImage: 'oras://repo:d4869cc39b8d7d55', @@ -170,6 +177,7 @@ class DockerBuildStrategyTest extends Specification { '-v', '/work/singularity-remote.yaml:/root/.singularity/remote.yaml:ro', '-v', '/host/spack/key:/opt/spack/key:ro', '--platform', 'linux/amd64', + '--name', 'build-1234', 'quay.io/singularity/singularity:v3.11.4-slim', 'sh', '-c', @@ -193,6 +201,7 @@ class DockerBuildStrategyTest extends Specification { def creds = Path.of('/work/creds.json') and: def req = new BuildRequest( + buildId: '1234', workDir: Path.of('/work/foo/9c68af894bb2419c'), platform: ContainerPlatform.of('linux/arm64'), targetImage: 'oras://repo:9c68af894bb2419c', @@ -212,6 +221,7 @@ class DockerBuildStrategyTest extends Specification { '-v', '/work/singularity-remote.yaml:/root/.singularity/remote.yaml:ro', '-v', '/host/spack/key:/opt/spack/key:ro', '--platform', 'linux/arm64', + '--name', 'build-1234', 'quay.io/singularity/singularity:v3.11.4-slim-arm64', 'sh', '-c', From d228dd98dd0300ae7acbd004de92b40ac027327c Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 18 Apr 2024 14:47:49 +0200 Subject: [PATCH 04/25] added BuildLogLocalServiceImpl --- .../builder/DockerBuildStrategy.groovy | 12 +-- .../logs/BuildLogLocalServiceImpl.groovy | 76 +++++++++++++++++++ .../service/logs/BuildLogServiceImpl.groovy | 9 ++- 3 files changed, 88 insertions(+), 9 deletions(-) create mode 100644 src/main/groovy/io/seqera/wave/service/logs/BuildLogLocalServiceImpl.groovy diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index 6f7b7d3154..94dad11efe 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -139,7 +139,8 @@ class DockerBuildStrategy extends BuildStrategy { } //add builder name - wrapper.addAll(builderName(buildId)) + wrapper.add('--name') + wrapper.add(builderName(buildId)) // the container image to be used t wrapper.add( buildConfig.kanikoImage ) @@ -175,20 +176,21 @@ class DockerBuildStrategy extends BuildStrategy { } //add builder name - wrapper.addAll(builderName(buildId)) + wrapper.add('--name') + wrapper.add(builderName(buildId)) wrapper.add(buildConfig.singularityImage(platform)) return wrapper } - protected static List builderName(String buildId) { - def name = "build-${buildId}".toString().replace('_', '-') - return List.of('--name', name) + protected static String builderName(String buildId) { + return "build-${buildId}".toString().replace('_', '-') } @Override String getLogs(String buildId) { def logCmd = ['docker', 'logs'] + builderName(buildId) + log.info("Get build logs: ${logCmd.join(' ')}") final proc = new ProcessBuilder() .command(logCmd) .redirectErrorStream(true) diff --git a/src/main/groovy/io/seqera/wave/service/logs/BuildLogLocalServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/logs/BuildLogLocalServiceImpl.groovy new file mode 100644 index 0000000000..f829cd7104 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/logs/BuildLogLocalServiceImpl.groovy @@ -0,0 +1,76 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.logs + +import java.nio.charset.StandardCharsets + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Requires +import io.micronaut.context.annotation.Value +import io.micronaut.http.MediaType +import io.micronaut.http.server.types.files.StreamedFile +import io.seqera.wave.service.builder.BuildStrategy +import jakarta.inject.Inject +import jakarta.inject.Singleton +import org.apache.commons.io.input.BoundedInputStream + +/** + * Implements Service to manage logs from local storage + * + * @author Munish Chouhan + */ +@Slf4j +@Singleton +@CompileStatic +@Requires(missingProperty = 'wave.build.logs.bucket') +class BuildLogLocalServiceImpl implements BuildLogService { + + @Inject + private BuildStrategy buildStrategy + + @Value('${wave.build.logs.maxLength:100000}') + private long maxLength + + Map logStore = new HashMap() + + @Override + void storeLog(String buildId, String log) { + logStore.put(buildId, log) + } + + @Override + StreamedFile fetchLogStream(String buildId) { + String log = logStore.get(buildId) ?: buildStrategy.getLogs(buildId) + if( !log ) + return null + def inputStream = new ByteArrayInputStream(log.getBytes(StandardCharsets.UTF_8)) + def result = inputStream ? new StreamedFile(inputStream, MediaType.TEXT_PLAIN_TYPE) : null + return result + } + + @Override + BuildLog fetchLogString(String buildId) { + final result = fetchLogStream(buildId) + if( !result ) + return null + final logs = new BoundedInputStream(result.getInputStream(), maxLength).getText() + return new BuildLog(logs, logs.length()>=maxLength) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy index f72de7438f..a657754f79 100644 --- a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy @@ -18,6 +18,7 @@ package io.seqera.wave.service.logs +import java.nio.charset.StandardCharsets import java.util.concurrent.CompletableFuture import io.micronaut.core.annotation.Nullable @@ -25,6 +26,7 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Requires import io.micronaut.context.annotation.Value +import io.micronaut.http.MediaType import io.micronaut.http.server.types.files.StreamedFile import io.micronaut.objectstorage.ObjectStorageEntry import io.micronaut.objectstorage.ObjectStorageOperations @@ -33,8 +35,6 @@ import io.micronaut.runtime.event.annotation.EventListener import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.builder.BuildStrategy -import io.seqera.wave.service.builder.KubeBuildStrategy -import io.seqera.wave.service.k8s.K8sService import io.seqera.wave.service.persistence.PersistenceService import jakarta.annotation.PostConstruct import jakarta.inject.Inject @@ -117,8 +117,9 @@ class BuildLogServiceImpl implements BuildLogService { } private StreamedFile fetchLogStream1(String buildId) { - StreamedFile result = null - buildStrategy.getLogs(buildId) + final log = buildStrategy.getLogs(buildId) + def inputStream = new ByteArrayInputStream(log.getBytes(StandardCharsets.UTF_8)) + def result = inputStream ? new StreamedFile(inputStream, MediaType.TEXT_PLAIN_TYPE) : null return result } From a5ba7d9da3678c1404cbf907d6be273ab8647d80 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 18 Apr 2024 17:11:39 +0200 Subject: [PATCH 05/25] added getCurrentLogsPod(String name) --- .../service/builder/KubeBuildStrategy.groovy | 2 +- .../seqera/wave/service/k8s/K8sService.groovy | 2 ++ .../wave/service/k8s/K8sServiceImpl.groovy | 21 ++++++++++++++++++- .../logs/BuildLogLocalServiceImpl.groovy | 13 +++++++++++- 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy index 7efd842bf4..c84ffc0c1b 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy @@ -141,7 +141,7 @@ class KubeBuildStrategy extends BuildStrategy { @Override String getLogs(String buildId) { - return k8sService.logsPod(podName(buildId)) + return k8sService.getCurrentLogsPod(podName(buildId)) } } diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy index cbea2605f3..16285b49e4 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy @@ -54,4 +54,6 @@ interface K8sService { V1Pod transferContainer(String name, String containerImage, List args, BlobCacheConfig blobConfig) V1ContainerStateTerminated waitPod(V1Pod pod, long timeout) + + String getCurrentLogsPod(String name) } diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index 8a16015a3b..5a3299a2b3 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -19,6 +19,7 @@ package io.seqera.wave.service.k8s import java.nio.file.Path +import java.util.stream.Collectors import javax.annotation.PostConstruct import groovy.transform.CompileDynamic @@ -35,6 +36,7 @@ import io.kubernetes.client.openapi.models.V1JobBuilder import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimVolumeSource import io.kubernetes.client.openapi.models.V1Pod import io.kubernetes.client.openapi.models.V1PodBuilder +import io.kubernetes.client.openapi.models.V1PodStatus import io.kubernetes.client.openapi.models.V1ResourceRequirements import io.kubernetes.client.openapi.models.V1Volume import io.kubernetes.client.openapi.models.V1VolumeMount @@ -442,7 +444,7 @@ class K8sServiceImpl implements K8sService { } } -/** + /** * Fetch the logs of a pod * * @param name The pod name @@ -460,6 +462,23 @@ class K8sServiceImpl implements K8sService { } } + /** + * Fetch current available logs of a running pod + * + * @param name The pod name + * @return The logs as a string or when logs are not available or cannot be accessed + */ + @Override + String getCurrentLogsPod(String name) { + try { + return k8sClient.coreV1Api().readNamespacedPodLog(name, namespace, null, null, null, null, null, null, null, null, null) + } catch (Exception e) { + // logging trace here because errors are expected when the pod is not running + log.trace "Unable to fetch logs for pod: $name", e + return null + } + } + /** * Delete a pod * diff --git a/src/main/groovy/io/seqera/wave/service/logs/BuildLogLocalServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/logs/BuildLogLocalServiceImpl.groovy index f829cd7104..19f7cba499 100644 --- a/src/main/groovy/io/seqera/wave/service/logs/BuildLogLocalServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/logs/BuildLogLocalServiceImpl.groovy @@ -19,6 +19,8 @@ package io.seqera.wave.service.logs import java.nio.charset.StandardCharsets +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentHashMap import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -26,6 +28,8 @@ import io.micronaut.context.annotation.Requires import io.micronaut.context.annotation.Value import io.micronaut.http.MediaType import io.micronaut.http.server.types.files.StreamedFile +import io.micronaut.runtime.event.annotation.EventListener +import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildStrategy import jakarta.inject.Inject import jakarta.inject.Singleton @@ -48,7 +52,14 @@ class BuildLogLocalServiceImpl implements BuildLogService { @Value('${wave.build.logs.maxLength:100000}') private long maxLength - Map logStore = new HashMap() + Map logStore = new ConcurrentHashMap() + + @EventListener + void onBuildEvent(BuildEvent event) { + if(event.result.logs) { + CompletableFuture.supplyAsync(() -> storeLog(event.result.id, event.result.logs)) + } + } @Override void storeLog(String buildId, String log) { From 6cb5e0e725935b03fba54d63b683021a25ed8972 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 18 Apr 2024 19:37:46 +0200 Subject: [PATCH 06/25] fixed logs color issue --- .../service/logs/BuildLogLocalServiceImpl.groovy | 13 ++++++------- .../wave/service/logs/BuildLogServiceImpl.groovy | 8 ++++---- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/logs/BuildLogLocalServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/logs/BuildLogLocalServiceImpl.groovy index 19f7cba499..2b2c9d9a72 100644 --- a/src/main/groovy/io/seqera/wave/service/logs/BuildLogLocalServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/logs/BuildLogLocalServiceImpl.groovy @@ -34,7 +34,6 @@ import io.seqera.wave.service.builder.BuildStrategy import jakarta.inject.Inject import jakarta.inject.Singleton import org.apache.commons.io.input.BoundedInputStream - /** * Implements Service to manage logs from local storage * @@ -52,7 +51,7 @@ class BuildLogLocalServiceImpl implements BuildLogService { @Value('${wave.build.logs.maxLength:100000}') private long maxLength - Map logStore = new ConcurrentHashMap() + Map logStore = new ConcurrentHashMap() @EventListener void onBuildEvent(BuildEvent event) { @@ -68,12 +67,12 @@ class BuildLogLocalServiceImpl implements BuildLogService { @Override StreamedFile fetchLogStream(String buildId) { - String log = logStore.get(buildId) ?: buildStrategy.getLogs(buildId) - if( !log ) + //replace all regex is removing color from log otherwise it will not be displayed correctly in browser + final logs = logStore.get(buildId) ?: buildStrategy.getLogs(buildId).replaceAll("\u001B\\[[;\\d]*m", "") + if( !logs ) return null - def inputStream = new ByteArrayInputStream(log.getBytes(StandardCharsets.UTF_8)) - def result = inputStream ? new StreamedFile(inputStream, MediaType.TEXT_PLAIN_TYPE) : null - return result + def inputStream = new ByteArrayInputStream(logs.getBytes(StandardCharsets.UTF_8)) + return inputStream ? new StreamedFile(inputStream, MediaType.TEXT_HTML_TYPE) : null } @Override diff --git a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy index a657754f79..ae0e23f6a9 100644 --- a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy @@ -117,10 +117,10 @@ class BuildLogServiceImpl implements BuildLogService { } private StreamedFile fetchLogStream1(String buildId) { - final log = buildStrategy.getLogs(buildId) - def inputStream = new ByteArrayInputStream(log.getBytes(StandardCharsets.UTF_8)) - def result = inputStream ? new StreamedFile(inputStream, MediaType.TEXT_PLAIN_TYPE) : null - return result + //replace all regex is removing color from log otherwise it will not be displayed correctly in browser + final logs = buildStrategy.getLogs(buildId).replaceAll("\u001B\\[[;\\d]*m", "") + def inputStream = new ByteArrayInputStream(logs.getBytes(StandardCharsets.UTF_8)) + return inputStream ? new StreamedFile(inputStream, MediaType.TEXT_HTML_TYPE) : null } @Override From eafcb949926bf3580d60fce49c0e537b180ecd38 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 18 Apr 2024 22:45:03 +0200 Subject: [PATCH 07/25] added build in progress phase and dynamic logs loading --- .../wave/controller/ViewController.groovy | 2 + .../logs/BuildLogLocalServiceImpl.groovy | 86 ------------------- .../service/logs/BuildLogServiceImpl.groovy | 6 +- .../resources/io/seqera/wave/build-view.hbs | 48 ++++++++++- 4 files changed, 53 insertions(+), 89 deletions(-) delete mode 100644 src/main/groovy/io/seqera/wave/service/logs/BuildLogLocalServiceImpl.groovy diff --git a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy index 49aac310da..327e646d1a 100644 --- a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy @@ -71,6 +71,8 @@ class ViewController { final binding = new HashMap(20) binding.build_id = result.buildId binding.build_success = result.succeeded() + binding.build_failed = result.exitStatus && result.exitStatus != 0 + binding.build_in_progress = result.exitStatus == null binding.build_exit_status = result.exitStatus binding.build_user = (result.userName ?: '-') + " (ip: ${result.requestIp})" binding.build_time = formatTimestamp(result.startTime, result.offsetId) ?: '-' diff --git a/src/main/groovy/io/seqera/wave/service/logs/BuildLogLocalServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/logs/BuildLogLocalServiceImpl.groovy deleted file mode 100644 index 2b2c9d9a72..0000000000 --- a/src/main/groovy/io/seqera/wave/service/logs/BuildLogLocalServiceImpl.groovy +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.logs - -import java.nio.charset.StandardCharsets -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ConcurrentHashMap - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import io.micronaut.context.annotation.Requires -import io.micronaut.context.annotation.Value -import io.micronaut.http.MediaType -import io.micronaut.http.server.types.files.StreamedFile -import io.micronaut.runtime.event.annotation.EventListener -import io.seqera.wave.service.builder.BuildEvent -import io.seqera.wave.service.builder.BuildStrategy -import jakarta.inject.Inject -import jakarta.inject.Singleton -import org.apache.commons.io.input.BoundedInputStream -/** - * Implements Service to manage logs from local storage - * - * @author Munish Chouhan - */ -@Slf4j -@Singleton -@CompileStatic -@Requires(missingProperty = 'wave.build.logs.bucket') -class BuildLogLocalServiceImpl implements BuildLogService { - - @Inject - private BuildStrategy buildStrategy - - @Value('${wave.build.logs.maxLength:100000}') - private long maxLength - - Map logStore = new ConcurrentHashMap() - - @EventListener - void onBuildEvent(BuildEvent event) { - if(event.result.logs) { - CompletableFuture.supplyAsync(() -> storeLog(event.result.id, event.result.logs)) - } - } - - @Override - void storeLog(String buildId, String log) { - logStore.put(buildId, log) - } - - @Override - StreamedFile fetchLogStream(String buildId) { - //replace all regex is removing color from log otherwise it will not be displayed correctly in browser - final logs = logStore.get(buildId) ?: buildStrategy.getLogs(buildId).replaceAll("\u001B\\[[;\\d]*m", "") - if( !logs ) - return null - def inputStream = new ByteArrayInputStream(logs.getBytes(StandardCharsets.UTF_8)) - return inputStream ? new StreamedFile(inputStream, MediaType.TEXT_HTML_TYPE) : null - } - - @Override - BuildLog fetchLogString(String buildId) { - final result = fetchLogStream(buildId) - if( !result ) - return null - final logs = new BoundedInputStream(result.getInputStream(), maxLength).getText() - return new BuildLog(logs, logs.length()>=maxLength) - } -} diff --git a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy index ae0e23f6a9..a1b02f7c4e 100644 --- a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy @@ -118,7 +118,11 @@ class BuildLogServiceImpl implements BuildLogService { private StreamedFile fetchLogStream1(String buildId) { //replace all regex is removing color from log otherwise it will not be displayed correctly in browser - final logs = buildStrategy.getLogs(buildId).replaceAll("\u001B\\[[;\\d]*m", "") + def logs = buildStrategy.getLogs(buildId).replaceAll("\u001B\\[[;\\d]*m", "") + if( !logs ) + return null + else + logs = logs.replaceAll("\u001B\\[[;\\d]*m", "") def inputStream = new ByteArrayInputStream(logs.getBytes(StandardCharsets.UTF_8)) return inputStream ? new StreamedFile(inputStream, MediaType.TEXT_HTML_TYPE) : null } diff --git a/src/main/resources/io/seqera/wave/build-view.hbs b/src/main/resources/io/seqera/wave/build-view.hbs index 5cb33235f9..962433ca6d 100644 --- a/src/main/resources/io/seqera/wave/build-view.hbs +++ b/src/main/resources/io/seqera/wave/build-view.hbs @@ -21,12 +21,18 @@ Container build completed successfully! - {{else}} + {{else build_failed}}

Container build failed

+ {{else build_in_progress}} +
+

+ Container build in progress +

+
{{/if}}

Summary

@@ -98,12 +104,50 @@ {{#if build_log_data}}

Build logs

- 
{{build_log_data}}
+ 
{{build_log_data}}
{{#if build_log_truncated}} Click here to download the complete build log {{/if}} {{/if}} +
From c335997a7d35a2dc2be9c9281a87324ec7c8dab7 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 18 Apr 2024 22:49:58 +0200 Subject: [PATCH 08/25] minor change --- .../io/seqera/wave/service/logs/BuildLogServiceImpl.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy index a1b02f7c4e..a7a064d890 100644 --- a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy @@ -117,11 +117,11 @@ class BuildLogServiceImpl implements BuildLogService { } private StreamedFile fetchLogStream1(String buildId) { - //replace all regex is removing color from log otherwise it will not be displayed correctly in browser - def logs = buildStrategy.getLogs(buildId).replaceAll("\u001B\\[[;\\d]*m", "") + def logs = buildStrategy.getLogs(buildId) if( !logs ) return null else + //replace all regex is removing color from log otherwise it will not be displayed correctly in browser logs = logs.replaceAll("\u001B\\[[;\\d]*m", "") def inputStream = new ByteArrayInputStream(logs.getBytes(StandardCharsets.UTF_8)) return inputStream ? new StreamedFile(inputStream, MediaType.TEXT_HTML_TYPE) : null From 26fdb65d3ada0238de632c034e2866707b807b82 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Fri, 19 Apr 2024 11:55:21 +0200 Subject: [PATCH 09/25] changed to inputstream --- .../seqera/wave/service/builder/BuildStrategy.groovy | 2 +- .../wave/service/builder/DockerBuildStrategy.groovy | 4 ++-- .../wave/service/builder/KubeBuildStrategy.groovy | 2 +- .../io/seqera/wave/service/k8s/K8sService.groovy | 2 +- .../io/seqera/wave/service/k8s/K8sServiceImpl.groovy | 6 ++++-- .../wave/service/logs/BuildLogServiceImpl.groovy | 10 +++------- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy index 4514a3dafd..9381425a64 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy @@ -37,7 +37,7 @@ abstract class BuildStrategy { abstract BuildResult build(BuildRequest req) - abstract String getLogs(String buildId) + abstract InputStream getLogs(String buildId) void cleanup(BuildRequest req) { req.workDir?.deleteDir() diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index 94dad11efe..4faedc6827 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -188,13 +188,13 @@ class DockerBuildStrategy extends BuildStrategy { } @Override - String getLogs(String buildId) { + InputStream getLogs(String buildId) { def logCmd = ['docker', 'logs'] + builderName(buildId) log.info("Get build logs: ${logCmd.join(' ')}") final proc = new ProcessBuilder() .command(logCmd) .redirectErrorStream(true) .start() - return proc.inputStream.text + return proc.inputStream } } diff --git a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy index c84ffc0c1b..f2d53e9830 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy @@ -140,7 +140,7 @@ class KubeBuildStrategy extends BuildStrategy { } @Override - String getLogs(String buildId) { + InputStream getLogs(String buildId) { return k8sService.getCurrentLogsPod(podName(buildId)) } diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy index 16285b49e4..30fd68935e 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy @@ -55,5 +55,5 @@ interface K8sService { V1ContainerStateTerminated waitPod(V1Pod pod, long timeout) - String getCurrentLogsPod(String name) + InputStream getCurrentLogsPod(String name) } diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index 5a3299a2b3..44383ea853 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -52,6 +52,7 @@ import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.scan.Trivy import jakarta.inject.Inject import jakarta.inject.Singleton +import okhttp3.Call /** * implements the support for Kubernetes cluster * @@ -469,9 +470,10 @@ class K8sServiceImpl implements K8sService { * @return The logs as a string or when logs are not available or cannot be accessed */ @Override - String getCurrentLogsPod(String name) { + InputStream getCurrentLogsPod(String name) { try { - return k8sClient.coreV1Api().readNamespacedPodLog(name, namespace, null, null, null, null, null, null, null, null, null) + Call call = k8sClient.coreV1Api().readNamespacedPodLogCall(name, namespace, name, false, null, null, "false", false, null, null, null, null); + return call.execute().body().byteStream() } catch (Exception e) { // logging trace here because errors are expected when the pod is not running log.trace "Unable to fetch logs for pod: $name", e diff --git a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy index a7a064d890..aee4d33992 100644 --- a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy @@ -117,14 +117,10 @@ class BuildLogServiceImpl implements BuildLogService { } private StreamedFile fetchLogStream1(String buildId) { - def logs = buildStrategy.getLogs(buildId) - if( !logs ) + def logStream = buildStrategy.getLogs(buildId) + if( !logStream ) return null - else - //replace all regex is removing color from log otherwise it will not be displayed correctly in browser - logs = logs.replaceAll("\u001B\\[[;\\d]*m", "") - def inputStream = new ByteArrayInputStream(logs.getBytes(StandardCharsets.UTF_8)) - return inputStream ? new StreamedFile(inputStream, MediaType.TEXT_HTML_TYPE) : null + return logStream ? new StreamedFile(logStream, MediaType.APPLICATION_OCTET_STREAM_TYPE) : null } @Override From 93491f54a00e6d17aa55ae256240a1592c8ac78b Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Fri, 19 Apr 2024 13:59:26 +0200 Subject: [PATCH 10/25] strip ansi escape codes --- .../groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy | 6 +++--- .../io/seqera/wave/service/logs/BuildLogServiceImpl.groovy | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index 44383ea853..a3cc29c23e 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -52,7 +52,6 @@ import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.scan.Trivy import jakarta.inject.Inject import jakarta.inject.Singleton -import okhttp3.Call /** * implements the support for Kubernetes cluster * @@ -472,8 +471,9 @@ class K8sServiceImpl implements K8sService { @Override InputStream getCurrentLogsPod(String name) { try { - Call call = k8sClient.coreV1Api().readNamespacedPodLogCall(name, namespace, name, false, null, null, "false", false, null, null, null, null); - return call.execute().body().byteStream() + def logs = k8sClient.coreV1Api().readNamespacedPodLog(name, namespace, name, false, null, null, "false", false, null, null, null) + logs = logs ? logs.replaceAll("\u001B\\[[;\\d]*m", "") : null // strip ansi escape codes + return new ByteArrayInputStream(logs.getBytes()) } catch (Exception e) { // logging trace here because errors are expected when the pod is not running log.trace "Unable to fetch logs for pod: $name", e diff --git a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy index aee4d33992..5307c6b7c6 100644 --- a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy @@ -120,7 +120,7 @@ class BuildLogServiceImpl implements BuildLogService { def logStream = buildStrategy.getLogs(buildId) if( !logStream ) return null - return logStream ? new StreamedFile(logStream, MediaType.APPLICATION_OCTET_STREAM_TYPE) : null + return logStream ? new StreamedFile(logStream, MediaType.TEXT_PLAIN_TYPE) : null } @Override From 7869323f8ee2a959390f37965bf788e49566662f Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Fri, 19 Apr 2024 14:11:22 +0200 Subject: [PATCH 11/25] added docs --- docs/api.mdx | 32 ++++++++++++++++++- .../service/logs/BuildLogServiceImpl.groovy | 2 +- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/docs/api.mdx b/docs/api.mdx index 525954b21a..b5f7b22b3e 100644 --- a/docs/api.mdx +++ b/docs/api.mdx @@ -238,7 +238,7 @@ curl --location 'http://localhost:9090/v1alpha2/container' \ Note: You need to add your container registry credentials in seqera platform to use freeze feature which is a requirement for singularity. -## `/v1alpha1/builds/{buildId}/status` +## GET `/v1alpha1/builds/{buildId}/status` Provides status of build against buildId passed as path variable @@ -270,6 +270,36 @@ Note: status can only be `PENDING` or `COMPLETED`. } ``` +## GET `/v1alpha1/builds/{buildId}/logs` + +Provides logs of build against buildId passed as path variable + +### Response + +```text +build container logs text +``` + +### Example + +``` +% curl --location 'http://localhost:9090/v1alpha1/builds//logs' +INFO[0001] Retrieving image manifest alpine:latest +INFO[0001] Retrieving image alpine:latest from registry index.docker.io +INFO[0002] Retrieving image manifest alpine:latest +INFO[0002] Returning cached image manifest +INFO[0002] Built cross stage deps: map[] +INFO[0002] Retrieving image manifest alpine:latest +INFO[0002] Returning cached image manifest +INFO[0002] Retrieving image manifest alpine:latest +INFO[0002] Returning cached image manifest +INFO[0002] Executing 0 build triggers +INFO[0002] Building stage 'alpine:latest' [idx: '0', base-idx: '-1'] +INFO[0002] Skipping unpacking as no commands require it. +INFO[0002] Pushing image to / +INFO[0005] Pushed index.docker.io// +``` + ## `/service-info` Provides basic information about the service status. diff --git a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy index 5307c6b7c6..aee4d33992 100644 --- a/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy @@ -120,7 +120,7 @@ class BuildLogServiceImpl implements BuildLogService { def logStream = buildStrategy.getLogs(buildId) if( !logStream ) return null - return logStream ? new StreamedFile(logStream, MediaType.TEXT_PLAIN_TYPE) : null + return logStream ? new StreamedFile(logStream, MediaType.APPLICATION_OCTET_STREAM_TYPE) : null } @Override From b287a56f499291d3567fb78a5abc11e74657ec31 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Fri, 19 Apr 2024 14:14:24 +0200 Subject: [PATCH 12/25] fixed tests --- .../service/builder/FutureContainerBuildServiceTest.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy index 2b96b3667b..d62094b5d3 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/FutureContainerBuildServiceTest.groovy @@ -55,8 +55,8 @@ class FutureContainerBuildServiceTest extends Specification { } @Override - String getLogs(String buildId) { - return "fake build logs" + InputStream getLogs(String buildId) { + return new ByteArrayInputStream("fake build logs".getBytes()) } } } From a294b6415e14b2cf422190c5840231f54d9c6310 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Fri, 19 Apr 2024 15:34:36 +0200 Subject: [PATCH 13/25] added test --- .../wave/service/k8s/K8sServiceImpl.groovy | 2 -- .../service/k8s/K8sServiceImplTest.groovy | 20 +++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index a3cc29c23e..edbd8cd56d 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -19,7 +19,6 @@ package io.seqera.wave.service.k8s import java.nio.file.Path -import java.util.stream.Collectors import javax.annotation.PostConstruct import groovy.transform.CompileDynamic @@ -36,7 +35,6 @@ import io.kubernetes.client.openapi.models.V1JobBuilder import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimVolumeSource import io.kubernetes.client.openapi.models.V1Pod import io.kubernetes.client.openapi.models.V1PodBuilder -import io.kubernetes.client.openapi.models.V1PodStatus import io.kubernetes.client.openapi.models.V1ResourceRequirements import io.kubernetes.client.openapi.models.V1Volume import io.kubernetes.client.openapi.models.V1VolumeMount diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy index 16184693c4..607499b55c 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy @@ -24,6 +24,7 @@ import java.nio.file.Path import java.time.Duration import io.kubernetes.client.custom.Quantity +import io.kubernetes.client.openapi.apis.CoreV1Api import io.kubernetes.client.openapi.models.V1EnvVar import io.micronaut.context.ApplicationContext import io.micronaut.test.extensions.spock.annotation.MicronautTest @@ -574,4 +575,23 @@ class K8sServiceImplTest extends Specification { cleanup: ctx.close() } + + def "should get pod logs"() { + given: + def k8sClient = Mock(K8sClient) + def k8sService = new K8sServiceImpl(k8sClient: k8sClient) + def name = "builder-pod" + def logs = "\u001B[31mINFO: Build is in progress" + + when: + InputStream result = k8sService.getCurrentLogsPod(name) + + then: + 2 * k8sClient.coreV1Api() >> Mock(CoreV1Api) + 1 * k8sClient.coreV1Api().readNamespacedPodLog(_, _, _, _, _, _, _, _, _, _, _) >> logs + and: + result instanceof ByteArrayInputStream + String resultString = result.text + resultString == "INFO: Build is in progress" + } } From 94f40e77a99e6eee7b75a6cce86aa689678a3f6a Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Fri, 19 Apr 2024 15:59:08 +0200 Subject: [PATCH 14/25] corrected docs --- docs/api.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api.mdx b/docs/api.mdx index b5f7b22b3e..b5abeb6779 100644 --- a/docs/api.mdx +++ b/docs/api.mdx @@ -272,12 +272,12 @@ Note: status can only be `PENDING` or `COMPLETED`. ## GET `/v1alpha1/builds/{buildId}/logs` -Provides logs of build against buildId passed as path variable +Provide logs of build belongs to the buildId supplied in the API ### Response ```text -build container logs text +string ``` ### Example From db5afdeacee138a74c19946914336d68336dfe18 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Fri, 19 Apr 2024 16:25:47 +0200 Subject: [PATCH 15/25] corrected docs --- docs/api.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api.mdx b/docs/api.mdx index b5abeb6779..7681ff2c4c 100644 --- a/docs/api.mdx +++ b/docs/api.mdx @@ -272,7 +272,7 @@ Note: status can only be `PENDING` or `COMPLETED`. ## GET `/v1alpha1/builds/{buildId}/logs` -Provide logs of build belongs to the buildId supplied in the API +Provide logs of build belongs to the buildId supplied in the API call ### Response From 1def98add9d34819596ebb4c1af5486030487a9c Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Mon, 22 Apr 2024 15:26:50 +0200 Subject: [PATCH 16/25] updated doc [ci skip] --- docs/api.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api.mdx b/docs/api.mdx index 7681ff2c4c..c16b885cf8 100644 --- a/docs/api.mdx +++ b/docs/api.mdx @@ -272,7 +272,7 @@ Note: status can only be `PENDING` or `COMPLETED`. ## GET `/v1alpha1/builds/{buildId}/logs` -Provide logs of build belongs to the buildId supplied in the API call +Supply logs corresponding to the specified buildId within the API request. ### Response From a5595846fc0aaebe6a2ad8171bae4c50d4485c79 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Mon, 22 Apr 2024 15:33:13 +0200 Subject: [PATCH 17/25] reverted doc changes --- docs/api.mdx | 36 +++--------------------------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/docs/api.mdx b/docs/api.mdx index c16b885cf8..504df4ab7e 100644 --- a/docs/api.mdx +++ b/docs/api.mdx @@ -45,7 +45,7 @@ The endpoint returns the name of the container request made available by Wave. towerWorkspaceId: number, } ``` -#### container token request attributes +#### container token request attributes | Attribute | Description | | ----------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -238,7 +238,7 @@ curl --location 'http://localhost:9090/v1alpha2/container' \ Note: You need to add your container registry credentials in seqera platform to use freeze feature which is a requirement for singularity. -## GET `/v1alpha1/builds/{buildId}/status` +## `/v1alpha1/builds/{buildId}/status` Provides status of build against buildId passed as path variable @@ -270,36 +270,6 @@ Note: status can only be `PENDING` or `COMPLETED`. } ``` -## GET `/v1alpha1/builds/{buildId}/logs` - -Supply logs corresponding to the specified buildId within the API request. - -### Response - -```text -string -``` - -### Example - -``` -% curl --location 'http://localhost:9090/v1alpha1/builds//logs' -INFO[0001] Retrieving image manifest alpine:latest -INFO[0001] Retrieving image alpine:latest from registry index.docker.io -INFO[0002] Retrieving image manifest alpine:latest -INFO[0002] Returning cached image manifest -INFO[0002] Built cross stage deps: map[] -INFO[0002] Retrieving image manifest alpine:latest -INFO[0002] Returning cached image manifest -INFO[0002] Retrieving image manifest alpine:latest -INFO[0002] Returning cached image manifest -INFO[0002] Executing 0 build triggers -INFO[0002] Building stage 'alpine:latest' [idx: '0', base-idx: '-1'] -INFO[0002] Skipping unpacking as no commands require it. -INFO[0002] Pushing image to / -INFO[0005] Pushed index.docker.io// -``` - ## `/service-info` Provides basic information about the service status. @@ -328,7 +298,7 @@ All Metrics API endpoints have these three common optional query parameters: | endDate | Format: `yyyy-mm-dd`, The data of this date will be included in computing the response. | 2024-02-29 | | limit | Integer from `0` to `1000` and default is `100`. It limits the number of records in response. | 10 | -Note: If you only provide startDate then endDate will be today's date. +Note: If you only provide startDate then endDate will be today's date. ### Build Metrics APIs From fd9ea32065533bd523ce4012d2e10a04f4e5e9de Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Tue, 11 Jun 2024 13:15:01 +0200 Subject: [PATCH 18/25] fix tests --- .../builder/DockerBuildStrategyTest.groovy | 29 ++----------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy index 6775791546..7ff0e1eba0 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy @@ -54,9 +54,9 @@ class DockerBuildStrategyTest extends Specification { '--rm', '--privileged', '-v', '/work/foo:/work/foo', - '--name', 'build-1234', '--entrypoint', 'buildctl-daemonless.sh', + '--name', 'build-1234', 'moby/buildkit:v0.13.2-rootless'] when: @@ -121,32 +121,7 @@ class DockerBuildStrategyTest extends Specification { '--platform', 'linux/amd64', '--name', 'build-1234', 'moby/buildkit:v0.13.2-rootless', - '--dockerfile', '/work/foo/89fb83ce6ec8627b/Containerfile', - '--context', '/work/foo/89fb83ce6ec8627b/context', - '--destination', 'repo:89fb83ce6ec8627b', - '--cache=true', - '--custom-platform', 'linux/amd64', - '--cache-repo', 'reg.io/wave/build/cache' ] - - cleanup: - ctx.close() - } - - def 'should disable compress-caching' () { - given: - def ctx = ApplicationContext.run(['wave.build.compress-caching': false]) - def service = ctx.getBean(DockerBuildStrategy) - and: - def req = new BuildRequest( - buildId: '1234', - workDir: Path.of('/work/foo/89fb83ce6ec8627b'), - platform: ContainerPlatform.of('linux/amd64'), - targetImage: 'repo:89fb83ce6ec8627b', - cacheRepository: 'reg.io/wave/build/cache' ) - when: - def cmd = service.launchCmd(req) - then: - cmd == ['build', + 'build', '--frontend', 'dockerfile.v0', '--local', From 1511ddb6e3f2e85a24b071be5c6a87ee135ec0d4 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 29 Aug 2024 11:30:03 +0200 Subject: [PATCH 19/25] fixed errors Signed-off-by: munishchouhan --- .../io/seqera/wave/service/builder/KubeBuildStrategy.groovy | 2 +- src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy index c9f185026e..c3b87fda63 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy @@ -96,7 +96,7 @@ class KubeBuildStrategy extends BuildStrategy { try { final buildImage = getBuildImage(req) final buildCmd = launchCmd(req) - final name = podName(req) + final name = podName(req.buildId) final timeout = req.maxDuration ?: buildConfig.defaultTimeout final selector= getSelectorLabel(req.platform, nodeSelectorMap) final spackCfg0 = req.isSpackBuild ? spackConfig : null diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy index 2993cf359b..0f99330f86 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy @@ -64,8 +64,6 @@ interface K8sService { InputStream getCurrentLogsPod(String name) - void deletePodWhenReachStatus(String podName, String statusName, long timeout) - V1Pod getLatestPodForJob(String jobName) } From 167a9ad4ab64f31624d41f4207055064b3a462e3 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Fri, 30 Aug 2024 11:09:17 +0200 Subject: [PATCH 20/25] fixed errors Signed-off-by: munishchouhan --- src/main/resources/io/seqera/wave/build-view.hbs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/resources/io/seqera/wave/build-view.hbs b/src/main/resources/io/seqera/wave/build-view.hbs index 0379f888c4..df9aaae664 100644 --- a/src/main/resources/io/seqera/wave/build-view.hbs +++ b/src/main/resources/io/seqera/wave/build-view.hbs @@ -193,12 +193,7 @@ const interval = setInterval(checkStatusAndFetchData, 2000); -

Build logs

- 
{{build_log_data}}
- {{#if build_log_truncated}} - Click here to download the complete build log - {{/if}} - {{/if}} +
From d6b19e56843fdca66fb2f409ab0955238d0545b2 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Tue, 22 Oct 2024 07:40:12 +0200 Subject: [PATCH 21/25] fixed error Signed-off-by: munishchouhan --- .../wave/service/builder/BuildStrategy.groovy | 2 +- .../service/builder/DockerBuildStrategy.groovy | 18 ++++-------------- .../service/builder/KubeBuildStrategy.groovy | 5 +++++ 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy index d2b64125a2..d41efdd1fb 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy @@ -35,7 +35,7 @@ abstract class BuildStrategy { @Inject private BuildConfig buildConfig - abstract BuildResult build(BuildRequest req) + abstract void build(String jobName, BuildRequest req) abstract InputStream getLogs(String buildId) diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index 694620a9ab..bd534e793a 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -108,7 +108,8 @@ class DockerBuildStrategy extends BuildStrategy { //checkout the documentation here to know more about these options https://github.com/moby/buildkit/blob/master/docs/rootless.md#docker final wrapper = ['docker', 'run', - '--rm', + '--detach', + '--name', name, '--privileged', '-v', "$workDir:$workDir".toString(), '--entrypoint', @@ -124,10 +125,6 @@ class DockerBuildStrategy extends BuildStrategy { wrapper.add(platform.toString()) } - //add builder name - wrapper.add('--name') - wrapper.add(builderName(buildId)) - // the container image to be used to build wrapper.add( buildConfig.buildkitImage ) // return it @@ -156,21 +153,14 @@ class DockerBuildStrategy extends BuildStrategy { wrapper.add(platform.toString()) } - //add builder name - wrapper.add('--name') - wrapper.add(builderName(buildId)) - wrapper.add(buildConfig.singularityImage(platform)) return wrapper } - protected static String builderName(String buildId) { - return "build-${buildId}".toString().replace('_', '-') - } @Override - InputStream getLogs(String buildId) { - def logCmd = ['docker', 'logs'] + builderName(buildId) + InputStream getLogs(String jobName) { + def logCmd = ['docker', 'logs'] + jobName log.info("Get build logs: ${logCmd.join(' ')}") final proc = new ProcessBuilder() .command(logCmd) diff --git a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy index f6ba6e041e..4b7d789d9e 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy @@ -98,6 +98,11 @@ class KubeBuildStrategy extends BuildStrategy { } } + @Override + InputStream getLogs(String podName) { + return k8sService.getCurrentLogsPod(podName) + } + protected String getBuildImage(BuildRequest buildRequest){ if( buildRequest.formatDocker() ) { return buildConfig.buildkitImage From ed403eef421b6bebec6c568e9dd8a639c0931cba Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Tue, 22 Oct 2024 07:59:20 +0200 Subject: [PATCH 22/25] fixed error Signed-off-by: munishchouhan --- .../io/seqera/wave/service/builder/KubeBuildStrategy.groovy | 5 +++-- .../wave/service/builder/ContainerBuildServiceTest.groovy | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy index 4b7d789d9e..084cb0074b 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy @@ -99,8 +99,9 @@ class KubeBuildStrategy extends BuildStrategy { } @Override - InputStream getLogs(String podName) { - return k8sService.getCurrentLogsPod(podName) + InputStream getLogs(String jobName) { + final pod = k8sService.getLatestPodForJob(jobName) + return k8sService.getCurrentLogsPod(pod.spec.containers.first().name) } protected String getBuildImage(BuildRequest buildRequest){ diff --git a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy index a7a0cc6b03..8303607d90 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy @@ -76,6 +76,11 @@ class ContainerBuildServiceTest extends Specification { // do nothing log.debug "Running fake build job=$jobName - request=$request" } + + @Override + InputStream getLogs(String podName) { + return "logs for pod name" + } } @Primary From 1c304b17861ebf3f304d22379631f7490ac30dd3 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Tue, 22 Oct 2024 10:07:35 +0200 Subject: [PATCH 23/25] fixed tests Signed-off-by: munishchouhan --- .../wave/service/builder/DockerBuildStrategyTest.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy index e9ccca0f4a..da39545f4e 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy @@ -41,12 +41,13 @@ class DockerBuildStrategyTest extends Specification { def work = Path.of('/work/foo') when: def cmd = service.cmdForBuildkit('build-job-name', work, null, null) + def name = 'build-job-name' then: cmd == ['docker', 'run', '--detach', '--name', - 'build-job-name', + name, '--privileged', '-v', '/work/foo:/work/foo', '--entrypoint', @@ -54,7 +55,7 @@ class DockerBuildStrategyTest extends Specification { 'moby/buildkit:v0.14.1-rootless'] when: - cmd = service.cmdForBuildkit(work, Path.of('/foo/creds.json'), null, ContainerPlatform.of('arm64')) + cmd = service.cmdForBuildkit(name, work, Path.of('/foo/creds.json'), ContainerPlatform.of('arm64')) then: cmd == ['docker', 'run', From f92a3019d6b12e927f635a1d479144e9fa36813e Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Wed, 15 Jan 2025 11:24:07 +0100 Subject: [PATCH 24/25] fixed error Signed-off-by: munishchouhan --- .../groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index 9fe07a621f..58fbde52bb 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -356,7 +356,11 @@ class K8sServiceImpl implements K8sService { @Override InputStream getCurrentLogsPod(String name) { try { - def logs = k8sClient.coreV1Api().readNamespacedPodLog(name, namespace, name, false, null, null, "false", false, null, null, null) + def logs = k8sClient.coreV1Api() + .readNamespacedPodLog(name, namespace) + .container(name) + .follow(false) + .execute() logs = logs ? logs.replaceAll("\u001B\\[[;\\d]*m", "") : null // strip ansi escape codes return new ByteArrayInputStream(logs.getBytes()) } catch (Exception e) { From 5942bd81dfd53fedb1cfb5f11e996e9b10bb66ff Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Wed, 15 Jan 2025 13:08:10 +0100 Subject: [PATCH 25/25] fixed tests Signed-off-by: munishchouhan --- .../seqera/wave/service/k8s/K8sServiceImplTest.groovy | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy index b58dee22a9..a3aaac0724 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy @@ -517,12 +517,19 @@ class K8sServiceImplTest extends Specification { def name = "builder-pod" def logs = "\u001B[31mINFO: Build is in progress" + def logRequest = Mock(CoreV1Api. APIreadNamespacedPodLogRequest) + def api = Mock(CoreV1Api) + when: InputStream result = k8sService.getCurrentLogsPod(name) then: - 2 * k8sClient.coreV1Api() >> Mock(CoreV1Api) - 1 * k8sClient.coreV1Api().readNamespacedPodLog(_, _, _, _, _, _, _, _, _, _, _) >> logs + 1 * k8sClient.coreV1Api() >> api + 1 * api.readNamespacedPodLog(name, _) >> logRequest + 1 * logRequest.container(_) >> logRequest + 1 * logRequest.follow(false) >> logRequest + 1 * logRequest.execute() >> logs + and: result instanceof ByteArrayInputStream String resultString = result.text