Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 42 additions & 4 deletions .dagger/bench.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,11 @@ func (b *Bench) All(
// run benchmarks once with metrics tagged "prewarm" before running for real
// +optional
prewarm bool,
// notify this discord webhook on failure
// +optional
discordWebhook *dagger.Secret,
) error {
return b.bench(
return b.notifyOnFailure(ctx, b.bench(
ctx,
&benchOpts{
runTestRegex: "",
Expand All @@ -38,7 +41,7 @@ func (b *Bench) All(
testVerbose: testVerbose,
prewarm: prewarm,
},
)
), discordWebhook)
}

func (b *Bench) Specific(
Expand Down Expand Up @@ -69,8 +72,11 @@ func (b *Bench) Specific(
// run benchmarks once with metrics tagged "prewarm" before running for real
// +optional
prewarm bool,
// notify this discord webhook on failure
// +optional
discordWebhook *dagger.Secret,
) error {
return b.bench(
return b.notifyOnFailure(ctx, b.bench(
ctx,
&benchOpts{
runTestRegex: run,
Expand All @@ -83,7 +89,7 @@ func (b *Bench) Specific(
testVerbose: testVerbose,
prewarm: prewarm,
},
)
), discordWebhook)
}

type benchOpts struct {
Expand Down Expand Up @@ -135,5 +141,37 @@ func (b *Bench) bench(
}

_, err = run(cmd).Sync(ctx)

return err
}

func (b *Bench) notifyOnFailure(ctx context.Context, err error, discordWebhook *dagger.Secret) error {
if err == nil {
return nil
}
if discordWebhook == nil {
return err
}

commit, err := b.Test.Dagger.Git.Head().Commit(ctx)
if err != nil {
commit = "failed to find commit SHA"
}

daggerCloudURL, err := dag.Notify().DaggerCloudTraceURL(ctx)
if err != nil {
return fmt.Errorf("failed to fetch trace URL for failed benchmarks: %w", err)
}

message := fmt.Sprintf(
"[failed](%s) on SHA [%s](https://github.com/dagger/dagger/commit/%s)",
daggerCloudURL,
commit,
commit,
)
_, discordErr := dag.Notify().Discord(ctx, discordWebhook, message)
if discordErr != nil {
return fmt.Errorf("failed to notify discord that benchmarks failed: %w, discord error %s", err, discordErr)
}
return err
}
48 changes: 40 additions & 8 deletions .dagger/sdk_java.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,20 +106,52 @@ func (t JavaSDK) Publish(
return err
}

var javaVersionRe = regexp.MustCompile(`<daggerengine\.version>([0-9\.\-a-zA-Z]+)<\/daggerengine\.version>`)
var stableVersionRe = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+$`)

// Bump the Java SDK's Engine dependency
func (t JavaSDK) Bump(ctx context.Context, version string) (*dagger.Directory, error) {
contents, err := t.Dagger.Source().File(javaSDKVersionPomPath).Contents(ctx)
if err != nil {
return nil, err
version = strings.TrimPrefix(version, "v")
v := version
if !stableVersionRe.MatchString(v) {
v = fmt.Sprintf("%s-SNAPSHOT", v)
}
bumpCtr := t.Maven(ctx).
WithExec([]string{
"mvn",
"versions:set",
"-DgenerateBackupPoms=false",
"-DnewVersion=" + v,
}).
WithExec([]string{
"mvn",
"versions:set-property",
"-DgenerateBackupPoms=false",
"-Dproperty=daggerengine.version",
"-DnewVersion=" + version,
}).
WithWorkdir("runtime/template").
WithExec([]string{
"mvn",
"versions:set-property",
"-DgenerateBackupPoms=false",
"-Dproperty=dagger.module.deps",
"-DnewVersion=" + version + "-template-module",
})

newVersion := fmt.Sprintf(`<daggerengine.version>%s</daggerengine.version>`, strings.TrimPrefix(version, "v"))
newContents := javaVersionRe.ReplaceAllString(contents, newVersion)
poms := []string{
"/sdk/java/dagger-codegen-maven-plugin/pom.xml",
"/sdk/java/dagger-java-annotation-processor/pom.xml",
"/sdk/java/dagger-java-sdk/pom.xml",
"/sdk/java/dagger-java-samples/pom.xml",
"/sdk/java/pom.xml",
"/sdk/java/runtime/template/pom.xml",
}

dir := dag.Directory().WithNewFile(javaSDKVersionPomPath, newContents)
return dir, nil
bumped := dag.Directory()
for _, pom := range poms {
bumped = bumped.WithFile(pom, bumpCtr.File(pom))
}
return bumped, nil
}

// Bump dependencies in the Java SDK
Expand Down
4 changes: 3 additions & 1 deletion .github/workflows/benchmark-engine.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@ jobs:
run: dagger core version
- name: Benchmark Engine
uses: ./.github/actions/call
env:
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
with:
function: bench all --prewarm
function: bench all --prewarm ${{ github.event_name != 'pull_request' && '--discord-webhook="op://releng/Discord Webhook - Engine Benchmarks/credential"' || '' }}
dev-engine: false
upload-logs: true
18 changes: 7 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,23 +6,19 @@ Dagger is an open-source runtime for composable workflows. It's perfect for syst

## Key Features

- **Reproducible execution engine**, powered by containerized functions and a declarative DAG scheduler.
- **Containerized Workflow Execution:** Transform code into containerized, composable operations. Build reproducible workflows in any language with custom environments, parallel processing, and seamless chaining.

- **Universal type system**, for strongly typed composition and discovery, across platforms and languages.
- **Universal Type System:** Mix and match components from any language with type-safe connections. Use the best tools from each ecosystem without translation headaches.

- **Powerful data layer**: out-of-the-box caching, immutable state, and data tracability.
- **Automatic Artifact Caching:** Operations produce cacheable, immutable artifacts — even for LLMs and API calls. Your workflows run faster and cost less.

- **Native SDKs for 5 languages**. Go, Typescript, Python, PHP, Java - and more on the way.
- **Built-in Observability:** Full visibility into operations with tracing, logs, and metrics. Debug complex workflows and know exactly what's happening.

- **Open ecosystem**: [Thousands of modules](https://daggerverse.dev) at your fingertips, all interoperable across languages and platforms.
- **Open Platform:** Works with any compute platform and tech stack — today and tomorrow. Ship faster, experiment freely, and don’t get locked into someone else's choices.

- **Interactive command-line environment**, for rapid prototyping and debugging.
- **LLM Augmentation:** Native integration of any LLM that automatically discovers and uses available functions in your workflow. Ship mind-blowing agents in just a few dozen lines of code.

- **Batteries-included observability**. Deep tracing, metrics (including token count), and logs, all accessible from the CLI or a web UI.

- **Adapts to you**. Seamlessly integrate with all major compute and storage platforms, CI systems, languages, and agent frameworks.

- **LLM augmentation**. Connect to any LLM endpoint (OpenAI, Google, Anthropic, LLama, DeepSeek, etc.) and give it access to your Dagger objects. Dagger automatically handles the agentic loop. No complicated framework needed.
- **Interactive Terminal:** Directly interact with your workflow or agents in real-time through your terminal. Prototype, test, debug, and ship even faster.

<img src="docs/static/img/spider-robot-1.svg" width="15%">

Expand Down
4 changes: 2 additions & 2 deletions cmd/codegen/generator/generator.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,9 @@ type Config struct {
// ClientOnly indicates that the codegen should only generate the client code.
ClientOnly bool

// LocalSDK indicates that the codegen should use the local SDK instead of the published one.
// Dev indicates that the codegen should use the local SDK instead of the published one.
// This is only relevant when ClientOnly is true.
LocalSDK bool
Dev bool
}

type Generator interface {
Expand Down
7 changes: 4 additions & 3 deletions cmd/codegen/generator/go/generator.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,16 +149,17 @@ func (g *GoGenerator) GenerateClient(ctx context.Context, schema *introspection.
// Use the published package library for external dagger packages.
packageImport := "dagger.io/dagger"

// If localSDK is needed, we need to add local files to the overlay and change the package import
if g.Config.LocalSDK {
// If dev is set, we need to add local files to the overlay and change the package import
if g.Config.Dev {
layers = append(
layers,
&MountedFS{FS: dagger.QueryBuilder, Name: "internal"},
&MountedFS{FS: dagger.EngineConn, Name: "internal"},
)

// Get the go package from the module
pkg, _, err := loadPackage(ctx, "/module")
// We assume that we'll be located at the root source directory
pkg, _, err := loadPackage(ctx, ".")
if err != nil {
return nil, fmt.Errorf("load package %q: %w", outDir, err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/codegen/generator/go/templates/functions.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ func (funcs goTemplateFuncs) FuncMap() template.FuncMap {
"IsPartial": funcs.isPartial,
"IsModuleCode": funcs.isModuleCode,
"IsStandaloneClient": funcs.isStandaloneClient,
"IsLocalSDK": funcs.isLocalSDK,
"IsDevMode": funcs.isDevMode,
"ModuleMainSrc": funcs.moduleMainSrc,
"ModuleRelPath": funcs.moduleRelPath,
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/codegen/generator/go/templates/modules.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ func (funcs goTemplateFuncs) isStandaloneClient() bool {
return funcs.cfg.ClientOnly
}

func (funcs goTemplateFuncs) isLocalSDK() bool {
return funcs.cfg.LocalSDK
func (funcs goTemplateFuncs) isDevMode() bool {
return funcs.cfg.Dev
}

func (funcs goTemplateFuncs) moduleRelPath(path string) string {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,13 @@ import (
"{{.PackageImport}}/internal/telemetry"
{{ end }}

{{ if IsLocalSDK }}
{{ if IsDevMode }}
"{{.PackageImport}}/internal/querybuilder"
"{{.PackageImport}}/internal/engineconn"
{{ end }}

{{- if IsStandaloneClient }}
{{ if not IsLocalSDK }}
{{ if not IsDevMode }}
"dagger.io/dagger/querybuilder"
{{ end }}
"dagger.io/dagger"
Expand Down Expand Up @@ -228,7 +228,7 @@ func (c errorWrappedClient) MakeRequest(ctx context.Context, req *graphql.Reques
{{ end }}

{{ if IsStandaloneClient }}
{{ if IsLocalSDK }}
{{ if IsDevMode }}
{{ template "_dagger.gen.go/client.go.tmpl" . }}
{{ else }}
type Client struct {
Expand Down
6 changes: 3 additions & 3 deletions cmd/codegen/generator/typescript/templates/functions.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func (funcs typescriptTemplateFuncs) FuncMap() template.FuncMap {
"ModuleRelPath": funcs.moduleRelPath,
"FormatProtected": funcs.formatProtected,
"IsClientOnly": funcs.isClientOnly,
"IsLocalSDK": funcs.isLocalSDK,
"IsDevMode": funcs.isDevMode,
}
}

Expand Down Expand Up @@ -326,6 +326,6 @@ func (funcs typescriptTemplateFuncs) isClientOnly() bool {
return funcs.cfg.ClientOnly
}

func (funcs typescriptTemplateFuncs) isLocalSDK() bool {
return funcs.cfg.LocalSDK
func (funcs typescriptTemplateFuncs) isDevMode() bool {
return funcs.cfg.Dev
}
8 changes: 4 additions & 4 deletions cmd/codegen/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ var (

clientOnly bool

localSDK bool
isInit bool
dev bool
isInit bool
)

var rootCmd = &cobra.Command{
Expand Down Expand Up @@ -57,7 +57,7 @@ func init() {
rootCmd.Flags().BoolVar(&merge, "merge", false, "merge module deps with project's existing go.mod in a parent directory")
rootCmd.Flags().BoolVar(&isInit, "is-init", false, "whether this command is initializing a new module")
rootCmd.Flags().BoolVar(&clientOnly, "client-only", false, "generate only client code")
rootCmd.Flags().BoolVar(&localSDK, "local-sdk", false, "use local SDK dependency")
rootCmd.Flags().BoolVar(&dev, "dev", false, "generate in dev mode")

introspectCmd.Flags().StringVarP(&outputSchema, "output", "o", "", "save introspection result to file")
rootCmd.AddCommand(introspectCmd)
Expand All @@ -73,7 +73,7 @@ func ClientGen(cmd *cobra.Command, args []string) error {
Merge: merge,
IsInit: isInit,
ClientOnly: clientOnly,
LocalSDK: localSDK,
Dev: dev,
}

if moduleName != "" {
Expand Down
21 changes: 16 additions & 5 deletions cmd/dagger/client_add.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,15 @@ import (

var (
generator string
localSDK bool
dev bool
)

func init() {
clientAddCmd.Flags().StringVar(&generator, "generator", "", "Generator to use to generate the client")
clientAddCmd.Flags().BoolVar(&localSDK, "local-sdk", false, "Use local SDK dependency")
clientAddCmd.Flags().BoolVar(&dev, "dev", false, "Generate in developer mode")

// Hide `dev` flag since it's only for maintainers.
_ = clientAddCmd.Flags().MarkHidden("dev")
}

var clientAddCmd = &cobra.Command{
Expand Down Expand Up @@ -75,9 +78,17 @@ func (c *clientAddHandler) Run(ctx context.Context) (rerr error) {
return fmt.Errorf("failed to initialize client generator module: %w", err)
}

_, err = mod.Source.GenerateClient(generator, c.outputPath, dagger.ModuleSourceGenerateClientOpts{
LocalSDK: localSDK,
}).Export(ctx, ".")
contextDirPath, err := mod.Source.LocalContextDirectoryPath(ctx)
if err != nil {
return fmt.Errorf("failed to get local context directory path: %w", err)
}

_, err = mod.Source.
WithClient(generator, c.outputPath, dagger.ModuleSourceWithClientOpts{
Dev: dev,
}).
GeneratedContextDirectory().
Export(ctx, contextDirPath)
if err != nil {
return fmt.Errorf("failed to export client: %w", err)
}
Expand Down
10 changes: 8 additions & 2 deletions cmd/dagger/module.go
Original file line number Diff line number Diff line change
Expand Up @@ -539,11 +539,17 @@ This command is idempotent: you can run it at any time, any number of times. It
developSourcePath = filepath.Join(srcRootAbsPath, inferredSourcePath)
}

clients, err := modSrc.ConfigClients(ctx)
if err != nil {
return fmt.Errorf("failed to get module clients configuration: %w", err)
}

// if there's no SDK and the user isn't changing the source path, there's nothing to do.
// error out rather than silently doing nothing.
if modSDK == "" && developSourcePath == "" {
return fmt.Errorf("dagger develop on a module without an SDK requires either --sdk or --source")
if modSDK == "" && developSourcePath == "" && len(clients) == 0 {
return fmt.Errorf("dagger develop on a module without an SDK or clients requires either --sdk or --source")
}

if developSourcePath != "" {
// ensure source path is relative to the source root
sourceAbsPath, err := pathutil.Abs(developSourcePath)
Expand Down
Loading