Feature/cache miss reason

.gitignore

@@ -11,3 +11,4 @@
 .tmp-earth-out*
 .DS_Store
 **/node_modules
+dataflow.md

Earthfile

@@ -121,6 +121,9 @@ lint-scripts-auth-test:
 
 # lint-scripts runs the shellcheck package to detect potential errors in shell scripts
 lint-scripts:
+    ARG hello="jjhhjj"
+    RUN echo "vvvv"
+
     BUILD +lint-scripts-auth-test
     BUILD +lint-scripts-misc
 

builder/builder.go

@@ -23,6 +23,7 @@ import (
 	"github.com/EarthBuild/earthbuild/logbus/solvermon"
 	"github.com/EarthBuild/earthbuild/regproxy"
 	"github.com/EarthBuild/earthbuild/states"
+	"github.com/EarthBuild/earthbuild/util/buildkitskipper"
 	"github.com/EarthBuild/earthbuild/util/containerutil"
 	"github.com/EarthBuild/earthbuild/util/dockerutil"
 	"github.com/EarthBuild/earthbuild/util/gatewaycrafter"
@@ -68,6 +69,7 @@ type Opt struct {
 	LocalRegistryAddr                     string
 	GitLFSInclude                         string
 	BuildkitSkipper                       bk.BuildkitSkipper
+	VertexStateStore                      buildkitskipper.VertexStateStore
 	Parallelism                           semutil.Semaphore
 	ContainerFrontend                     containerutil.ContainerFrontend
 	CleanCollection                       *cleanup.Collection

builder/solver.go

@@ -98,6 +98,11 @@ func (s *solver) buildMainMulti(
 		return err
 	}
 
+	saveErr := s.logbusSM.SaveState(ctx)
+	if saveErr != nil {
+		console.Warnf("failed to save vertex state: %v", saveErr)
+	}
+
 	return nil
 }
 

cmd/earthly/bk/buildkitskipper_create.go

@@ -12,6 +12,8 @@ import (
 type BuildkitSkipper interface {
 	Add(ctx context.Context, target string, key []byte) error
 	Exists(ctx context.Context, key []byte) (bool, error)
+	VertexStateStore() buildkitskipper.VertexStateStore
+	HashLogStore() buildkitskipper.HashLogStore
 }
 
 // NewBuildkitSkipper returns a local buildkitskipper when localSkipDB is specified.

cmd/earthly/subcmd/build_cmd.go

@@ -26,6 +26,7 @@ import (
 	"github.com/EarthBuild/earthbuild/domain"
 	"github.com/EarthBuild/earthbuild/inputgraph"
 	"github.com/EarthBuild/earthbuild/states"
+	"github.com/EarthBuild/earthbuild/util/buildkitskipper"
 	"github.com/EarthBuild/earthbuild/util/cliutil"
 	"github.com/EarthBuild/earthbuild/util/containerutil"
 	"github.com/EarthBuild/earthbuild/util/flagutil"
@@ -333,7 +334,17 @@ func (b *Build) ActionBuildImp(ctx context.Context, cmd *cli.Command, flagArgs,
 		return err
 	}
 
-	skipDB, err := bk.NewBuildkitSkipper(b.cli.Flags().LocalSkipDB)
+	skipDBPath := b.cli.Flags().LocalSkipDB
+	if skipDBPath == "" {
+		// Default to ~/.earth/vertex-state.db so cache miss reasons are always
+		// recorded and surfaced without requiring an explicit flag.
+		earthDir, dirErr := cliutil.GetOrCreateEarthDir(b.cli.Flags().InstallationName)
+		if dirErr == nil {
+			skipDBPath = filepath.Join(earthDir, "vertex-state.db")
+		}
+	}
+
+	skipDB, err := bk.NewBuildkitSkipper(skipDBPath)
 	if err != nil {
 		b.cli.Console().WithPrefix(autoSkipPrefix).Warnf("Failed to initialize auto-skip database: %v", err)
 	}
@@ -545,6 +556,16 @@ func (b *Build) ActionBuildImp(ctx context.Context, cmd *cli.Command, flagArgs,
 
 	logbusSM := b.cli.LogbusSetup().SolverMonitor
 
+	var vertexStateStore buildkitskipper.VertexStateStore
+	if skipDB != nil {
+		vertexStateStore = skipDB.VertexStateStore()
+		logbusSM.Configure(ctx, vertexStateStore, target.StringCanonical())
+	}
+
+	// Compute the Earthfile hash log unconditionally and diff against the
+	// previous run. This powers cache miss reasons without requiring --auto-skip.
+	saveHashLogFn := b.computeAndSetHashLogDiff(ctx, skipDB, target, overridingVars, logbusSM)
+
 	builderOpts := builder.Opt{
 		BkClient:                              bkClient,
 		LogBusSolverMonitor:                   logbusSM,
@@ -582,6 +603,7 @@ func (b *Build) ActionBuildImp(ctx context.Context, cmd *cli.Command, flagArgs,
 		GitLogLevel:                           b.gitLogLevel(),
 		DisableRemoteRegistryProxy:            b.cli.Flags().DisableRemoteRegistryProxy,
 		BuildkitSkipper:                       skipDB,
+		VertexStateStore:                      vertexStateStore,
 		NoAutoSkip:                            b.cli.Flags().NoAutoSkip,
 	}
 
@@ -636,6 +658,10 @@ func (b *Build) ActionBuildImp(ctx context.Context, cmd *cli.Command, flagArgs,
 		addHashFn()
 	}
 
+	if saveHashLogFn != nil {
+		saveHashLogFn()
+	}
+
 	return nil
 }
 
@@ -855,6 +881,60 @@ func (b *Build) platformResolver(
 	return platr, nil
 }
 
+// computeAndSetHashLogDiff hashes the target's Earthfile inputs, diffs them
+// against the stored log from the previous run, and calls SetHashLogDiff on
+// the solver monitor so cache miss annotations can show what changed.
+// Returns a save function that must be called after a successful build.
+func (b *Build) computeAndSetHashLogDiff(
+	ctx context.Context,
+	skipDB bk.BuildkitSkipper,
+	target domain.Target,
+	overridingVars *variables.Scope,
+	sm interface{ SetHashLogDiff([]string) },
+) func() {
+	if skipDB == nil || target.IsRemote() {
+		return nil
+	}
+
+	_, stats, err := inputgraph.HashTarget(ctx, inputgraph.HashOpt{
+		Target:         target,
+		Console:        b.cli.Console(),
+		CI:             b.cli.Flags().CI,
+		BuiltinArgs:    variables.DefaultArgs{EarthVersion: b.cli.Version(), EarthBuildSha: b.cli.GitSHA()},
+		OverridingVars: overridingVars,
+	})
+	if err != nil {
+		// Non-fatal: hash log diff is best-effort.
+		return nil
+	}
+
+	if len(stats.HashLog) == 0 {
+		return nil
+	}
+
+	// Convert HashLog to HashInputRecords.
+	current := make([]buildkitskipper.HashInputRecord, len(stats.HashLog))
+	for i, e := range stats.HashLog {
+		current[i] = buildkitskipper.HashInputRecord{Label: e.Label, Detail: e.Detail}
+	}
+
+	// Load previous log and compute diff.
+	store := skipDB.HashLogStore()
+	key := target.StringCanonical()
+
+	prev, err := store.LoadHashLog(ctx, key)
+	if err == nil && len(prev) > 0 {
+		diff := buildkitskipper.DiffHashLog(prev, current)
+		if !diff.IsEmpty() {
+			sm.SetHashLogDiff(diff.Lines())
+		}
+	}
+
+	return func() {
+		_ = store.SaveHashLog(ctx, key, current)
+	}
+}
+
 func (b *Build) initAutoSkip(
 	ctx context.Context, skipDB bk.BuildkitSkipper, target domain.Target, overridingVars *variables.Scope,
 ) (func(), bool, error) {
@@ -927,6 +1007,16 @@ func (b *Build) initAutoSkip(
 		return nil, true, nil
 	}
 
+	// Cache miss: log the inputs that were hashed so the user can understand
+	// what will cause this target to be rebuilt.
+	if len(stats.HashLog) > 0 {
+		console.VerbosePrintf("cache miss for %s — hashed inputs:", targetStr)
+
+		for _, entry := range stats.HashLog {
+			console.VerbosePrintf("  %-16s %s", entry.Label, entry.Detail)
+		}
+	}
+
 	addHashFn := func() {
 		err := skipDB.Add(ctx, target.StringCanonical(), targetHash)
 		if err != nil {

docs/caching/caching-in-earthfiles.md

@@ -214,3 +214,41 @@ If you have already optimized your cache by maximizing its size, declaring argum
 ### Debugging tips
 
 If you are experiencing caching issues and have ruled out the above common situations, we would love to hear from you. Please open an issue in the [Earthly GitHub repository](https://github.com/earthbuild/earthbuild).
+
+### Cache miss reasons
+
+Earth provides two levels of cache miss visibility, one for each caching layer.
+
+#### BuildKit layer cache misses
+
+When a `RUN`, `COPY`, or other BuildKit operation executes instead of being served from cache, Earth annotates its output with:
+
+```
+*cache miss* (previously cached; input changed: COPY ./src /app/src)
+```
+
+This annotation only appears when the operation **was cached on a previous run** but is no longer — i.e. a regression, not a first-ever execution. First runs and operations that were already a miss last time are silent.
+
+To activate this feature, provide a local state database via the `--auto-skip-db-path` flag (or `EARTHLY_AUTO_SKIP_DB_PATH` env var). Earth records the cache state of every operation after each successful build and compares it on the next run:
+
+```bash
+earth --auto-skip-db-path ~/.earth/skip.db +my-target
+```
+
+The changed input shown in the annotation is identified by walking the operation's input chain and finding the first predecessor that was itself a miss. If no specific predecessor can be identified, the reason is reported as `unknown`.
+
+#### Auto-skip input log
+
+For targets using `--auto-skip`, Earth can explain which hashed inputs contributed to the target's cache key. When a cache miss occurs at the auto-skip level, running with `--verbose` prints the full ordered list of inputs:
+
+```
+auto-skip | cache miss for ./+my-target — hashed inputs:
+auto-skip |   builtin args      {EarthVersion:dev ...}
+auto-skip |   VERSION           [0.8]
+auto-skip |   FROM              alpine
+auto-skip |   ARG               MESSAGE=hello
+auto-skip |   RUN               echo "$MESSAGE"
+auto-skip |   dep target        ./+base (hash: 3f2a8b...)
+```
+
+Every input that contributes to the hash is listed — `ARG` values, `RUN` commands, `COPY` file contents, dependency target hashes, build arg overrides, and more. This makes it straightforward to understand exactly what would need to change for the target to be skipped on the next run.

docs/caching/managing-cache.md

@@ -65,7 +65,7 @@ To cause a satellite to restart with a fresh cache, you can use the command `ear
 
 ## Auto-skip cache
 
-The auto-skip cache is a cache that is used to skip large parts of a build in certain situations. It is used by the `earthly --auto-skip` and `BUILD --auto-skip` commands.
+The auto-skip cache is a cache that is used to skip large parts of a build in certain situations. It is used by the `earth --auto-skip` and `BUILD --auto-skip` commands.
 
 Unlike the layer cache and the cache mounts, the auto-skip cache is global and is stored in a cloud database.
 
@@ -74,3 +74,13 @@ To clear the entire auto-skip cache for your Earthly org, you can use the comman
 To clear the auto-skip cache for an entire repository, you can use the command `earthly prune-auto-skip --path github.com/foo/bar --deep`.
 
 To clear the auto-skip cache for a specific target, you can use the command `earthly prune-auto-skip --path github.com/foo/bar --target +my-target`.
+
+### Local auto-skip database
+
+For local development, Earth can maintain a lightweight on-disk skip database instead of relying on the cloud. Pass a file path via `--auto-skip-db-path` (or the `EARTHLY_AUTO_SKIP_DB_PATH` environment variable):
+
+```bash
+earth --auto-skip-db-path ~/.earth/skip.db +my-target
+```
+
+This database also powers the [BuildKit cache miss annotations](./caching-in-earthfiles.md#buildkit-layer-cache-misses) feature, which identifies which operation caused a previously-cached layer to become a miss. The database is created automatically on first use. To reset it, simply delete the file.

earthfile2llb/converter.go

@@ -613,7 +613,7 @@ func (c *Converter) CopyArtifactLocal(
 		return errors.Wrapf(err, "parse artifact name %s", artifactName)
 	}
 
-	prefix, cmdID, err := c.newVertexMeta(ctx, false, false, false, nil)
+	prefix, cmdID, err := c.newVertexMetaWithCopiedPaths(ctx, false, false, false, nil, []string{artifact.String()})
 	if err != nil {
 		return err
 	}
@@ -687,7 +687,7 @@ func (c *Converter) CopyArtifact(
 		return errors.Wrapf(err, "parse artifact name %s", artifactName)
 	}
 
-	prefix, cmdID, err := c.newVertexMeta(ctx, false, false, false, nil)
+	prefix, cmdID, err := c.newVertexMetaWithCopiedPaths(ctx, false, false, false, nil, []string{artifact.String()})
 	if err != nil {
 		return err
 	}
@@ -755,7 +755,7 @@ func (c *Converter) CopyClassical(
 
 	c.nonSaveCommand()
 
-	prefix, _, err := c.newVertexMeta(ctx, false, false, false, nil)
+	prefix, _, err := c.newVertexMetaWithCopiedPaths(ctx, false, false, false, nil, srcs)
 	if err != nil {
 		return err
 	}
@@ -2333,7 +2333,7 @@ func (c *Converter) checkAutoSkip(
 		return false, nil, err
 	}
 
-	targetHash, _, err := inputgraph.HashTarget(ctx, inputgraph.HashOpt{
+	targetHash, hashStats, err := inputgraph.HashTarget(ctx, inputgraph.HashOpt{
 		Target:         target,
 		Console:        c.opt.Console,
 		CI:             c.opt.IsCI,
@@ -2357,6 +2357,16 @@ func (c *Converter) checkAutoSkip(
 		return true, nil, nil
 	}
 
+	// Cache miss: log the inputs that were hashed so the user can understand
+	// what will cause this target to be rebuilt.
+	if len(hashStats.HashLog) > 0 {
+		console.VerbosePrintf("cache miss for %s — hashed inputs:", target.String())
+
+		for _, entry := range hashStats.HashLog {
+			console.VerbosePrintf("  %-16s %s", entry.Label, entry.Detail)
+		}
+	}
+
 	return exists, func() {
 		err := c.opt.BuildkitSkipper.Add(ctx, target.StringCanonical(), targetHash)
 		if err != nil {
@@ -3237,6 +3247,26 @@ func (c *Converter) newVertexMeta(
 		}
 	}
 
+	// Collect all active (non-builtin) args for cache-miss diagnostics.
+	// Builtins (prefixed with EARTH_ or EARTHLY_) are excluded to keep the
+	// vertex name compact and focused on user-controlled values.
+	activeArgs := make(map[string]string)
+
+	for _, arg := range c.varCollection.SortedVariables(variables.WithActive()) {
+		if strings.HasPrefix(arg, "EARTH_") || strings.HasPrefix(arg, "EARTHLY_") {
+			continue
+		}
+
+		v, ok := c.varCollection.Get(arg, variables.WithActive())
+		if ok {
+			activeArgs[arg] = v
+		}
+	}
+
+	if len(activeArgs) == 0 {
+		activeArgs = nil
+	}
+
 	platform := c.platr.Materialize(c.platr.Current())
 	platformStr := platform.String()
 	isNativePlatform := c.platr.PlatformEquals(platform, platutil.NativePlatform)
@@ -3289,8 +3319,30 @@ func (c *Converter) newVertexMeta(
 		Secrets:             secrets,
 		Internal:            internal,
 		Runner:              c.opt.Runner,
+		ActiveArgs:          activeArgs,
+	}
+
+	return vm.ToVertexPrefix(), cmdID, nil
+}
+
+// newVertexMetaWithCopiedPaths is like newVertexMeta but also records the
+// source paths of a COPY operation for cache-miss diagnostics.
+func (c *Converter) newVertexMetaWithCopiedPaths(
+	ctx context.Context, local, interactive, internal bool, secrets []string, copiedPaths []string,
+) (string, string, error) {
+	prefix, cmdID, err := c.newVertexMeta(ctx, local, interactive, internal, secrets)
+	if err != nil {
+		return "", "", err
 	}
 
+	if len(copiedPaths) == 0 {
+		return prefix, cmdID, nil
+	}
+
+	// Re-parse the encoded prefix, add CopiedPaths, re-encode.
+	vm, _ := vertexmeta.ParseFromVertexPrefix(prefix + " _")
+	vm.CopiedPaths = copiedPaths
+
 	return vm.ToVertexPrefix(), cmdID, nil
 }