expand gomod paths with glob patterns

frontend/debug/handle_gomod.go

@@ -39,6 +39,10 @@ func Gomods(ctx context.Context, client gwclient.Client) (*gwclient.Result, erro
 		}
 		worker = worker.With(addedHosts(client))
 
+		if err := spec.Preprocess(sOpt, worker, dalec.Platform(platform), pg); err != nil {
+			return nil, nil, err
+		}
+
 		st := spec.GomodDeps(sOpt, worker, dalec.Platform(platform), pg)
 
 		def, err := st.Marshal(ctx)

frontend/gateway.go

@@ -3,6 +3,7 @@ package frontend
 import (
 	"context"
 	"fmt"
+	"io/fs"
 	"sync"
 	"sync/atomic"
 
@@ -14,7 +15,9 @@ import (
 	"github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
+
 	"github.com/project-dalec/dalec"
+	"github.com/project-dalec/dalec/frontend/pkg/bkfs"
 )
 
 const (
@@ -128,6 +131,14 @@ func SourceOptFromUIClient(ctx context.Context, c gwclient.Client, dc *dockerui.
 			return st, err
 		},
 		GitCredHelperOpt: withCredHelper(c),
+		Glob: func(st llb.State, pattern string) ([]string, error) {
+			fsys, err := bkfs.FromState(ctx, &st, c)
+			if err != nil {
+				return nil, err
+			}
+
+			return fs.Glob(fsys, pattern)
+		},
 	}
 
 	sOpt.SourceFilter = sync.OnceValues(func() (dalec.SourceFilterConfig, error) {

preprocess.go

@@ -4,6 +4,7 @@ import (
 	_ "embed"
 	"fmt"
 	"path/filepath"
+	"slices"
 	"strings"
 
 	"golang.org/x/mod/module"
@@ -31,13 +32,67 @@ const (
 // Preprocessing generates LLB states for patches and registers them as context sources
 // that can be retrieved later when sources are fetched.
 func (s *Spec) Preprocess(sOpt SourceOpts, worker llb.State, opts ...llb.ConstraintsOpt) error {
+	if err := s.preprocessGomodPaths(sOpt, worker, opts...); err != nil {
+		return errors.Wrap(err, "failed to preprocess gomod paths")
+	}
+
 	if err := s.preprocessGomodEdits(sOpt, worker, opts...); err != nil {
 		return errors.Wrap(err, "failed to preprocess gomod edits")
 	}
 
 	return nil
 }
 
+// preprocessGomodPaths expands glob patterns in GeneratorGomod.Paths against
+// the source content.
+func (s *Spec) preprocessGomodPaths(sOpt SourceOpts, worker llb.State, opts ...llb.ConstraintsOpt) error {
+	gomodSources := s.gomodSources()
+	if len(gomodSources) == 0 {
+		return nil
+	}
+
+	// Get sources with base patches applied
+	patchedSources := s.getPatchedSources(sOpt, worker, func(name string) bool {
+		_, ok := gomodSources[name]
+		return ok
+	}, opts...)
+
+	// Expand paths for each source with gomod generators
+	for sourceName, src := range gomodSources {
+		patchedState, ok := patchedSources[sourceName]
+		if !ok {
+			continue
+		}
+
+		for _, gen := range src.Generate {
+			if gen == nil || gen.Gomod == nil || gen.Gomod.Paths == nil {
+				continue
+			}
+
+			basePath := filepath.Join(sourceName, gen.Subpath)
+			oldPaths := gen.Gomod.Paths
+
+			newPaths := make([]string, 0, len(oldPaths))
+			for _, pattern := range oldPaths {
+				matches, err := sOpt.Glob(patchedState, filepath.Join(basePath, pattern))
+				if err != nil {
+					return err
+				}
+
+				for _, match := range matches {
+					newPath := "." + strings.TrimPrefix(match, basePath)
+					newPaths = append(newPaths, newPath)
+				}
+			}
+			slices.Sort(newPaths)
+
+			gen.Gomod.Paths = newPaths
+		}
+	}
+
+	return nil
+}
+
 // preprocessGomodEdits generates patch LLB states for all gomod replace directives
 // and registers them as context sources that can be retrieved later.
 func (s *Spec) preprocessGomodEdits(sOpt SourceOpts, worker llb.State, opts ...llb.ConstraintsOpt) error {

preprocess_test.go

@@ -0,0 +1,204 @@
+package dalec
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/moby/buildkit/client/llb"
+	"gotest.tools/v3/assert"
+)
+
+func TestPreprocessGomodPaths(t *testing.T) {
+	t.Parallel()
+
+	t.Run("bare wildcard globs from the source root", func(t *testing.T) {
+		t.Parallel()
+
+		spec := newSpecWithGomod(&GeneratorGomod{Paths: []string{"*"}})
+
+		var gotPattern string
+		sOpt := newGlobSourceOpts(func(st llb.State, pattern string) ([]string, error) {
+			gotPattern = pattern
+			return []string{"foo/module1", "foo/module2"}, nil
+		})
+
+		err := spec.preprocessGomodPaths(sOpt, llb.Scratch())
+		assert.NilError(t, err)
+
+		gen := spec.Sources["foo"].Generate[0].Gomod
+		assert.DeepEqual(t, gen.Paths, []string{"./module1", "./module2"})
+		assert.Equal(t, gotPattern, "foo/*")
+	})
+
+	t.Run("every entry is globbed, including literal paths", func(t *testing.T) {
+		t.Parallel()
+
+		spec := newSpecWithGomod(&GeneratorGomod{Paths: []string{".", "plugins/*"}})
+
+		var gotPatterns []string
+		sOpt := newGlobSourceOpts(func(st llb.State, pattern string) ([]string, error) {
+			gotPatterns = append(gotPatterns, pattern)
+			switch pattern {
+			case "foo":
+				return []string{"foo"}, nil
+			case "foo/plugins/*":
+				return []string{"foo/plugins/v1"}, nil
+			default:
+				t.Fatalf("unexpected pattern %q", pattern)
+				return nil, nil
+			}
+		})
+
+		err := spec.preprocessGomodPaths(sOpt, llb.Scratch())
+		assert.NilError(t, err)
+
+		gen := spec.Sources["foo"].Generate[0].Gomod
+		assert.DeepEqual(t, gen.Paths, []string{".", "./plugins/v1"})
+		assert.DeepEqual(t, gotPatterns, []string{"foo", "foo/plugins/*"})
+	})
+
+	t.Run("results are fully sorted regardless of input order", func(t *testing.T) {
+		t.Parallel()
+
+		spec := newSpecWithGomod(&GeneratorGomod{Paths: []string{"moduleB/*", "moduleA/*"}})
+
+		sOpt := newGlobSourceOpts(func(st llb.State, pattern string) ([]string, error) {
+			switch pattern {
+			case "foo/moduleA/*":
+				return []string{"foo/moduleA/v2", "foo/moduleA/v1"}, nil
+			case "foo/moduleB/*":
+				return []string{"foo/moduleB/v1"}, nil
+			default:
+				t.Fatalf("unexpected pattern %q", pattern)
+				return nil, nil
+			}
+		})
+
+		err := spec.preprocessGomodPaths(sOpt, llb.Scratch())
+		assert.NilError(t, err)
+
+		gen := spec.Sources["foo"].Generate[0].Gomod
+		assert.DeepEqual(t, gen.Paths, []string{"./moduleA/v1", "./moduleA/v2", "./moduleB/v1"})
+	})
+
+	t.Run("combines with the generator's Subpath", func(t *testing.T) {
+		t.Parallel()
+
+		spec := newSpecWithGomod(&GeneratorGomod{Paths: []string{"plugins/*"}})
+		spec.Sources["foo"].Generate[0].Subpath = "some/nested/dir"
+
+		var gotPattern string
+		sOpt := newGlobSourceOpts(func(st llb.State, pattern string) ([]string, error) {
+			gotPattern = pattern
+			return []string{"foo/some/nested/dir/plugins/v1"}, nil
+		})
+
+		err := spec.preprocessGomodPaths(sOpt, llb.Scratch())
+		assert.NilError(t, err)
+
+		gen := spec.Sources["foo"].Generate[0].Gomod
+		assert.DeepEqual(t, gen.Paths, []string{"./plugins/v1"})
+		assert.Equal(t, gotPattern, "foo/some/nested/dir/plugins/*")
+	})
+
+	t.Run("does not touch or glob sources with no Paths", func(t *testing.T) {
+		t.Parallel()
+
+		spec := newSpecWithGomod(&GeneratorGomod{})
+
+		sOpt := newGlobSourceOpts(func(st llb.State, pattern string) ([]string, error) {
+			t.Fatal("FSGlob should not be called when Paths is empty")
+			return nil, nil
+		})
+
+		err := spec.preprocessGomodPaths(sOpt, llb.Scratch())
+		assert.NilError(t, err)
+		assert.Assert(t, spec.Sources["foo"].Generate[0].Gomod.Paths == nil)
+	})
+
+	t.Run("is a no-op for specs without any gomod generator", func(t *testing.T) {
+		t.Parallel()
+
+		spec := &Spec{Sources: map[string]Source{"foo": {Git: &SourceGit{URL: "https://localhost/test.git", Commit: "deadbeef"}}}}
+
+		err := spec.preprocessGomodPaths(SourceOpts{}, llb.Scratch())
+		assert.NilError(t, err)
+	})
+
+	t.Run("a pattern matching nothing is silently dropped", func(t *testing.T) {
+		t.Parallel()
+
+		spec := newSpecWithGomod(&GeneratorGomod{Paths: []string{"module1/*", "typo/*"}})
+
+		sOpt := newGlobSourceOpts(func(st llb.State, pattern string) ([]string, error) {
+			if pattern == "foo/module1/*" {
+				return []string{"foo/module1/v1"}, nil
+			}
+			return nil, nil
+		})
+
+		err := spec.preprocessGomodPaths(sOpt, llb.Scratch())
+		assert.NilError(t, err)
+
+		gen := spec.Sources["foo"].Generate[0].Gomod
+		assert.DeepEqual(t, gen.Paths, []string{"./module1/v1"})
+	})
+
+	t.Run("Paths ends up a non-nil empty slice if every pattern matches nothing", func(t *testing.T) {
+		t.Parallel()
+
+		spec := newSpecWithGomod(&GeneratorGomod{Paths: []string{"typo/*"}})
+
+		sOpt := newGlobSourceOpts(func(st llb.State, pattern string) ([]string, error) {
+			return nil, nil
+		})
+
+		err := spec.preprocessGomodPaths(sOpt, llb.Scratch())
+		assert.NilError(t, err)
+
+		gen := spec.Sources["foo"].Generate[0].Gomod
+		assert.Assert(t, gen.Paths != nil)
+		assert.DeepEqual(t, gen.Paths, []string{})
+	})
+
+	t.Run("propagates glob errors", func(t *testing.T) {
+		t.Parallel()
+
+		spec := newSpecWithGomod(&GeneratorGomod{Paths: []string{"*"}})
+
+		sOpt := newGlobSourceOpts(func(st llb.State, pattern string) ([]string, error) {
+			return nil, errors.New("boom")
+		})
+
+		err := spec.preprocessGomodPaths(sOpt, llb.Scratch())
+		assert.ErrorContains(t, err, "boom")
+	})
+}
+
+func newSpecWithGomod(gen *GeneratorGomod) *Spec {
+	return &Spec{
+		Sources: map[string]Source{
+			"foo": {
+				Git: &SourceGit{
+					URL:    "https://localhost/bar.git",
+					Commit: "deadbeef",
+				},
+				Generate: []*SourceGenerator{
+					{
+						Gomod: gen,
+					},
+				},
+			},
+		},
+	}
+}
+
+func newGlobSourceOpts(glob func(st llb.State, pattern string) ([]string, error)) SourceOpts {
+	return SourceOpts{
+		GetContext: func(name string, opts ...llb.LocalOption) (*llb.State, error) {
+			st := llb.Local(name, opts...)
+			return &st, nil
+		},
+		Glob: glob,
+	}
+}

source.go

@@ -155,6 +155,7 @@ type SourceOpts struct {
 	TargetPlatform   *ocispecs.Platform
 	GitCredHelperOpt func() (llb.RunOption, error)
 	SourceFilter     func() (SourceFilterConfig, error)
+	Glob             func(llb.State, string) ([]string, error)
 	// ExtraEnvs contains environment variables that source generators may use
 	// while preparing generated dependency sources.
 	ExtraEnvs map[string]string