diff --git a/frontend/debug/handle_gomod.go b/frontend/debug/handle_gomod.go index ffe5cb41a..5e6e05e9e 100644 --- a/frontend/debug/handle_gomod.go +++ b/frontend/debug/handle_gomod.go @@ -39,6 +39,10 @@ func Gomods(ctx context.Context, client gwclient.Client) (*gwclient.Result, erro } worker = worker.With(addedHosts(client)) + if err := spec.Preprocess(sOpt, worker, dalec.Platform(platform), pg); err != nil { + return nil, nil, err + } + st := spec.GomodDeps(sOpt, worker, dalec.Platform(platform), pg) def, err := st.Marshal(ctx) diff --git a/frontend/gateway.go b/frontend/gateway.go index d387523ce..f2050f427 100644 --- a/frontend/gateway.go +++ b/frontend/gateway.go @@ -3,6 +3,7 @@ package frontend import ( "context" "fmt" + "io/fs" "sync" "sync/atomic" @@ -14,7 +15,9 @@ import ( "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "github.com/project-dalec/dalec" + "github.com/project-dalec/dalec/frontend/pkg/bkfs" ) const ( @@ -128,6 +131,14 @@ func SourceOptFromUIClient(ctx context.Context, c gwclient.Client, dc *dockerui. return st, err }, GitCredHelperOpt: withCredHelper(c), + Glob: func(st llb.State, pattern string) ([]string, error) { + fsys, err := bkfs.FromState(ctx, &st, c) + if err != nil { + return nil, err + } + + return fs.Glob(fsys, pattern) + }, } sOpt.SourceFilter = sync.OnceValues(func() (dalec.SourceFilterConfig, error) { diff --git a/preprocess.go b/preprocess.go index a2bb8b22e..f5eeb6c31 100644 --- a/preprocess.go +++ b/preprocess.go @@ -4,6 +4,7 @@ import ( _ "embed" "fmt" "path/filepath" + "slices" "strings" "golang.org/x/mod/module" @@ -31,6 +32,10 @@ const ( // Preprocessing generates LLB states for patches and registers them as context sources // that can be retrieved later when sources are fetched. func (s *Spec) Preprocess(sOpt SourceOpts, worker llb.State, opts ...llb.ConstraintsOpt) error { + if err := s.preprocessGomodPaths(sOpt, worker, opts...); err != nil { + return errors.Wrap(err, "failed to preprocess gomod paths") + } + if err := s.preprocessGomodEdits(sOpt, worker, opts...); err != nil { return errors.Wrap(err, "failed to preprocess gomod edits") } @@ -38,6 +43,56 @@ func (s *Spec) Preprocess(sOpt SourceOpts, worker llb.State, opts ...llb.Constra return nil } +// preprocessGomodPaths expands glob patterns in GeneratorGomod.Paths against +// the source content. +func (s *Spec) preprocessGomodPaths(sOpt SourceOpts, worker llb.State, opts ...llb.ConstraintsOpt) error { + gomodSources := s.gomodSources() + if len(gomodSources) == 0 { + return nil + } + + // Get sources with base patches applied + patchedSources := s.getPatchedSources(sOpt, worker, func(name string) bool { + _, ok := gomodSources[name] + return ok + }, opts...) + + // Expand paths for each source with gomod generators + for sourceName, src := range gomodSources { + patchedState, ok := patchedSources[sourceName] + if !ok { + continue + } + + for _, gen := range src.Generate { + if gen == nil || gen.Gomod == nil || gen.Gomod.Paths == nil { + continue + } + + basePath := filepath.Join(sourceName, gen.Subpath) + oldPaths := gen.Gomod.Paths + + newPaths := make([]string, 0, len(oldPaths)) + for _, pattern := range oldPaths { + matches, err := sOpt.Glob(patchedState, filepath.Join(basePath, pattern)) + if err != nil { + return err + } + + for _, match := range matches { + newPath := "." + strings.TrimPrefix(match, basePath) + newPaths = append(newPaths, newPath) + } + } + slices.Sort(newPaths) + + gen.Gomod.Paths = newPaths + } + } + + return nil +} + // preprocessGomodEdits generates patch LLB states for all gomod replace directives // and registers them as context sources that can be retrieved later. func (s *Spec) preprocessGomodEdits(sOpt SourceOpts, worker llb.State, opts ...llb.ConstraintsOpt) error { diff --git a/preprocess_test.go b/preprocess_test.go new file mode 100644 index 000000000..bbf799588 --- /dev/null +++ b/preprocess_test.go @@ -0,0 +1,204 @@ +package dalec + +import ( + "errors" + "testing" + + "github.com/moby/buildkit/client/llb" + "gotest.tools/v3/assert" +) + +func TestPreprocessGomodPaths(t *testing.T) { + t.Parallel() + + t.Run("bare wildcard globs from the source root", func(t *testing.T) { + t.Parallel() + + spec := newSpecWithGomod(&GeneratorGomod{Paths: []string{"*"}}) + + var gotPattern string + sOpt := newGlobSourceOpts(func(st llb.State, pattern string) ([]string, error) { + gotPattern = pattern + return []string{"foo/module1", "foo/module2"}, nil + }) + + err := spec.preprocessGomodPaths(sOpt, llb.Scratch()) + assert.NilError(t, err) + + gen := spec.Sources["foo"].Generate[0].Gomod + assert.DeepEqual(t, gen.Paths, []string{"./module1", "./module2"}) + assert.Equal(t, gotPattern, "foo/*") + }) + + t.Run("every entry is globbed, including literal paths", func(t *testing.T) { + t.Parallel() + + spec := newSpecWithGomod(&GeneratorGomod{Paths: []string{".", "plugins/*"}}) + + var gotPatterns []string + sOpt := newGlobSourceOpts(func(st llb.State, pattern string) ([]string, error) { + gotPatterns = append(gotPatterns, pattern) + switch pattern { + case "foo": + return []string{"foo"}, nil + case "foo/plugins/*": + return []string{"foo/plugins/v1"}, nil + default: + t.Fatalf("unexpected pattern %q", pattern) + return nil, nil + } + }) + + err := spec.preprocessGomodPaths(sOpt, llb.Scratch()) + assert.NilError(t, err) + + gen := spec.Sources["foo"].Generate[0].Gomod + assert.DeepEqual(t, gen.Paths, []string{".", "./plugins/v1"}) + assert.DeepEqual(t, gotPatterns, []string{"foo", "foo/plugins/*"}) + }) + + t.Run("results are fully sorted regardless of input order", func(t *testing.T) { + t.Parallel() + + spec := newSpecWithGomod(&GeneratorGomod{Paths: []string{"moduleB/*", "moduleA/*"}}) + + sOpt := newGlobSourceOpts(func(st llb.State, pattern string) ([]string, error) { + switch pattern { + case "foo/moduleA/*": + return []string{"foo/moduleA/v2", "foo/moduleA/v1"}, nil + case "foo/moduleB/*": + return []string{"foo/moduleB/v1"}, nil + default: + t.Fatalf("unexpected pattern %q", pattern) + return nil, nil + } + }) + + err := spec.preprocessGomodPaths(sOpt, llb.Scratch()) + assert.NilError(t, err) + + gen := spec.Sources["foo"].Generate[0].Gomod + assert.DeepEqual(t, gen.Paths, []string{"./moduleA/v1", "./moduleA/v2", "./moduleB/v1"}) + }) + + t.Run("combines with the generator's Subpath", func(t *testing.T) { + t.Parallel() + + spec := newSpecWithGomod(&GeneratorGomod{Paths: []string{"plugins/*"}}) + spec.Sources["foo"].Generate[0].Subpath = "some/nested/dir" + + var gotPattern string + sOpt := newGlobSourceOpts(func(st llb.State, pattern string) ([]string, error) { + gotPattern = pattern + return []string{"foo/some/nested/dir/plugins/v1"}, nil + }) + + err := spec.preprocessGomodPaths(sOpt, llb.Scratch()) + assert.NilError(t, err) + + gen := spec.Sources["foo"].Generate[0].Gomod + assert.DeepEqual(t, gen.Paths, []string{"./plugins/v1"}) + assert.Equal(t, gotPattern, "foo/some/nested/dir/plugins/*") + }) + + t.Run("does not touch or glob sources with no Paths", func(t *testing.T) { + t.Parallel() + + spec := newSpecWithGomod(&GeneratorGomod{}) + + sOpt := newGlobSourceOpts(func(st llb.State, pattern string) ([]string, error) { + t.Fatal("FSGlob should not be called when Paths is empty") + return nil, nil + }) + + err := spec.preprocessGomodPaths(sOpt, llb.Scratch()) + assert.NilError(t, err) + assert.Assert(t, spec.Sources["foo"].Generate[0].Gomod.Paths == nil) + }) + + t.Run("is a no-op for specs without any gomod generator", func(t *testing.T) { + t.Parallel() + + spec := &Spec{Sources: map[string]Source{"foo": {Git: &SourceGit{URL: "https://localhost/test.git", Commit: "deadbeef"}}}} + + err := spec.preprocessGomodPaths(SourceOpts{}, llb.Scratch()) + assert.NilError(t, err) + }) + + t.Run("a pattern matching nothing is silently dropped", func(t *testing.T) { + t.Parallel() + + spec := newSpecWithGomod(&GeneratorGomod{Paths: []string{"module1/*", "typo/*"}}) + + sOpt := newGlobSourceOpts(func(st llb.State, pattern string) ([]string, error) { + if pattern == "foo/module1/*" { + return []string{"foo/module1/v1"}, nil + } + return nil, nil + }) + + err := spec.preprocessGomodPaths(sOpt, llb.Scratch()) + assert.NilError(t, err) + + gen := spec.Sources["foo"].Generate[0].Gomod + assert.DeepEqual(t, gen.Paths, []string{"./module1/v1"}) + }) + + t.Run("Paths ends up a non-nil empty slice if every pattern matches nothing", func(t *testing.T) { + t.Parallel() + + spec := newSpecWithGomod(&GeneratorGomod{Paths: []string{"typo/*"}}) + + sOpt := newGlobSourceOpts(func(st llb.State, pattern string) ([]string, error) { + return nil, nil + }) + + err := spec.preprocessGomodPaths(sOpt, llb.Scratch()) + assert.NilError(t, err) + + gen := spec.Sources["foo"].Generate[0].Gomod + assert.Assert(t, gen.Paths != nil) + assert.DeepEqual(t, gen.Paths, []string{}) + }) + + t.Run("propagates glob errors", func(t *testing.T) { + t.Parallel() + + spec := newSpecWithGomod(&GeneratorGomod{Paths: []string{"*"}}) + + sOpt := newGlobSourceOpts(func(st llb.State, pattern string) ([]string, error) { + return nil, errors.New("boom") + }) + + err := spec.preprocessGomodPaths(sOpt, llb.Scratch()) + assert.ErrorContains(t, err, "boom") + }) +} + +func newSpecWithGomod(gen *GeneratorGomod) *Spec { + return &Spec{ + Sources: map[string]Source{ + "foo": { + Git: &SourceGit{ + URL: "https://localhost/bar.git", + Commit: "deadbeef", + }, + Generate: []*SourceGenerator{ + { + Gomod: gen, + }, + }, + }, + }, + } +} + +func newGlobSourceOpts(glob func(st llb.State, pattern string) ([]string, error)) SourceOpts { + return SourceOpts{ + GetContext: func(name string, opts ...llb.LocalOption) (*llb.State, error) { + st := llb.Local(name, opts...) + return &st, nil + }, + Glob: glob, + } +} diff --git a/source.go b/source.go index 8220ded44..43a8b2e47 100644 --- a/source.go +++ b/source.go @@ -155,6 +155,7 @@ type SourceOpts struct { TargetPlatform *ocispecs.Platform GitCredHelperOpt func() (llb.RunOption, error) SourceFilter func() (SourceFilterConfig, error) + Glob func(llb.State, string) ([]string, error) // ExtraEnvs contains environment variables that source generators may use // while preparing generated dependency sources. ExtraEnvs map[string]string diff --git a/test/source_test.go b/test/source_test.go index 607bdc3de..c73c40670 100644 --- a/test/source_test.go +++ b/test/source_test.go @@ -644,6 +644,34 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ` +const thirdGomodFixtureMain = `package main + +import ( + "fmt" + + "golang.org/x/sync/syncmap" +) + +func main() { + var m syncmap.Map + m.Clear() + + fmt.Println("Hello, playground") +} +` + +const thirdGomodFixtureMod = `module example.com/m/v2 + +go 1.25.0 + +require golang.org/x/sync v0.22.0 +` + +const thirdGomodFixtureSum = ` +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +` + const npmPackageJson = ` { "name": "npm-test", @@ -892,6 +920,94 @@ index ea874f5..ba38f84 100644 }) }) + t.Run("multi-module with path expansion", func(t *testing.T) { + t.Parallel() + /* + dir/ + go.mod + go.sum + main.go + module1/ + go.mod + go.sum + main.go + module2/ + go.mod + go.sum + main.go + */ + + pg := dalec.ProgressGroup("test-multi-module-gomod-expansion") + + contextSt := llb.Scratch(). + File(llb.Mkdir("/dir", 0644), pg). + File(llb.Mkfile("/dir/go.mod", 0644, []byte(gomodFixtureMod)), pg). + File(llb.Mkfile("/dir/go.sum", 0644, []byte(gomodFixtureSum)), pg). + File(llb.Mkfile("/dir/main.go", 0644, []byte(gomodFixtureMain)), pg). + File(llb.Mkdir("/dir/module1", 0644), pg). + File(llb.Mkfile("/dir/module1/go.mod", 0644, []byte(alternativeGomodFixtureMod)), pg). + File(llb.Mkfile("/dir/module1/go.sum", 0644, []byte(alternativeGomodFixtureSum)), pg). + File(llb.Mkfile("/dir/module1/main.go", 0644, []byte(alternativeGomodFixtureMain)), pg). + File(llb.Mkdir("/dir/module2", 0644), pg). + File(llb.Mkfile("/dir/module2/go.mod", 0644, []byte(thirdGomodFixtureMod)), pg). + File(llb.Mkfile("/dir/module2/go.sum", 0644, []byte(thirdGomodFixtureSum)), pg). + File(llb.Mkfile("/dir/module2/main.go", 0644, []byte(thirdGomodFixtureMain)), pg) + + const contextName = "multi-module-expansion" + spec := &dalec.Spec{ + Name: "test-dalec-context-source", + Sources: map[string]dalec.Source{ + "src": { + Context: &dalec.SourceContext{Name: contextName}, + Generate: []*dalec.SourceGenerator{ + { + Gomod: &dalec.GeneratorGomod{ + Paths: []string{ + "dir/module?", // expands to both dir/module1 and dir/module2. + }, + }, + }, + }, + }, + }, + Dependencies: &dalec.PackageDependencies{ + Build: map[string]dalec.PackageConstraints{ + "golang": { + Version: []string{}, + }, + }, + }, + } + + runTest(t, func(ctx context.Context, gwc gwclient.Client) { + req := newSolveRequest(withSpec(ctx, t, spec), withBuildContext(ctx, t, contextName, contextSt), withBuildTarget("debug/gomods")) + res := solveT(ctx, t, gwc, req) + ref, err := res.SingleRef() + if err != nil { + t.Fatal(err) + } + + for _, dep := range []string{"github.com/cpuguy83/tar2go@v0.3.1"} { + if _, err := ref.StatFile(ctx, gwclient.StatRequest{Path: dep}); err == nil { + t.Fatalf("expected %q not to be fetched", dep) + } + } + + for _, dep := range []string{"github.com/stretchr/testify@v1.7.0", "golang.org/x/sync@v0.22.0"} { + stat, err := ref.StatFile(ctx, gwclient.StatRequest{ + Path: dep, + }) + if err != nil { + t.Fatal(err) + } + + if !fs.FileMode(stat.Mode).IsDir() { + t.Fatal("expected directory") + } + } + }) + }) + t.Run("with replace directive", func(t *testing.T) { t.Parallel() testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) { diff --git a/website/content/sources.md b/website/content/sources.md index 729acaf5e..98111cee2 100644 --- a/website/content/sources.md +++ b/website/content/sources.md @@ -456,6 +456,23 @@ sources: - module2 ``` +Instead of maintaining `paths` by hand, an entry can be a glob pattern, matched +against the source content using [`io/fs.Glob`](https://pkg.go.dev/io/fs#Glob) +semantics (e.g. `*` matches every entry in the directory). Each match replaces +the pattern in place, so the spec keeps working as modules are added, +renamed, or removed. + +```yaml +sources: + src: + path: ./ + context: {} + generate: + - gomod: + paths: + - module? +``` + The `gomod` generator supports private go modules. The following example illustrates this: ```yaml