diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a52e9865c..2a039333f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,8 @@ on: - '.github/workflows/worker-images.yml' - '.github/workflows/dependabot.yml' - '.github/workflows/release.yml' + - '.github/workflows/deploy-docs.yml' + - '.github/workflows/codeql.yml' push: branches: @@ -287,9 +289,7 @@ jobs: docker buildx bake test - name: Build go-md2man example in docs run: | - version=$(cat docs/examples/go-md2man.yml | yq .version) - docker build -t go-md2man:$version -f docs/examples/go-md2man.yml --target=mariner2/rpm --output=_output . - docker build -t go-md2man:$version -f docs/examples/go-md2man.yml --target=mariner2 . + docker buildx bake examples - name: dump logs if: failure() id: dump-logs diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 44f01b514..80e377e48 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -13,12 +13,12 @@ name: "CodeQL" on: push: - branches: ["main"] + branches: ["main", "release/**"] paths-ignore: - '.github/copilot-instructions.md' pull_request: # The branches below must be a subset of the branches above - branches: ["main"] + branches: ["main", "release/**"] paths-ignore: - '.github/copilot-instructions.md' schedule: diff --git a/frontend/gateway.go b/frontend/gateway.go index 137a0b04c..6ce4f3a90 100644 --- a/frontend/gateway.go +++ b/frontend/gateway.go @@ -103,7 +103,7 @@ func GetBuildArg(client gwclient.Client, k string) (string, bool) { } func SourceOptFromUIClient(ctx context.Context, c gwclient.Client, dc *dockerui.Client, platform *ocispecs.Platform) dalec.SourceOpts { - return dalec.SourceOpts{ + sOpt := dalec.SourceOpts{ TargetPlatform: platform, Resolver: c, Forward: ForwarderFromClient(ctx, c), @@ -129,6 +129,12 @@ func SourceOptFromUIClient(ctx context.Context, c gwclient.Client, dc *dockerui. }, GitCredHelperOpt: withCredHelper(c), } + + sOpt.SourceFilter = sync.OnceValues(func() (dalec.SourceFilterConfig, error) { + return loadSourceFilterConfig(ctx, c, sOpt.GetContext) + }) + + return sOpt } func SourceOptFromClient(ctx context.Context, c gwclient.Client, platform *ocispecs.Platform) (dalec.SourceOpts, error) { diff --git a/frontend/request.go b/frontend/request.go index 5a6fb4d0c..ef862c247 100644 --- a/frontend/request.go +++ b/frontend/request.go @@ -143,10 +143,43 @@ func marshalDockerfile(ctx context.Context, dt []byte, opts ...llb.ConstraintsOp } func getSigningConfigFromContext(ctx context.Context, client gwclient.Client, cfgPath string, configCtxName string, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (*dalec.PackageSigner, error) { + dt, err := readConfigFromContext(ctx, client, cfgPath, configCtxName, sOpt, opts...) + if err != nil { + return nil, err + } + + var pc dalec.PackageConfig + if err := yaml.Unmarshal(dt, &pc); err != nil { + return nil, err + } + + return pc.Signer, nil +} + +func getSourceFilterConfigFromContext(ctx context.Context, client gwclient.Client, cfgPath string, configCtxName string, getContext func(string, ...llb.LocalOption) (*llb.State, error), opts ...llb.ConstraintsOpt) (dalec.SourceFilterConfig, error) { + dt, err := readConfigFromContext(ctx, client, cfgPath, configCtxName, dalec.SourceOpts{ + GetContext: getContext, + }, opts...) + if err != nil { + return dalec.SourceFilterConfig{}, err + } + + return decodeSourceFilterConfig(ctx, dt) +} + +func decodeSourceFilterConfig(ctx context.Context, dt []byte) (dalec.SourceFilterConfig, error) { + var cfg dalec.SourceFilterConfig + if err := yaml.UnmarshalContext(ctx, dt, &cfg, yaml.Strict()); err != nil { + return dalec.SourceFilterConfig{}, err + } + return cfg, nil +} + +func readConfigFromContext(ctx context.Context, client gwclient.Client, cfgPath string, configCtxName string, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]byte, error) { src := dalec.Source{Path: cfgPath, Context: &dalec.SourceContext{Name: configCtxName}} - signConfigState := src.ToState("", sOpt, opts...) + configState := src.ToState("", dalec.SourceOpts{GetContext: sOpt.GetContext}, opts...) - scDef, err := signConfigState.Marshal(ctx) + scDef, err := configState.Marshal(ctx) if err != nil { return nil, err } @@ -163,19 +196,9 @@ func getSigningConfigFromContext(ctx context.Context, client gwclient.Client, cf return nil, err } - dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{ + return ref.ReadFile(ctx, gwclient.ReadRequest{ Filename: cfgPath, }) - if err != nil { - return nil, err - } - - var pc dalec.PackageConfig - if err := yaml.Unmarshal(dt, &pc); err != nil { - return nil, err - } - - return pc.Signer, nil } func MaybeSign(ctx context.Context, client gwclient.Client, st llb.State, spec *dalec.Spec, targetKey string, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, error) { @@ -243,6 +266,27 @@ func getSignConfigCtxName(client gwclient.Client) string { return client.BuildOpts().Opts["build-arg:"+buildArgDalecSigningConfigContextName] } +func getSourceFilterConfigPath(client gwclient.Client) string { + return client.BuildOpts().Opts["build-arg:"+dalec.BuildArgDalecSourceFilterConfigPath] +} + +func getSourceFilterContextNameWithDefault(client gwclient.Client) string { + configCtxName := dalec.DefaultSourceOptionsContextName + if cn := client.BuildOpts().Opts["build-arg:"+dalec.BuildArgDalecSourceFilterContextName]; cn != "" { + configCtxName = cn + } + return configCtxName +} + +func loadSourceFilterConfig(ctx context.Context, client gwclient.Client, getContext func(string, ...llb.LocalOption) (*llb.State, error)) (dalec.SourceFilterConfig, error) { + cfgPath := getSourceFilterConfigPath(client) + if cfgPath == "" { + return dalec.SourceFilterConfig{}, nil + } + + return getSourceFilterConfigFromContext(ctx, client, cfgPath, getSourceFilterContextNameWithDefault(client), getContext) +} + func forwardToSigner(ctx context.Context, client gwclient.Client, cfg *dalec.PackageSigner, s llb.State, opts ...llb.ConstraintsOpt) (llb.State, error) { const ( // See https://github.com/moby/buildkit/blob/d8d946b85c52095d34a52ce210960832f4e06775/frontend/dockerui/context.go#L29 diff --git a/generator_cargohome.go b/generator_cargohome.go index 12392a3dc..5ec26c191 100644 --- a/generator_cargohome.go +++ b/generator_cargohome.go @@ -101,5 +101,6 @@ func (s *Spec) CargohomeDeps(sOpt SourceOpts, worker llb.State, opts ...llb.Cons }) } + deps = deps.With(sourceFilter(sOpt, opts...)) return &deps, nil } diff --git a/generator_gomod.go b/generator_gomod.go index 31047ded0..907abbdf3 100644 --- a/generator_gomod.go +++ b/generator_gomod.go @@ -242,5 +242,6 @@ func (s *Spec) GomodDeps(sOpt SourceOpts, worker llb.State, opts ...llb.Constrai }) } + deps = deps.With(sourceFilter(sOpt, opts...)) return &deps, nil } diff --git a/generator_nodemodules.go b/generator_nodemodules.go index 168004962..86145ae5c 100644 --- a/generator_nodemodules.go +++ b/generator_nodemodules.go @@ -104,7 +104,7 @@ func (s *Spec) NodeModDeps(sOpt SourceOpts, worker llb.State, opts ...llb.Constr } merged = merged.With(withNodeMod(gen, worker, key, opts...)) } - result[key] = merged + result[key] = merged.With(sourceFilterAtPath(sOpt, key, opts...)) } return result, nil } diff --git a/generator_pip.go b/generator_pip.go index 42e72c48b..462cdf9ff 100644 --- a/generator_pip.go +++ b/generator_pip.go @@ -132,6 +132,7 @@ func (s *Spec) PipDeps(sOpt SourceOpts, worker llb.State, opts ...llb.Constraint } // Merge all cache states into a single state - merged := MergeAtPath(llb.Scratch(), cacheStates, "/") + merged := MergeAtPath(llb.Scratch(), cacheStates, "/", opts...) + merged = merged.With(sourceFilter(sOpt, opts...)) return &merged, nil } diff --git a/load.go b/load.go index 4f89c71ce..3a368b508 100644 --- a/load.go +++ b/load.go @@ -37,6 +37,10 @@ func knownArg(key string) bool { return true case "DALEC_SKIP_TESTS": return true + case "DALEC_SOURCE_FILTER_CONFIG_PATH": + return true + case "DALEC_SOURCE_FILTER_CONFIG_CONTEXT_NAME": + return true case KeyDalecTarget: return true } diff --git a/packaging/linux/rpm/buildroot.go b/packaging/linux/rpm/buildroot.go index 0533f5afc..71620332d 100644 --- a/packaging/linux/rpm/buildroot.go +++ b/packaging/linux/rpm/buildroot.go @@ -11,6 +11,10 @@ import ( ) func ToSpecLLB(spec *dalec.Spec, in llb.State, targetKey, dir string, opts ...llb.ConstraintsOpt) (llb.State, error) { + return RPMSpecWithSourceFilter(spec, in, targetKey, dir, dalec.SourceFilterConfig{}, opts...) +} + +func RPMSpecWithSourceFilter(spec *dalec.Spec, in llb.State, targetKey, dir string, filter dalec.SourceFilterConfig, opts ...llb.ConstraintsOpt) (llb.State, error) { if err := ValidateSpec(spec); err != nil { return llb.Scratch(), fmt.Errorf("invalid spec: %w", err) } @@ -20,7 +24,7 @@ func ToSpecLLB(spec *dalec.Spec, in llb.State, targetKey, dir string, opts ...ll buf.WriteString("# Automatically generated by " + info.Main.Path + "\n") buf.WriteString("\n") - if err := WriteSpec(spec, targetKey, buf); err != nil { + if err := WriteSpecWithSourceFilter(spec, targetKey, filter, buf); err != nil { return llb.Scratch(), err } @@ -39,11 +43,14 @@ func SpecToBuildrootLLB(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpt return llb.Scratch(), fmt.Errorf("invalid spec: %w", err) } opts = append(opts, dalec.ProgressGroup("Create RPM buildroot")) - sources, err := ToSourcesLLB(worker, spec, sOpt, opts...) if err != nil { - return llb.Scratch(), err + return llb.Scratch(), fmt.Errorf("error generating sources: %w", err) } - return ToSpecLLB(spec, dalec.MergeAtPath(llb.Scratch(), sources, "SOURCES"), targetKey, "", opts...) + filter, err := sOpt.GetSourceFilter() + if err != nil { + return llb.Scratch(), err + } + return RPMSpecWithSourceFilter(spec, dalec.MergeAtPath(llb.Scratch(), sources, "SOURCES", opts...), targetKey, "", filter, opts...) } diff --git a/packaging/linux/rpm/template.go b/packaging/linux/rpm/template.go index d4fe50381..a6493d04c 100644 --- a/packaging/linux/rpm/template.go +++ b/packaging/linux/rpm/template.go @@ -67,7 +67,8 @@ var tmplFuncs = map[string]any{ type specWrapper struct { *dalec.Spec - Target string + Target string + SourceFilter dalec.SourceFilterConfig } func (w *specWrapper) Changelog() (fmt.Stringer, error) { @@ -313,21 +314,46 @@ func (w *specWrapper) Sources() (fmt.Stringer, error) { if scanner.Err() != nil { return nil, scanner.Err() } + if !w.SourceFilter.IsEmpty() && isDir { + if err := docSourceFilter(b, "Exclusions", w.SourceFilter.GlobalExcludes); err != nil { + return nil, err + } + } fmt.Fprintf(b, "Source%d: %s\n", idx, ref) } sourceIdx := len(keys) if w.Spec.HasGomods() { + if !w.SourceFilter.IsEmpty() { + if err := docSourceFilter(b, "Exclusions", w.SourceFilter.GlobalExcludes); err != nil { + return nil, err + } + } fmt.Fprintf(b, "Source%d: %s.tar.gz\n", sourceIdx, gomodsName) sourceIdx += 1 } if w.Spec.HasCargohomes() { + if !w.SourceFilter.IsEmpty() { + if err := docSourceFilter(b, "Exclusions", w.SourceFilter.GlobalExcludes); err != nil { + return nil, err + } + } fmt.Fprintf(b, "Source%d: %s.tar.gz\n", sourceIdx, cargohomeName) sourceIdx += 1 } + if w.Spec.HasPips() { + if !w.SourceFilter.IsEmpty() { + if err := docSourceFilter(b, "Exclusions", w.SourceFilter.GlobalExcludes); err != nil { + return nil, err + } + } + fmt.Fprintf(b, "Source%d: %s.tar.gz\n", sourceIdx, pipDepsName) + sourceIdx += 1 + } + if len(w.Spec.Build.Steps) > 0 { fmt.Fprintf(b, "Source%d: %s\n", sourceIdx, buildScriptName) } @@ -338,6 +364,24 @@ func (w *specWrapper) Sources() (fmt.Stringer, error) { return b, nil } +func docSourceFilter(w io.Writer, name string, values []string) error { + if _, err := fmt.Fprintf(w, "# %s:\n", name); err != nil { + return err + } + for _, value := range values { + scanner := bufio.NewScanner(strings.NewReader(value)) + for scanner.Scan() { + if _, err := fmt.Fprintf(w, "# \t%s\n", scanner.Text()); err != nil { + return err + } + } + if err := scanner.Err(); err != nil { + return err + } + } + return nil +} + func (w *specWrapper) Release() string { if w.Spec.Revision == "" { return "1" @@ -911,7 +955,11 @@ func (w *specWrapper) DisableAutoReq() string { // WriteSpec generates an rpm spec from the provided [dalec.Spec] and distro target and writes it to the passed in writer func WriteSpec(spec *dalec.Spec, target string, w io.Writer) error { - s := &specWrapper{spec, target} + return WriteSpecWithSourceFilter(spec, target, dalec.SourceFilterConfig{}, w) +} + +func WriteSpecWithSourceFilter(spec *dalec.Spec, target string, filter dalec.SourceFilterConfig, w io.Writer) error { + s := &specWrapper{Spec: spec, Target: target, SourceFilter: filter} err := specTmpl.Execute(w, s) if err != nil { diff --git a/packaging/linux/rpm/template_test.go b/packaging/linux/rpm/template_test.go index 54b6557e3..226d4adbe 100644 --- a/packaging/linux/rpm/template_test.go +++ b/packaging/linux/rpm/template_test.go @@ -161,8 +161,16 @@ func TestTemplateSources(t *testing.T) { t.Fatalf("unexpected error: %v", err) } s2 := out2.String() - if s2 != s { - t.Fatalf("expected no additional sources for pip, got: %q", s2) + // trim last newline from the first output since that has shifted + s3 := s[:len(s)-1] + if !strings.HasPrefix(s2, s3) { + t.Fatalf("expected output to start with %q, got %q", s, out2.String()) + } + + s2 = strings.TrimPrefix(out2.String(), s3) + expected := "Source1: " + pipDepsName + ".tar.gz\n\n" + if s2 != expected { + t.Fatalf("unexpected sources: expected %q, got: %q", expected, s2) } }) @@ -253,11 +261,10 @@ func TestTemplateSources(t *testing.T) { s = s[len(expected):] } - // Now we should have entries for gomods and cargohome. + // Now we should have entries for gomods, cargohome, and pip deps. // Note there are 2 gomod sources but they should be combined into one entry. - // Pip no longer creates a separate cache source. - expected := "Source7: " + gomodsName + ".tar.gz\nSource8: " + cargohomeName + ".tar.gz\n\n" + expected := "Source7: " + gomodsName + ".tar.gz\nSource8: " + cargohomeName + ".tar.gz\nSource9: " + pipDepsName + ".tar.gz\n\n" if s != expected { t.Fatalf("generators: unexpected sources: expected %q, got: %q", expected, s) } @@ -266,6 +273,49 @@ func TestTemplateSources(t *testing.T) { t.Fatalf("unexpected trailing sources: %q", s) } }) + + t.Run("source filter docs", func(t *testing.T) { + w := &specWrapper{ + Spec: &dalec.Spec{ + Sources: map[string]dalec.Source{ + "src1": { + Includes: []string{"cmd/**"}, + Excludes: []string{"testdata/**"}, + Inline: &dalec.SourceInline{ + Dir: &dalec.SourceInlineDir{}, + }, + }, + }, + }, + SourceFilter: dalec.SourceFilterConfig{GlobalExcludes: []string{"vendor/**"}}, + } + + out, err := w.Sources() + assert.NilError(t, err) + s := out.String() + assert.Check(t, strings.Contains(s, "# \tIncludes:\n# \t\t cmd/**\n")) + assert.Check(t, strings.Contains(s, "# \tExcludes:\n# \t\t testdata/**\n")) + assert.Check(t, strings.Contains(s, "# Exclusions:\n# \tvendor/**\n")) + }) + + t.Run("source filter docs multiline exclusion", func(t *testing.T) { + w := &specWrapper{ + Spec: &dalec.Spec{ + Sources: map[string]dalec.Source{ + "src1": { + Inline: &dalec.SourceInline{Dir: &dalec.SourceInlineDir{}}, + }, + }, + }, + SourceFilter: dalec.SourceFilterConfig{GlobalExcludes: []string{"vendor/**\nmalicious: value"}}, + } + + out, err := w.Sources() + assert.NilError(t, err) + s := out.String() + assert.Check(t, strings.Contains(s, "# Exclusions:\n# \tvendor/**\n# \tmalicious: value\n")) + assert.Check(t, !strings.Contains(s, "\nmalicious: value\n")) + }) } func TestTemplate_Artifacts(t *testing.T) { diff --git a/source.go b/source.go index 3a3fe35bb..84fe7027c 100644 --- a/source.go +++ b/source.go @@ -9,6 +9,7 @@ import ( "io" "os" "path/filepath" + "slices" "strings" "github.com/moby/buildkit/client/llb" @@ -95,6 +96,7 @@ type SourceOpts struct { GetContext func(string, ...llb.LocalOption) (*llb.State, error) TargetPlatform *ocispecs.Platform GitCredHelperOpt func() (llb.RunOption, error) + SourceFilter func() (SourceFilterConfig, error) } var errInvalidMountConfig = errors.New("invalid mount config") @@ -140,10 +142,22 @@ func (s Source) Doc(name string) io.Reader { if s.Path != "" { fmt.Fprintln(buf, " Extracted path:", s.Path) } + writeSourceDocList(buf, "Includes", s.Includes) + writeSourceDocList(buf, "Excludes", s.Excludes) return buf } -func patchSource(worker, sourceState llb.State, sourceToState map[string]llb.State, patchNames []PatchSpec, subPath string, opts ...llb.ConstraintsOpt) llb.State { +func writeSourceDocList(w io.Writer, name string, values []string) { + if len(values) == 0 { + return + } + printDocLn(w, "\t"+name+":") + for _, value := range values { + printDocLn(w, "\t\t", value) + } +} + +func patchSource(worker, sourceState llb.State, sourceToState map[string]llb.State, patchNames []PatchSpec, opts ...llb.ConstraintsOpt) llb.State { for _, p := range patchNames { patchState := sourceToState[p.Source] // on each iteration, mount source state to /src to run `patch`, and @@ -153,7 +167,7 @@ func patchSource(worker, sourceState llb.State, sourceToState map[string]llb.Sta sourceState = worker.Run( llb.AddMount("/patch", patchState, llb.Readonly, llb.SourcePath(patchPath)), - llb.Dir(filepath.Join("src", subPath)), + llb.Dir("src"), ShArgs(fmt.Sprintf("patch -p%d < /patch", *p.Strip)), WithConstraints(opts...), ).AddMount("/src", sourceState) @@ -180,7 +194,8 @@ func PatchSources(worker llb.State, spec *Spec, sourceToState map[string]llb.Sta continue } pg := llb.ProgressGroup(pgID, "Patch spec source: "+sourceName+" ", false) - states[sourceName] = patchSource(worker, sourceState, states, patches, sourceName, pg, withConstraints(opts)) + opts = append(opts, pg) + states[sourceName] = patchSource(worker, sourceState, states, patches, withConstraints(opts)) } return states @@ -376,7 +391,10 @@ func (s *Source) validate() error { if !invalid { // Only validate the source if it is a valid source variant so as to avoid panics. - if err := s.toInterface().validate(s.fetchOptions(SourceOpts{})); err != nil { + fo, err := s.fetchOptions(SourceOpts{}) + if err != nil { + errs = append(errs, err) + } else if err := s.toInterface().validate(fo); err != nil { errs = append(errs, err) } } @@ -409,17 +427,31 @@ type fetchOptions struct { SourceOpt SourceOpts } -func (s *Source) fetchOptions(sOpt SourceOpts) fetchOptions { +func (s *Source) fetchOptions(sOpt SourceOpts) (fetchOptions, error) { + excludes := s.Excludes + if s.IsDir() { + globalExcludes, err := sOpt.sourceFilterExcludes() + if err != nil { + return fetchOptions{}, err + } + if len(globalExcludes) > 0 { + excludes = append(slices.Clone(excludes), globalExcludes...) + } + } + return fetchOptions{ Includes: s.Includes, - Excludes: s.Excludes, + Excludes: excludes, Path: s.Path, SourceOpt: sOpt, - } + }, nil } func (s *Source) ToState(name string, sOpt SourceOpts, opts ...llb.ConstraintsOpt) llb.State { - fo := s.fetchOptions(sOpt) + fo, err := s.fetchOptions(sOpt) + if err != nil { + return ErrorState(llb.Scratch(), err) + } fo.Constraints = opts fo.Rename = name st := s.toInterface().toState(fo) @@ -427,7 +459,10 @@ func (s *Source) ToState(name string, sOpt SourceOpts, opts ...llb.ConstraintsOp } func (s *Source) ToMount(sOpt SourceOpts, constraints ...llb.ConstraintsOpt) (llb.State, []llb.MountOption) { - fo := s.fetchOptions(sOpt) + fo, err := s.fetchOptions(sOpt) + if err != nil { + return ErrorState(llb.Scratch(), err), nil + } fo.Constraints = append(fo.Constraints, constraints...) st, mountOpts := s.toInterface().toMount(fo) diff --git a/source_filter.go b/source_filter.go new file mode 100644 index 000000000..2e9e891fa --- /dev/null +++ b/source_filter.go @@ -0,0 +1,96 @@ +package dalec + +import ( + "path/filepath" + + "github.com/moby/buildkit/client/llb" +) + +const ( + BuildArgDalecSourceFilterConfigPath = "DALEC_SOURCE_FILTER_CONFIG_PATH" + BuildArgDalecSourceFilterContextName = "DALEC_SOURCE_FILTER_CONFIG_CONTEXT_NAME" + DefaultSourceOptionsContextName = "dalec-source-options" +) + +// SourceFilterConfig configures build-time filtering for source package inputs. +// It is intentionally global; future versions may add more specific filter +// scopes alongside GlobalExcludes. +type SourceFilterConfig struct { + GlobalExcludes []string `yaml:"global_excludes,omitempty" json:"global_excludes,omitempty"` +} + +func (sOpt SourceOpts) GetSourceFilter() (SourceFilterConfig, error) { + if sOpt.SourceFilter == nil { + return SourceFilterConfig{}, nil + } + return sOpt.SourceFilter() +} + +func (cfg SourceFilterConfig) IsEmpty() bool { + return len(cfg.GlobalExcludes) == 0 +} + +func (sOpt SourceOpts) sourceFilterExcludes() ([]string, error) { + cfg, err := sOpt.GetSourceFilter() + if err != nil { + return nil, err + } + return cfg.GlobalExcludes, nil +} + +func sourceFilter(sOpt SourceOpts, opts ...llb.ConstraintsOpt) llb.StateOption { + return func(in llb.State) llb.State { + excludes, err := sOpt.sourceFilterExcludes() + if err != nil { + return ErrorState(in, err) + } + if len(excludes) == 0 { + return in + } + return in.With(SourceFilter(SourceFilterConfig{GlobalExcludes: excludes}, opts...)) + } +} + +func sourceFilterAtPath(sOpt SourceOpts, base string, opts ...llb.ConstraintsOpt) llb.StateOption { + return func(in llb.State) llb.State { + excludes, err := sOpt.sourceFilterExcludes() + if err != nil { + return ErrorState(in, err) + } + if len(excludes) == 0 { + return in + } + return in.With(SourceFilterAtPath(base, SourceFilterConfig{GlobalExcludes: excludes}, opts...)) + } +} + +// SourceFilter filters source package content from the root of the input state. +func SourceFilter(cfg SourceFilterConfig, opts ...llb.ConstraintsOpt) llb.StateOption { + return func(in llb.State) llb.State { + if cfg.IsEmpty() { + return in + } + return llb.Scratch().File(llb.Copy(in, "/", "/", WithDirContentsOnly(), WithExcludes(cfg.GlobalExcludes)), opts...) + } +} + +// SourceFilterAtPath applies a global source filter to content nested under base. +// The external config remains global while named source states keep their source +// name as a top-level directory in package source assembly. +func SourceFilterAtPath(base string, cfg SourceFilterConfig, opts ...llb.ConstraintsOpt) llb.StateOption { + return func(in llb.State) llb.State { + if cfg.IsEmpty() { + return in + } + if isRoot(base) { + return in.With(SourceFilter(cfg, opts...)) + } + + excludes := make([]string, 0, len(cfg.GlobalExcludes)) + for _, exclude := range cfg.GlobalExcludes { + excludes = append(excludes, filepath.ToSlash(filepath.Join(base, exclude))) + } + + return SourceFilter(SourceFilterConfig{GlobalExcludes: excludes}, opts...)(in) + } +} diff --git a/source_inline.go b/source_inline.go index b0dad2f6c..8b81e79f2 100644 --- a/source_inline.go +++ b/source_inline.go @@ -251,7 +251,25 @@ func (s *SourceInlineFile) toMount(opts fetchOptions) (llb.State, []llb.MountOpt func (s *SourceInlineDir) toState(opts fetchOptions) llb.State { base := s.baseState(opts) // inline dir handles dir names and subpaths itself - // Do not pass rename to sourceFilters + // Include/exclude patterns are relative to the requested source root, but + // inline dirs create files under Rename before sourceFilters runs. + if opts.Rename != "" { + if len(opts.Includes) > 0 { + includes := make([]string, len(opts.Includes)) + for i, include := range opts.Includes { + includes[i] = filepath.ToSlash(filepath.Join(opts.Rename, include)) + } + opts.Includes = includes + } + if len(opts.Excludes) > 0 { + excludes := make([]string, len(opts.Excludes)) + for i, exclude := range opts.Excludes { + excludes[i] = filepath.ToSlash(filepath.Join(opts.Rename, exclude)) + } + opts.Excludes = excludes + } + } + // Do not pass rename to sourceFilters as a destination rename. opts.Rename = "" return base.With(sourceFilters(opts)) } diff --git a/source_test.go b/source_test.go index 1d361103b..5c6ce1e51 100644 --- a/source_test.go +++ b/source_test.go @@ -318,6 +318,127 @@ func TestSourceHTTP(t *testing.T) { }) } +func TestSourceFilterAtPath(t *testing.T) { + t.Parallel() + + ctx := context.Background() + filter := SourceFilterConfig{GlobalExcludes: []string{"nested/bad.txt", "*.tmp"}} + st := llb.Scratch().File(llb.Mkfile("src/nested/bad.txt", 0o644, nil)).With(SourceFilterAtPath("src", filter)) + + def, err := st.Marshal(ctx) + assert.NilError(t, err) + + var fileOp *pb.FileOp + for _, dt := range def.Def { + var op pb.Op + assert.NilError(t, op.Unmarshal(dt)) + if op.GetFile() != nil { + fileOp = op.GetFile() + } + } + if fileOp == nil { + t.Fatal("expected file op") + } + + cp := fileOp.Actions[0].GetCopy() + if cp == nil { + t.Fatal("expected copy action") + } + assert.Check(t, cmp.DeepEqual(cp.ExcludePatterns, []string{"src/nested/bad.txt", "src/*.tmp"})) +} + +func TestSourceFilter(t *testing.T) { + t.Parallel() + + ctx := context.Background() + filter := SourceFilterConfig{GlobalExcludes: []string{"bad.txt"}} + st := llb.Scratch().File(llb.Mkfile("bad.txt", 0o644, nil)).With(SourceFilter(filter)) + + def, err := st.Marshal(ctx) + assert.NilError(t, err) + + var fileOp *pb.FileOp + for _, dt := range def.Def { + var op pb.Op + assert.NilError(t, op.Unmarshal(dt)) + if op.GetFile() != nil { + fileOp = op.GetFile() + } + } + if fileOp == nil { + t.Fatal("expected file op") + } + + cp := fileOp.Actions[0].GetCopy() + if cp == nil { + t.Fatal("expected copy action") + } + assert.Check(t, cmp.DeepEqual(cp.ExcludePatterns, filter.GlobalExcludes)) +} + +func TestSourceFilterEmptyNoop(t *testing.T) { + t.Parallel() + + ctx := context.Background() + base := llb.Scratch().File(llb.Mkfile("keep.txt", 0o644, nil)) + filtered := base.With(SourceFilter(SourceFilterConfig{})) + + baseDef, err := base.Marshal(ctx) + assert.NilError(t, err) + filteredDef, err := filtered.Marshal(ctx) + assert.NilError(t, err) + + assert.Check(t, cmp.DeepEqual(filteredDef.Def, baseDef.Def)) +} + +func TestNodeModDepsSourceFilter(t *testing.T) { + t.Parallel() + + ctx := context.Background() + spec := &Spec{Sources: map[string]Source{ + "src": { + Inline: &SourceInline{Dir: &SourceInlineDir{Files: map[string]*SourceInlineFile{ + "package.json": {Contents: "{}"}, + }}}, + Generate: []*SourceGenerator{{NodeMod: &GeneratorNodeMod{}}}, + }, + }} + spec.FillDefaults() + + result, err := spec.NodeModDeps(SourceOpts{SourceFilter: func() (SourceFilterConfig, error) { + return SourceFilterConfig{GlobalExcludes: []string{"node_modules/**"}}, nil + }}, llb.Scratch()) + assert.NilError(t, err) + + st, ok := result["src"] + if !ok { + t.Fatal("expected generated node module source") + } + + def, err := st.Marshal(ctx) + assert.NilError(t, err) + + for _, dt := range def.Def { + var op pb.Op + assert.NilError(t, op.Unmarshal(dt)) + fileOp := op.GetFile() + if fileOp == nil { + continue + } + for _, action := range fileOp.Actions { + cp := action.GetCopy() + if cp == nil { + continue + } + if slices.Equal(cp.ExcludePatterns, []string{"src/node_modules/**"}) { + return + } + } + } + + t.Fatal("expected node module dependency output to be filtered with source-prefixed excludes") +} + func toImageRef(ref string) string { return "docker-image://" + ref } @@ -688,6 +809,16 @@ func TestSourceContext(t *testing.T) { checkContext(t, ops[0].GetSource(), &src) testWithFilters(t, src) }) + + t.Run("with source filter", func(t *testing.T) { + src := Source{Context: &SourceContext{}} + sOpt := prepareGetSourceOp(ctx, t, &src) + sOpt.SourceFilter = func() (SourceFilterConfig, error) { + return SourceFilterConfig{GlobalExcludes: []string{"drop.txt"}}, nil + } + ops := getSourceOpWithOpts(ctx, t, src, sOpt) + checkContext(t, ops[0].GetSource(), &Source{Context: &SourceContext{}, Excludes: []string{"drop.txt"}}) + }) } func TestSourceInlineFile(t *testing.T) { @@ -808,6 +939,26 @@ func TestSourceInlineDir(t *testing.T) { }) }) } + + t.Run("with source filter", func(t *testing.T) { + src := Source{Inline: &SourceInline{Dir: &SourceInlineDir{Files: map[string]*SourceInlineFile{ + "keep.txt": {Contents: "keep"}, + "drop.txt": {Contents: "drop"}, + }}}} + sOpt := SourceOpts{SourceFilter: func() (SourceFilterConfig, error) { + return SourceFilterConfig{GlobalExcludes: []string{"drop.txt"}}, nil + }} + ops := getSourceOpWithOpts(ctx, t, src, sOpt) + if len(ops) != 4 { + t.Fatalf("expected mkdir, two mkfile ops, and filter copy op, got %d", len(ops)) + } + + cp := ops[3].GetFile().Actions[0].GetCopy() + if cp == nil { + t.Fatal("expected copy action") + } + assert.Check(t, cmp.DeepEqual(cp.ExcludePatterns, []string{"drop.txt"})) + }) } func checkMkdir(t *testing.T, op *pb.FileOp, src *SourceInlineDir) { @@ -990,6 +1141,17 @@ func getSourceOp(ctx context.Context, t *testing.T, src Source) []*pb.Op { return sourceOpsFromState(ctx, t, st) } +func getSourceOpWithOpts(ctx context.Context, t *testing.T, src Source, sOpt SourceOpts) []*pb.Op { + t.Helper() + src.fillDefaults() + name := "" + if !src.IsDir() { + name = "test" + } + st := src.ToState(name, sOpt) + return sourceOpsFromState(ctx, t, st) +} + func getMountOp(ctx context.Context, t *testing.T, src Source, target string) []*pb.Op { t.Helper() diff --git a/spec_test.go b/spec_test.go index d342f0c56..cebac5960 100644 --- a/spec_test.go +++ b/spec_test.go @@ -34,3 +34,23 @@ func TestDate(t *testing.T) { assert.NilError(t, err) assert.Check(t, cmp.Equal(d3.Format(time.DateOnly), expect)) } + +func TestSourceFilterConfig(t *testing.T) { + t.Parallel() + + var cfg SourceFilterConfig + err := yaml.Unmarshal([]byte(` +global_excludes: + - github.com/klauspost/compress@*/zip/corpus/14.zip + - cache/download/github.com/klauspost/compress/@v/*.zip +`), &cfg) + assert.NilError(t, err) + assert.Check(t, cmp.DeepEqual(cfg.GlobalExcludes, []string{ + "github.com/klauspost/compress@*/zip/corpus/14.zip", + "cache/download/github.com/klauspost/compress/@v/*.zip", + })) + assert.Check(t, !cfg.IsEmpty()) + + var empty SourceFilterConfig + assert.Check(t, empty.IsEmpty()) +} diff --git a/test/linux_target_test.go b/test/linux_target_test.go index 09428ec81..1eae2ca73 100644 --- a/test/linux_target_test.go +++ b/test/linux_target_test.go @@ -26,7 +26,6 @@ import ( moby_buildkit_v1_frontend "github.com/moby/buildkit/frontend/gateway/pb" "github.com/moby/go-archive/compression" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" - pkgerrors "github.com/pkg/errors" "golang.org/x/exp/maps" "gotest.tools/v3/assert" "gotest.tools/v3/assert/cmp" @@ -95,8 +94,6 @@ type testLinuxConfig struct { Worker workerConfig Release OSRelease - SkipStripTest bool - Platforms []ocispecs.Platform PackageOutputPath func(spec *dalec.Spec, platform ocispecs.Platform) string } @@ -3205,13 +3202,8 @@ func testMixGlobalTargetDependencies(ctx context.Context, t *testing.T, cfg test } func testDisableStrip(ctx context.Context, t *testing.T, cfg testLinuxConfig) { - skip.If(t, cfg.SkipStripTest, "skipping test as it is not supported for this target: "+cfg.Target.Container) - newSpec := func() *dalec.Spec { spec := newSimpleSpec() - spec.Args = map[string]string{ - "TARGETARCH": "", - } spec.Sources = map[string]dalec.Source{ "src": { @@ -3236,44 +3228,46 @@ func testDisableStrip(ctx context.Context, t *testing.T, cfg testLinuxConfig) { Build: map[string]dalec.PackageConstraints{ cfg.GetPackage("golang"): {}, }, + Test: []string{ + "bash", + "coreutils", + "binutils", + }, } spec.Artifacts = dalec.Artifacts{ Binaries: map[string]dalec.ArtifactConfig{ - "bad-executable": {}, - }, - Libs: map[string]dalec.ArtifactConfig{ - "bad-executable": {}, + "the-executable": {}, }, } - spec.Build.Env = map[string]string{ - "TARGETARCH": "$TARGETARCH", - } - spec.Build.Steps = []dalec.BuildStep{ - // Build a binary for a different architecture - // This should make `strip` fail. - // - // Note: The test is specifically using ppc64le as GOARCH - // because it seems alma/rockylinux do not error ons trip except for ppc64le. - // Even this is a stretch as that does not even work as expected at version < v9. { - Command: `cd src; if [ "${TARGETARCH}" = "ppc64le" ]; then export GOARCH=amd64; else export GOARCH=ppc64le; fi; go build -o ../bad-executable main.go`, + Command: "cd src; go build -o ../the-executable main.go", }, } + return spec } t.Run("strip enabled", func(t *testing.T) { - // Make sure that we get a build failure when strip is enabled t.Parallel() ctx := startTestSpan(ctx, t) testEnv.RunTest(ctx, t, func(ctx context.Context, client gwclient.Client) { spec := newSpec() - req := newSolveRequest(withSpec(ctx, t, spec), withBuildTarget(cfg.Target.Container)) - _, err := client.Solve(ctx, req) - assert.ErrorType(t, pkgerrors.Cause(err), &moby_buildkit_v1_frontend.ExitError{}) + spec.Tests = append(spec.Tests, &dalec.TestSpec{ + Name: "Check that binary IS stripped", + Steps: []dalec.TestStep{ + { + // dalec test assertions can't handle negative checks directly, + // so we use exit code 42 to indicate presence of debug info and fail the test + Command: `/bin/bash -eo pipefail -c "grep -q '\.debug_info' < <(readelf -S /usr/bin/the-executable) && exit 42 || exit 0"`, + }, + }, + }) + + req := newSolveRequest(withSpec(ctx, t, spec), withBuildTarget(cfg.Target.Package)) + solveT(ctx, t, client, req) }) }) @@ -3284,7 +3278,16 @@ func testDisableStrip(ctx context.Context, t *testing.T, cfg testLinuxConfig) { spec := newSpec() spec.Artifacts.DisableStrip = true - req := newSolveRequest(withSpec(ctx, t, spec), withBuildTarget(cfg.Target.Container)) + spec.Tests = append(spec.Tests, &dalec.TestSpec{ + Name: "Check that binary is NOT stripped", + Steps: []dalec.TestStep{ + { + Command: `/bin/bash -eo pipefail -c "grep -q '\.debug_info' < <(readelf -S /usr/bin/the-executable)"`, + }, + }, + }) + + req := newSolveRequest(withSpec(ctx, t, spec), withBuildTarget(cfg.Target.Package)) solveT(ctx, t, client, req) }) }) diff --git a/test/source_test.go b/test/source_test.go index dc92f00f3..8b3c1ad4b 100644 --- a/test/source_test.go +++ b/test/source_test.go @@ -992,6 +992,115 @@ func TestSourceWithCargohome(t *testing.T) { }) } +func TestDebugGomodSourceFilterConfig(t *testing.T) { + t.Parallel() + + spec := &dalec.Spec{ + Name: "test-source-filter-gomod", + Version: "0.0.1", + Revision: "1", + License: "MIT", + Website: "https://github.com/project-dalec/dalec", + Vendor: "Dalec", + Packager: "Dalec", + Description: "Testing source filter config with gomod", + Sources: map[string]dalec.Source{ + "src": { + Generate: []*dalec.SourceGenerator{{Gomod: &dalec.GeneratorGomod{}}}, + Inline: &dalec.SourceInline{ + Dir: &dalec.SourceInlineDir{ + Files: map[string]*dalec.SourceInlineFile{ + "main.go": {Contents: gomodFixtureMain}, + "go.mod": {Contents: gomodFixtureMod}, + "go.sum": {Contents: gomodFixtureSum}, + }, + }, + }, + }, + }, + Dependencies: &dalec.PackageDependencies{ + Build: map[string]dalec.PackageConstraints{ + "golang": {}, + }, + }, + } + + filterConfig := llb.Scratch().File(llb.Mkfile("/source-filter.yml", 0o644, []byte(` +global_excludes: + - github.com/cpuguy83/tar2go@v0.3.1 +`))) + + runTest(t, func(ctx context.Context, gwc gwclient.Client) { + req := newSolveRequest( + withBuildTarget("debug/gomods"), + withSpec(ctx, t, spec), + withBuildContext(ctx, t, dalec.DefaultSourceOptionsContextName, filterConfig), + withBuildArg(dalec.BuildArgDalecSourceFilterConfigPath, "/source-filter.yml"), + ) + + res := solveT(ctx, t, gwc, req) + ref, err := res.SingleRef() + assert.NilError(t, err) + + _, err = ref.StatFile(ctx, gwclient.StatRequest{Path: "github.com/cpuguy83/tar2go@v0.3.1"}) + assert.Assert(t, err != nil, "expected filtered gomod directory to be absent") + + _, err = ref.StatFile(ctx, gwclient.StatRequest{Path: "cache"}) + assert.NilError(t, err) + }) +} + +func TestDebugSourcesSourceFilterConfig(t *testing.T) { + t.Parallel() + + spec := &dalec.Spec{ + Name: "test-source-filter-debug-sources", + Version: "0.0.1", + Revision: "1", + License: "MIT", + Website: "https://github.com/project-dalec/dalec", + Vendor: "Dalec", + Packager: "Dalec", + Description: "Testing source filter config with debug sources", + Sources: map[string]dalec.Source{ + "src": { + Inline: &dalec.SourceInline{ + Dir: &dalec.SourceInlineDir{ + Files: map[string]*dalec.SourceInlineFile{ + "keep.txt": {Contents: "keep"}, + "drop.txt": {Contents: "drop"}, + }, + }, + }, + }, + }, + } + + filterConfig := llb.Scratch().File(llb.Mkfile("/source-filter.yml", 0o644, []byte(` +global_excludes: + - drop.txt +`))) + + runTest(t, func(ctx context.Context, gwc gwclient.Client) { + req := newSolveRequest( + withBuildTarget("debug/sources"), + withSpec(ctx, t, spec), + withBuildContext(ctx, t, dalec.DefaultSourceOptionsContextName, filterConfig), + withBuildArg(dalec.BuildArgDalecSourceFilterConfigPath, "/source-filter.yml"), + ) + + res := solveT(ctx, t, gwc, req) + ref, err := res.SingleRef() + assert.NilError(t, err) + + _, err = ref.StatFile(ctx, gwclient.StatRequest{Path: "src/keep.txt"}) + assert.NilError(t, err) + + _, err = ref.StatFile(ctx, gwclient.StatRequest{Path: "src/drop.txt"}) + assert.Assert(t, err != nil, "expected filtered source file to be absent") + }) +} + func TestSourceContext(t *testing.T) { t.Parallel() diff --git a/test/target_almalinux_test.go b/test/target_almalinux_test.go index 08016c985..90d2041df 100644 --- a/test/target_almalinux_test.go +++ b/test/target_almalinux_test.go @@ -90,7 +90,6 @@ func TestAlmalinux8(t *testing.T) { ID: "almalinux", VersionID: "8", }, - SkipStripTest: true, Platforms: []ocispecs.Platform{ {OS: "linux", Architecture: "amd64"}, {OS: "linux", Architecture: "arm64"}, diff --git a/test/target_rockylinux_test.go b/test/target_rockylinux_test.go index 8ee3948fb..8df074f93 100644 --- a/test/target_rockylinux_test.go +++ b/test/target_rockylinux_test.go @@ -90,7 +90,6 @@ func TestRockylinux8(t *testing.T) { ID: "rocky", VersionID: "8", }, - SkipStripTest: true, Platforms: []ocispecs.Platform{ {OS: "linux", Architecture: "amd64"}, {OS: "linux", Architecture: "arm64"}, diff --git a/website/docs/sources.md b/website/docs/sources.md index c26aa596f..6374ad3d7 100644 --- a/website/docs/sources.md +++ b/website/docs/sources.md @@ -339,6 +339,39 @@ Build sources are considered to be "directory" sources. Generators are used to generate a source from another source. Currently the generators supported are `gomod`, `cargohome`, `pip`, and `nodemod`. +## Build-time source filters + +Dalec can load a build-time source filter configuration from a build context. +This is useful when a packaging or signing environment must reject specific +source files, but the package spec should not be changed. + +The filter config is a YAML file with a `global_excludes` list: + +```yaml +global_excludes: + - "testdata/large-fixture.zip" + - "vendor/**/examples/**" +``` + +Pass the config path with `DALEC_SOURCE_FILTER_CONFIG_PATH`. The file is read +from the `dalec-source-options` build context unless +`DALEC_SOURCE_FILTER_CONFIG_CONTEXT_NAME` names a different build context. + +Example with Docker Buildx: + +```console +$ docker buildx build \ + --build-arg DALEC_SOURCE_FILTER_CONFIG_PATH=source-filter.yml \ + --build-context dalec-source-options=./ci/dalec \ + ... +``` + +`global_excludes` patterns are evaluated relative to each source root. For +normal directory sources this uses the same filtering path as `sources.*.excludes`; +local context filters are pushed into BuildKit context loading. For generated +aggregate sources such as `gomod`, patterns are relative to the generated cache +root and are applied after generation. + ### Gomod The `gomod` generator manages a single go module cache for all sources that