Apply source filters without requiring build args

frontend/pkg/bkfs/bkfs.go

@@ -17,6 +17,7 @@ import (
 var (
 	_ fs.DirEntry    = (*stateRefDirEntry)(nil)
 	_ fs.ReadDirFS   = (*StateRefFS)(nil)
+	_ fs.ReadFileFS  = (*StateRefFS)(nil)
 	_ io.ReaderAt    = (*stateRefFile)(nil)
 	_ fs.ReadDirFile = (*stateRefFile)(nil)
 	_ fs.ReadDirFS   = (*nullFS)(nil)
@@ -118,6 +119,29 @@ func (st *StateRefFS) ReadDir(name string) ([]fs.DirEntry, error) {
 	return entries, nil
 }
 
+func (st *StateRefFS) ReadFile(name string) ([]byte, error) {
+	if !fs.ValidPath(name) {
+		return nil, &fs.PathError{Err: fs.ErrInvalid, Path: name, Op: "readfile"}
+	}
+
+	dt, err := st.ref.ReadFile(st.ctx, gwclient.ReadRequest{
+		Filename: name,
+	})
+	if err != nil {
+		return nil, pathError("readfile", name, err)
+	}
+	return dt, nil
+}
+
+// pathError converts a buildkit error into the [io/fs] error the caller expects.
+// Buildkit gives us no typed error for a missing path, so match on the message.
+func pathError(op, name string, err error) error {
+	if strings.Contains(err.Error(), "no such file or directory") {
+		err = fmt.Errorf("%w: %w", fs.ErrNotExist, err)
+	}
+	return &fs.PathError{Err: err, Op: op, Path: name}
+}
+
 type stateRefFile struct {
 	eof    bool   // has file been read to EOF?
 	path   string // the full path of the file from root
@@ -205,10 +229,7 @@ func (st *StateRefFS) Open(name string) (fs.File, error) {
 		Path: name,
 	})
 	if err != nil {
-		if strings.Contains(err.Error(), "no such file or directory") {
-			err = fmt.Errorf("%w: %w", fs.ErrNotExist, err)
-		}
-		return nil, &fs.PathError{Err: err, Op: "open", Path: name}
+		return nil, pathError("open", name, err)
 	}
 
 	f := &stateRefFile{

frontend/request.go

@@ -2,6 +2,9 @@ package frontend
 
 import (
 	"context"
+	"fmt"
+	"io/fs"
+	"path"
 	"strconv"
 	"strings"
 
@@ -13,6 +16,7 @@ import (
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/pkg/errors"
 	"github.com/project-dalec/dalec"
+	"github.com/project-dalec/dalec/frontend/pkg/bkfs"
 )
 
 const (
@@ -155,25 +159,6 @@ func getSigningConfigFromContext(ctx context.Context, client gwclient.Client, cf
 	return pc.Signer, nil
 }
 
-func getSourceFilterConfigFromContext(ctx context.Context, client gwclient.Client, cfgPath string, configCtxName string, getContext func(string, ...llb.LocalOption) (*llb.State, error), opts ...llb.ConstraintsOpt) (dalec.SourceFilterConfig, error) {
-	dt, err := readConfigFromContext(ctx, client, cfgPath, configCtxName, dalec.SourceOpts{
-		GetContext: getContext,
-	}, opts...)
-	if err != nil {
-		return dalec.SourceFilterConfig{}, err
-	}
-
-	return decodeSourceFilterConfig(ctx, dt)
-}
-
-func decodeSourceFilterConfig(ctx context.Context, dt []byte) (dalec.SourceFilterConfig, error) {
-	var cfg dalec.SourceFilterConfig
-	if err := yaml.UnmarshalContext(ctx, dt, &cfg, yaml.Strict()); err != nil {
-		return dalec.SourceFilterConfig{}, err
-	}
-	return cfg, nil
-}
-
 func readConfigFromContext(ctx context.Context, client gwclient.Client, cfgPath string, configCtxName string, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]byte, error) {
 	src := dalec.Source{Path: cfgPath, Context: &dalec.SourceContext{Name: configCtxName}}
 	configState := src.ToState("", dalec.SourceOpts{GetContext: sOpt.GetContext}, opts...)
@@ -265,25 +250,57 @@ func getSignConfigCtxName(client gwclient.Client) string {
 	return client.BuildOpts().Opts["build-arg:"+buildArgDalecSigningConfigContextName]
 }
 
-func getSourceFilterConfigPath(client gwclient.Client) string {
-	return client.BuildOpts().Opts["build-arg:"+dalec.BuildArgDalecSourceFilterConfigPath]
-}
+func loadSourceFilterConfig(ctx context.Context, client gwclient.Client, getContext func(string, ...llb.LocalOption) (*llb.State, error)) (dalec.SourceFilterConfig, error) {
+	var (
+		nopFilter           dalec.SourceFilterConfig
+		clientSetFilterArgs bool
+
+		opts = client.BuildOpts().Opts
+	)
+
+	cfgPath := dalec.DefaultSourceFilterConfigPath
+	if p := opts["build-arg:"+dalec.BuildArgDalecSourceFilterConfigPath]; p != "" {
+		clientSetFilterArgs = true
+		cfgPath = p
+	}
 
-func getSourceFilterContextNameWithDefault(client gwclient.Client) string {
 	configCtxName := dalec.DefaultSourceOptionsContextName
-	if cn := client.BuildOpts().Opts["build-arg:"+dalec.BuildArgDalecSourceFilterContextName]; cn != "" {
+	if cn := opts["build-arg:"+dalec.BuildArgDalecSourceFilterContextName]; cn != "" {
+		clientSetFilterArgs = true
 		configCtxName = cn
 	}
-	return configCtxName
-}
 
-func loadSourceFilterConfig(ctx context.Context, client gwclient.Client, getContext func(string, ...llb.LocalOption) (*llb.State, error)) (dalec.SourceFilterConfig, error) {
-	cfgPath := getSourceFilterConfigPath(client)
-	if cfgPath == "" {
-		return dalec.SourceFilterConfig{}, nil
+	contextPath := strings.TrimPrefix(path.Clean(cfgPath), "/")
+	st, err := getContext(configCtxName, llb.IncludePatterns([]string{contextPath}), llb.FollowPaths([]string{contextPath}))
+	if err != nil {
+		return nopFilter, fmt.Errorf("error getting global filter context: %w", err)
 	}
+	if st == nil {
+		if clientSetFilterArgs {
+			// The client specifically set the build-args like the context would be included.
+			// No context was supplied, so assume there is a problem.
+			return nopFilter, fmt.Errorf("client set filter args but context is not available: context=%q", configCtxName)
+		}
 
-	return getSourceFilterConfigFromContext(ctx, client, cfgPath, getSourceFilterContextNameWithDefault(client), getContext)
+		// Context doesn't exist, no filters.
+		return nopFilter, nil
+	}
+
+	fSys, err := bkfs.FromState(ctx, st, client)
+	if err != nil {
+		return nopFilter, fmt.Errorf("error solving global filter context: %w", err)
+	}
+
+	dt, err := fs.ReadFile(fSys, contextPath)
+	if err != nil {
+		return nopFilter, errors.Wrapf(err, "error reading source filter config %q", cfgPath)
+	}
+
+	var cfg dalec.SourceFilterConfig
+	if err := yaml.UnmarshalContext(ctx, dt, &cfg, yaml.Strict()); err != nil {
+		return nopFilter, errors.Wrapf(err, "error decoding source filter config %q", cfgPath)
+	}
+	return cfg, nil
 }
 
 func forwardToSigner(ctx context.Context, client gwclient.Client, cfg *dalec.PackageSigner, s llb.State, opts ...llb.ConstraintsOpt) (llb.State, error) {

frontend/request_test.go

@@ -0,0 +1,102 @@
+package frontend
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/moby/buildkit/client/llb"
+	"github.com/project-dalec/dalec"
+)
+
+func TestSourceFilterConfig(t *testing.T) {
+	t.Parallel()
+
+	filterArgs := []struct {
+		name        string
+		args        []stubOpt
+		wantContext string
+	}{
+		{
+			name:        "a config path build arg",
+			args:        []stubOpt{withStubBuildArg(dalec.BuildArgDalecSourceFilterConfigPath, "custom-filter.yml")},
+			wantContext: dalec.DefaultSourceOptionsContextName,
+		},
+		{
+			name:        "a context name build arg",
+			args:        []stubOpt{withStubBuildArg(dalec.BuildArgDalecSourceFilterContextName, "other-context")},
+			wantContext: "other-context",
+		},
+		{
+			name: "both filter build args",
+			args: []stubOpt{
+				withStubBuildArg(dalec.BuildArgDalecSourceFilterConfigPath, "custom-filter.yml"),
+				withStubBuildArg(dalec.BuildArgDalecSourceFilterContextName, "other-context"),
+			},
+			wantContext: "other-context",
+		},
+	}
+
+	// A context that is not part of the build resolves to nil. The stub client
+	// cannot solve, so reading a config at all fails the test.
+	var requestedContext string
+	getContext := func(name string, _ ...llb.LocalOption) (*llb.State, error) {
+		requestedContext = name
+		return nil, nil
+	}
+
+	t.Run("a build without the source options context is not filtered", func(t *testing.T) {
+		cfg, err := loadSourceFilterConfig(t.Context(), newStubClient(), getContext)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !cfg.IsEmpty() {
+			t.Fatalf("expected no filtering, got %v", cfg.GlobalExcludes)
+		}
+		if requestedContext != dalec.DefaultSourceOptionsContextName {
+			t.Errorf("expected build context %q to be looked up, got %q", dalec.DefaultSourceOptionsContextName, requestedContext)
+		}
+	})
+
+	for _, tc := range filterArgs {
+		t.Run(tc.name+" without the source options context fails the build", func(t *testing.T) {
+			_, err := loadSourceFilterConfig(t.Context(), newStubClient(tc.args...), getContext)
+			if err == nil {
+				t.Fatal("expected an error when the build asks for a filter config the context cannot provide")
+			}
+			if requestedContext != tc.wantContext {
+				t.Errorf("expected build context %q to be looked up, got %q", tc.wantContext, requestedContext)
+			}
+			if !strings.Contains(err.Error(), tc.wantContext) {
+				t.Errorf("expected error to name build context %q, got %v", tc.wantContext, err)
+			}
+		})
+	}
+}
+
+func TestSourceFilterConfigUsesContextRelativePath(t *testing.T) {
+	t.Parallel()
+
+	var gotOpts []llb.LocalOption
+	getContext := func(_ string, opts ...llb.LocalOption) (*llb.State, error) {
+		gotOpts = opts
+		return nil, nil
+	}
+
+	client := newStubClient(withStubBuildArg(dalec.BuildArgDalecSourceFilterConfigPath, "/./source-filter.yml"))
+	if _, err := loadSourceFilterConfig(t.Context(), client, getContext); err == nil {
+		t.Fatal("expected an error because the source options context is missing")
+	}
+
+	var li llb.LocalInfo
+	for _, o := range gotOpts {
+		o.SetLocalOption(&li)
+	}
+
+	const want = `["source-filter.yml"]`
+	if li.IncludePatterns != want {
+		t.Errorf("expected include patterns %s, got %s", want, li.IncludePatterns)
+	}
+	if li.FollowPaths != want {
+		t.Errorf("expected follow paths %s, got %s", want, li.FollowPaths)
+	}
+}

source_filter.go

@@ -10,6 +10,11 @@ const (
 	BuildArgDalecSourceFilterConfigPath  = "DALEC_SOURCE_FILTER_CONFIG_PATH"
 	BuildArgDalecSourceFilterContextName = "DALEC_SOURCE_FILTER_CONFIG_CONTEXT_NAME"
 	DefaultSourceOptionsContextName      = "dalec-source-options"
+
+	// DefaultSourceFilterConfigPath is the path, relative to the source options
+	// build context, that the source filter config is read from when
+	// [BuildArgDalecSourceFilterConfigPath] is not set.
+	DefaultSourceFilterConfigPath = "source-filter.yml"
 )
 
 // SourceFilterConfig configures build-time filtering for source package inputs.
@@ -39,16 +44,7 @@ func (sOpt SourceOpts) sourceFilterExcludes() ([]string, error) {
 }
 
 func sourceFilter(sOpt SourceOpts, opts ...llb.ConstraintsOpt) llb.StateOption {
-	return func(in llb.State) llb.State {
-		excludes, err := sOpt.sourceFilterExcludes()
-		if err != nil {
-			return ErrorState(in, err)
-		}
-		if len(excludes) == 0 {
-			return in
-		}
-		return in.With(SourceFilter(SourceFilterConfig{GlobalExcludes: excludes}, opts...))
-	}
+	return sourceFilterAtPath(sOpt, "", opts...)
 }
 
 func sourceFilterAtPath(sOpt SourceOpts, base string, opts ...llb.ConstraintsOpt) llb.StateOption {
@@ -60,37 +56,20 @@ func sourceFilterAtPath(sOpt SourceOpts, base string, opts ...llb.ConstraintsOpt
 		if len(excludes) == 0 {
 			return in
 		}
-		return in.With(SourceFilterAtPath(base, SourceFilterConfig{GlobalExcludes: excludes}, opts...))
-	}
-}
-
-// SourceFilter filters source package content from the root of the input state.
-func SourceFilter(cfg SourceFilterConfig, opts ...llb.ConstraintsOpt) llb.StateOption {
-	return func(in llb.State) llb.State {
-		if cfg.IsEmpty() {
-			return in
-		}
-		return llb.Scratch().File(llb.Copy(in, "/", "/", WithDirContentsOnly(), WithExcludes(cfg.GlobalExcludes)), opts...)
-	}
-}
-
-// SourceFilterAtPath applies a global source filter to content nested under base.
-// The external config remains global while named source states keep their source
-// name as a top-level directory in package source assembly.
-func SourceFilterAtPath(base string, cfg SourceFilterConfig, opts ...llb.ConstraintsOpt) llb.StateOption {
-	return func(in llb.State) llb.State {
-		if cfg.IsEmpty() {
-			return in
-		}
-		if isRoot(base) {
-			return in.With(SourceFilter(cfg, opts...))
-		}
 
-		excludes := make([]string, 0, len(cfg.GlobalExcludes))
-		for _, exclude := range cfg.GlobalExcludes {
-			excludes = append(excludes, filepath.ToSlash(filepath.Join(base, exclude)))
+		if !isRoot(base) {
+			// prepend the base path to each excluded path
+			joined := make([]string, 0, len(excludes))
+			for _, path := range excludes {
+				newPath := filepath.ToSlash(filepath.Join(base, path))
+				joined = append(joined, newPath)
+			}
+			excludes = joined
 		}
 
-		return SourceFilter(SourceFilterConfig{GlobalExcludes: excludes}, opts...)(in)
+		return llb.Scratch().File(
+			llb.Copy(in, "/", "/", WithDirContentsOnly(), WithExcludes(excludes)),
+			opts...,
+		)
 	}
 }