diff --git a/common/filesystem/fs.go b/common/filesystem/fs.go index 46ef81c1..3d5dabaa 100644 --- a/common/filesystem/fs.go +++ b/common/filesystem/fs.go @@ -46,11 +46,23 @@ type Provider interface { // entire file, but should use optimized methods like fallocate() or truncate(). This also means // it is not safe to rely on CreatePreallocatedFile to securely wipe an existing file. CreatePreallocatedFile(name string, size int64, overwrite bool) error + // Creates or resizes a file at the specified path, returning an error if the file already + // exists unless overwrite is true. If overwrite is true and the file already exists, the file + // will be resized to the specified size without first zeroing or truncating it to zero. If the + // file is extended, the new region may be sparse and may not reserve physical disk space. If + // the file is reduced, data beyond the specified size is discarded and cannot be restored by + // later extending the file. This is useful for setting the logical size of a file before + // subsequent operations, such as a multipart download, but callers must still handle write-time + // space errors. Note that expanding a file with overwrite==true will not cause and error to be + // returned. + CreateOrResizeFile(name string, size int64, overwrite bool) error // CreateWriteClose creates the file specified by name and immediately writes the specified buf // as the file contents then closes the file. CreateWriteClose(name string, buf []byte, mode uint32, overwrite bool) error // Removes the file specified by name. Remove(name string) error + // Removes the specified path and any children it contains. + RemoveAll(name string) error // Opens the file specified by name and returns it as an io.ReadCloser. The caller must close the // file when it is no longer required. Open(name string) (io.ReadCloser, error) @@ -82,6 +94,8 @@ type Provider interface { CopyOwnerAndMode(fromStat fs.FileInfo, dstPath string) error // CopyTimestamps sets the atime/mtime in fromStat on dstPath. CopyTimestamps(fromStat fs.FileInfo, dstPath string) error + // Chtimes changes the access and modification times of the named file. + Chtimes(path string, atime time.Time, mtime time.Time) error // Atomically renames srcPath to dstPath overwriting the dstPath with srcPath's contents. OverwriteFile(srcPath, dstPath string) error Readlink(path string) (string, error) @@ -206,6 +220,26 @@ func (fs BeeGFS) CreatePreallocatedFile(path string, size int64, overwrite bool) return file.Close() } +func (fs BeeGFS) CreateOrResizeFile(path string, size int64, overwrite bool) error { + absPath := filepath.Join(fs.MountPoint, path) + flags := os.O_RDWR | os.O_CREATE + if !overwrite { + flags |= os.O_EXCL + } + + file, err := os.OpenFile(absPath, flags, 0666) + if err != nil { + return err + } + defer file.Close() + + if err := file.Truncate(size); err != nil { + return err + } + + return nil +} + func (fs BeeGFS) CreateWriteClose(path string, buf []byte, mode uint32, overwrite bool) error { var file *os.File var err error @@ -231,6 +265,10 @@ func (fs BeeGFS) Remove(path string) error { return os.Remove(filepath.Join(fs.MountPoint, path)) } +func (fs BeeGFS) RemoveAll(path string) error { + return os.RemoveAll(filepath.Join(fs.MountPoint, path)) +} + func (fs BeeGFS) Open(path string) (io.ReadCloser, error) { return os.Open(filepath.Join(fs.MountPoint, path)) } @@ -434,6 +472,11 @@ func (fs BeeGFS) CopyTimestamps(fromStat fs.FileInfo, dstPath string) error { return nil } +func (fs BeeGFS) Chtimes(path string, atime time.Time, mtime time.Time) error { + absPath := filepath.Join(fs.MountPoint, path) + return os.Chtimes(absPath, atime, mtime) +} + func (fs BeeGFS) OverwriteFile(srcPath, dstPath string) error { srcPath = filepath.Join(fs.MountPoint, srcPath) dstPath = filepath.Join(fs.MountPoint, dstPath) diff --git a/common/filesystem/fs_test.go b/common/filesystem/fs_test.go index 75c8f08e..dac9080b 100644 --- a/common/filesystem/fs_test.go +++ b/common/filesystem/fs_test.go @@ -2,6 +2,7 @@ package filesystem import ( "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -88,3 +89,20 @@ func TestBeeGFSWriteAndReadFileParts(t *testing.T) { require.NoError(t, err) assert.Equal(t, expectedFileLen, readBytes) } + +func TestBeeGFSRemoveAll(t *testing.T) { + testDir, cleanup, err := tempPathForTesting(baseTestDir) + require.NoError(t, err) + defer cleanup(t) + + mount := BeeGFS{MountPoint: testDir} + targetDir := filepath.Join(testDir, testFileName, "nested") + targetFile := filepath.Join(targetDir, "data") + require.NoError(t, os.MkdirAll(targetDir, 0755)) + require.NoError(t, os.WriteFile(targetFile, []byte("test"), 0644)) + + require.NoError(t, mount.RemoveAll(testFileName)) + + _, err = os.Stat(filepath.Join(testDir, testFileName)) + require.ErrorIs(t, err, os.ErrNotExist) +} diff --git a/common/filesystem/mock.go b/common/filesystem/mock.go index f0d41bc2..1cda8993 100644 --- a/common/filesystem/mock.go +++ b/common/filesystem/mock.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/spf13/afero" ) @@ -47,6 +48,16 @@ func (fs MockFS) CreatePreallocatedFile(path string, size int64, overwrite bool) return nil } +func (fs MockFS) CreateOrResizeFile(path string, size int64, overwrite bool) error { + file, err := fs.Fs.Create(path) + if err != nil { + return err + } + defer file.Close() + file.Truncate(size) + return nil +} + func (fs MockFS) CreateWriteClose(path string, buf []byte, mode uint32, overwrite bool) error { file, err := fs.Fs.Create(path) if err != nil { @@ -65,6 +76,10 @@ func (fs MockFS) Remove(path string) error { return fs.Fs.Remove(path) } +func (fs MockFS) RemoveAll(path string) error { + return fs.Fs.RemoveAll(path) +} + func (fs MockFS) Open(path string) (io.ReadCloser, error) { return fs.Fs.Open(path) } @@ -117,6 +132,10 @@ func (fs MockFS) CopyTimestamps(fromStat fs.FileInfo, dstPath string) error { return fmt.Errorf("not implemented") } +func (fs MockFS) Chtimes(path string, atime time.Time, mtime time.Time) error { + return fs.Fs.Chtimes(path, atime, mtime) +} + func (fs MockFS) OverwriteFile(srcPath, dstPath string) error { return fmt.Errorf("not implemented") } diff --git a/common/filesystem/unmounted.go b/common/filesystem/unmounted.go index ec0c4432..606baede 100644 --- a/common/filesystem/unmounted.go +++ b/common/filesystem/unmounted.go @@ -5,6 +5,7 @@ import ( "io/fs" "os" "path/filepath" + "time" ) // UnmountedFS can be used whenever a mounted file system may not actually be required. This allows @@ -40,6 +41,10 @@ func (fs UnmountedFS) CreatePreallocatedFile(path string, size int64, overwrite return ErrUnmounted } +func (fs UnmountedFS) CreateOrResizeFile(path string, size int64, overwrite bool) error { + return ErrUnmounted +} + func (fs UnmountedFS) CreateWriteClose(path string, buf []byte, mode uint32, overwrite bool) error { return ErrUnmounted } @@ -48,6 +53,10 @@ func (fs UnmountedFS) Remove(path string) error { return ErrUnmounted } +func (fs UnmountedFS) RemoveAll(path string) error { + return ErrUnmounted +} + func (fs UnmountedFS) Open(path string) (io.ReadCloser, error) { return nil, ErrUnmounted } @@ -85,6 +94,10 @@ func (fs UnmountedFS) CopyTimestamps(fromStat fs.FileInfo, dstPath string) error return ErrUnmounted } +func (fs UnmountedFS) Chtimes(path string, atime time.Time, mtime time.Time) error { + return ErrUnmounted +} + func (fs UnmountedFS) OverwriteFile(srcPath, dstPath string) error { return ErrUnmounted } diff --git a/common/filesystem/walk.go b/common/filesystem/walk.go index 5d8d2fb9..da63e168 100644 --- a/common/filesystem/walk.go +++ b/common/filesystem/walk.go @@ -108,24 +108,34 @@ type StreamPathResult struct { Err error } -// StreamPathsLexicographically returns a *StreamPathResult channel that returns the pattern's paths in a -// lexicographically increasing order. If startAfter != "" then only files lexically greater than -// will be considered. maxPaths limits the number of paths returned and can be set to -1 for all -// paths. chanSize is the buffer size for the returned *StreamPathResult channel. -func StreamPathsLexicographically(ctx context.Context, mountPoint Provider, pattern string, startAfter string, maxPaths int, chanSize int, filter FileInfoFilter) (<-chan *StreamPathResult, error) { - return streamPathsLexicographically(ctx, mountPoint, pattern, startAfter, maxPaths, chanSize, filter, false) +type PathStreamFunc func( + ctx context.Context, + mountPoint Provider, + pattern string, + startAfter string, + maxFiles int, + chanSize int, + filter FileInfoFilter, +) (<-chan *StreamPathResult, error) + +// StreamPathsLexicographically returns a *StreamPathResult channel that returns the pattern's paths +// in a lexicographically increasing order. If startAfter != "" then only files lexically greater +// than will be considered. maxFiles limits the number of files returned and can be set to -1 for +// all files. chanSize is the buffer size for the returned *StreamPathResult channel. +func StreamPathsLexicographically(ctx context.Context, mountPoint Provider, pattern string, startAfter string, maxFiles int, chanSize int, filter FileInfoFilter) (<-chan *StreamPathResult, error) { + return streamPathsLexicographically(ctx, mountPoint, pattern, startAfter, maxFiles, chanSize, filter, false) } -// StreamPathsLexicographicallyWithDirs behaves like StreamPathsLexicographically but also emits -// directories that match the filter (if provided). Directories are still traversed even if they -// don't match the filter. -func StreamPathsLexicographicallyWithDirs(ctx context.Context, mountPoint Provider, pattern string, startAfter string, maxPaths int, chanSize int, filter FileInfoFilter) (<-chan *StreamPathResult, error) { - return streamPathsLexicographically(ctx, mountPoint, pattern, startAfter, maxPaths, chanSize, filter, true) +// StreamPathsLexicographicallyWithDirs behaves like StreamPathsLexicographically but also emit +// directories. Emitted directories are not counted against maxFiles. Also, the ResumeToken returned +// will only be file paths. +func StreamPathsLexicographicallyWithDirs(ctx context.Context, mountPoint Provider, pattern string, startAfter string, maxFiles int, chanSize int, filter FileInfoFilter) (<-chan *StreamPathResult, error) { + return streamPathsLexicographically(ctx, mountPoint, pattern, startAfter, maxFiles, chanSize, filter, true) } -func streamPathsLexicographically(ctx context.Context, mountPoint Provider, pattern string, startAfter string, maxPaths int, chanSize int, filter FileInfoFilter, includeDirs bool) (<-chan *StreamPathResult, error) { - if maxPaths != -1 && maxPaths <= 0 { - return nil, fmt.Errorf("maxPaths must be greater than zero or -1") +func streamPathsLexicographically(ctx context.Context, mountPoint Provider, pattern string, startAfter string, maxFiles int, chanSize int, filter FileInfoFilter, includeDirs bool) (<-chan *StreamPathResult, error) { + if maxFiles != -1 && maxFiles <= 0 { + return nil, fmt.Errorf("maxFiles must be greater than zero or -1") } preparePath := func(path string) string { @@ -203,20 +213,31 @@ func streamPathsLexicographically(ctx context.Context, mountPoint Provider, patt return true } } - emitPath := func(path string, resumeToken string) bool { - if maxPaths == 0 { - send(&StreamPathResult{ResumeToken: resumeToken}) + + // Only files count against maxFiles or anchor the resume token: a directory never produces + // a job request on its own (see jobRequestBuilder.Process), and reapplying its RST config on + // a later pass is a no-op, so it's always sent immediately regardless of the remaining + // budget. + var lastSent string + emitFile := func(path string) bool { + if maxFiles == 0 { + send(&StreamPathResult{ResumeToken: lastSent}) return false } if !send(&StreamPathResult{Path: path}) { return false } - if maxPaths > 0 { - maxPaths-- + lastSent = path + if maxFiles > 0 { + maxFiles-- } return true } + emitDir := func(path string) bool { + return send(&StreamPathResult{Path: path}) + } + var walkDir func(string) bool walkDir = func(directory string) bool { if err := ctx.Err(); err != nil { @@ -233,29 +254,25 @@ func streamPathsLexicographically(ctx context.Context, mountPoint Provider, patt return false } - lastPath := directory for _, entry := range entries { path := filepath.Join(directory, entry.Name()) inMountPath := "/" + path if entry.IsDir() { if includeDirs { - emitDir := false + shouldEmitDir := false if !isGlob { - emitDir = path > startAfter + shouldEmitDir = path > startAfter } else if match := doublestar.MatchUnvalidated(pattern, path); match { - emitDir = path > startAfter + shouldEmitDir = path > startAfter } - if emitDir { + if shouldEmitDir { if keep, err := ApplyFilter(inMountPath, filter, mountPoint); err != nil { send(&StreamPathResult{Err: fmt.Errorf("unable to filter files: %w", err)}) return false - } else if keep { - if !emitPath(inMountPath, lastPath) { - return false - } - lastPath = path + } else if keep && !emitDir(inMountPath) { + return false } } } @@ -281,10 +298,9 @@ func streamPathsLexicographically(ctx context.Context, mountPoint Provider, patt continue } - if !emitPath(inMountPath, lastPath) { + if !emitFile(inMountPath) { return false } - lastPath = path } return true @@ -298,7 +314,7 @@ func streamPathsLexicographically(ctx context.Context, mountPoint Provider, patt send(&StreamPathResult{Err: fmt.Errorf("unable to filter files: %w", err)}) return } else if keep { - if !emitPath(inMountPath, root) { + if !emitDir(inMountPath) { return } } diff --git a/common/filesystem/walk_test.go b/common/filesystem/walk_test.go index 84ddc950..c76347db 100644 --- a/common/filesystem/walk_test.go +++ b/common/filesystem/walk_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "reflect" "slices" + "strings" "testing" "github.com/dgraph-io/badger/v4" @@ -435,6 +436,166 @@ func TestWalkSortedPathFileAndDirectory(t *testing.T) { } } +// TestWalkResumeAcrossNestedDirectory reproduces a bug where the resume token emitted when a walk +// is cut off can point earlier than the last path actually sent. This happens when the cutoff +// lands on a directory entry's sibling immediately after a nested subdirectory: the parent +// directory's resume token only advances when it emits a file directly, so it does not account for +// paths already sent from within the subdirectory. Resuming from that stale token causes the +// subdirectory to be walked again and its already-sent paths to be duplicated. +func TestWalkResumeAcrossNestedDirectory(t *testing.T) { + mountDir := t.TempDir() + paths := []string{ + "/data/a/a.txt", + "/data/a/z.txt", + "/data/b.txt", + "/data/b/b.txt", + "/data/b/b/b.txt", + "/data/b/y.txt", + "/data/b0.txt", + } + for _, path := range paths { + path := filepath.Join(mountDir, path) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(""), 0o644)) + } + provider := BeeGFS{MountPoint: mountDir} + ctx := context.Background() + + // Cut the walk off right after /data/b/b/b.txt, which is sent from within the nested "b/b" + // subdirectory rather than directly from "b". This is the boundary that triggers the stale + // resume token. + firstChan, err := StreamPathsLexicographically(ctx, provider, "/data", "", 5, 0, nil) + require.NoError(t, err) + + var firstPass []string + var resumeToken string + for resp := range firstChan { + require.NoError(t, resp.Err) + if resp.ResumeToken != "" { + resumeToken = resp.ResumeToken + continue + } + firstPass = append(firstPass, resp.Path) + } + require.Equal(t, []string{ + "/data/a/a.txt", + "/data/a/z.txt", + "/data/b.txt", + "/data/b/b.txt", + "/data/b/b/b.txt", + }, firstPass) + require.NotEmpty(t, resumeToken, "expected more work to remain after the cutoff") + + secondChan, err := StreamPathsLexicographically(ctx, provider, "/data", resumeToken, -1, 0, nil) + require.NoError(t, err) + + var secondPass []string + for resp := range secondChan { + require.NoError(t, resp.Err) + if resp.ResumeToken != "" { + continue + } + secondPass = append(secondPass, resp.Path) + } + + seen := make(map[string]bool, len(firstPass)) + for _, path := range firstPass { + seen[path] = true + } + for _, path := range secondPass { + assert.Falsef(t, seen[path], "path %q was sent in both the first and resumed pass", path) + } +} + +// TestWalkWithDirsDirectoriesDoNotAnchorResume guards against a bug specific to +// StreamPathsLexicographicallyWithDirs: a directory can be a prefix of a sibling file's name (e.g. +// directory "b" next to file "b.txt"), and under this package's directory-aware ordering (a period +// sorts before a slash) the file sorts before the directory. If the directory's own bare path were +// ever used as the resume anchor, plain string comparison of that sibling file's full path against +// it would disagree with the true walk order and the file could be resent. Directories never produce +// a job request on their own (see jobRequestBuilder.Process) and are idempotent to reprocess, so +// they're emitted for free without counting against maxFiles or ever becoming the resume anchor - +// this confirms the resume token always lands on a file even when a directory sits directly at the +// cutoff boundary. +func TestWalkWithDirsDirectoriesDoNotAnchorResume(t *testing.T) { + mountDir := t.TempDir() + paths := []string{ + "/data/a/a.txt", + "/data/a/z.txt", + "/data/b.txt", + "/data/b/b.txt", + "/data/b/b/b.txt", + "/data/b/y.txt", + "/data/b0.txt", + } + for _, path := range paths { + path := filepath.Join(mountDir, path) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(""), 0o644)) + } + provider := BeeGFS{MountPoint: mountDir} + ctx := context.Background() + + // Cut the walk off right after the file "/data/b.txt", which is immediately followed in + // directory-aware sort order by the "/data/b" directory itself - so the directory is emitted + // for free directly at the cutoff boundary. + firstChan, err := StreamPathsLexicographicallyWithDirs(ctx, provider, "/data", "", 3, 0, nil) + require.NoError(t, err) + + var firstPass []string + var resumeToken string + for resp := range firstChan { + require.NoError(t, resp.Err) + if resp.ResumeToken != "" { + resumeToken = resp.ResumeToken + continue + } + firstPass = append(firstPass, resp.Path) + } + require.Equal(t, []string{ + "/data", + "/data/a", + "/data/a/a.txt", + "/data/a/z.txt", + "/data/b.txt", + "/data/b", + }, firstPass) + require.Equal(t, "/data/b.txt", resumeToken, "the resume token should anchor on the last file sent, not the directory emitted alongside it") + + secondChan, err := StreamPathsLexicographicallyWithDirs(ctx, provider, "/data", resumeToken, -1, 0, nil) + require.NoError(t, err) + + var secondPass []string + for resp := range secondChan { + require.NoError(t, resp.Err) + if resp.ResumeToken != "" { + continue + } + secondPass = append(secondPass, resp.Path) + } + require.Equal(t, []string{ + "/data/b/b.txt", + "/data/b/b", + "/data/b/b/b.txt", + "/data/b/y.txt", + "/data/b0.txt", + }, secondPass) + + seenFiles := map[string]bool{} + for _, path := range firstPass { + if !strings.HasSuffix(path, ".txt") { + continue + } + seenFiles[path] = true + } + for _, path := range secondPass { + if !strings.HasSuffix(path, ".txt") { + continue + } + assert.Falsef(t, seenFiles[path], "file %q was sent in both the first and resumed pass", path) + } +} + func TestIsGlobPattern(t *testing.T) { tests := []struct { pattern string diff --git a/common/filesystem/walkmultiplexer.go b/common/filesystem/walkmultiplexer.go new file mode 100644 index 00000000..85006e29 --- /dev/null +++ b/common/filesystem/walkmultiplexer.go @@ -0,0 +1,100 @@ +package filesystem + +import ( + "context" + "sync" +) + +type WalkMultiplexer struct { + ctx context.Context + mergeCh chan *StreamPathResult + mergeChClosed bool + mu sync.Mutex + done *sync.Cond + activeInputs int + closed bool +} + +func NewWalkMultiplexer(ctx context.Context, bufferSize int) *WalkMultiplexer { + mergeCh := make(chan *StreamPathResult, max(1, bufferSize)) + multiplexer := &WalkMultiplexer{ctx: ctx, mergeCh: mergeCh} + multiplexer.done = sync.NewCond(&multiplexer.mu) + return multiplexer +} + +func (m *WalkMultiplexer) Output() <-chan *StreamPathResult { + return m.mergeCh +} + +func (m *WalkMultiplexer) AddWalks(walkChs []<-chan *StreamPathResult) func() { + if len(walkChs) == 0 { + return func() {} + } + + m.mu.Lock() + if m.closed { + m.mu.Unlock() + return func() {} + } + m.activeInputs += len(walkChs) + m.mu.Unlock() + + var wg sync.WaitGroup + for _, ch := range walkChs { + wg.Add(1) + go func(ch <-chan *StreamPathResult) { + defer wg.Done() + defer m.addWalksDone() + + for { + select { + case <-m.ctx.Done(): + return + case walkPath, ok := <-ch: + if !ok { + return + } + + select { + case <-m.ctx.Done(): + return + case m.mergeCh <- walkPath: + } + } + } + }(ch) + } + + return func() { + wg.Wait() + } +} + +func (m *WalkMultiplexer) addWalksDone() { + m.mu.Lock() + defer m.mu.Unlock() + + m.activeInputs-- + if m.closed && m.activeInputs == 0 && !m.mergeChClosed { + close(m.mergeCh) + m.mergeChClosed = true + m.done.Broadcast() + } +} + +func (m *WalkMultiplexer) Close() { + m.mu.Lock() + defer m.mu.Unlock() + + m.closed = true + + if m.activeInputs == 0 && !m.mergeChClosed { + close(m.mergeCh) + m.mergeChClosed = true + m.done.Broadcast() + } + + for !m.mergeChClosed { + m.done.Wait() + } +} diff --git a/common/filesystem/walkmultiplexer_test.go b/common/filesystem/walkmultiplexer_test.go new file mode 100644 index 00000000..8ef029cb --- /dev/null +++ b/common/filesystem/walkmultiplexer_test.go @@ -0,0 +1,157 @@ +package filesystem + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWalkMultiplexer_ForwardsFromMultipleInputs(t *testing.T) { + mux := NewWalkMultiplexer(context.Background(), 2) + inputA := make(chan *StreamPathResult, 1) + inputB := make(chan *StreamPathResult, 1) + waitForInputs := mux.AddWalks([]<-chan *StreamPathResult{inputA, inputB}) + + inputA <- &StreamPathResult{Path: "/path-a"} + inputB <- &StreamPathResult{Path: "/path-b"} + close(inputA) + close(inputB) + + waitForInputs() + mux.Close() + + received := map[string]bool{} + for result := range mux.Output() { + require.NotNil(t, result) + received[result.Path] = true + } + + assert.Equal(t, map[string]bool{ + "/path-a": true, + "/path-b": true, + }, received) +} + +func TestWalkMultiplexer_CloseWaitsForInputsToDrain(t *testing.T) { + mux := NewWalkMultiplexer(context.Background(), 1) + input := make(chan *StreamPathResult, 1) + waitForInput := mux.AddWalks([]<-chan *StreamPathResult{input}) + + closeDone := make(chan struct{}) + go func() { + mux.Close() + close(closeDone) + }() + + assertNotClosed(t, closeDone) + + input <- &StreamPathResult{Path: "/test-path"} + close(input) + waitForInput() + + select { + case result, ok := <-mux.Output(): + require.True(t, ok) + require.NotNil(t, result) + assert.Equal(t, "/test-path", result.Path) + case <-time.After(time.Second): + t.Fatal("timed out waiting for multiplexed walk result") + } + + select { + case <-closeDone: + case <-time.After(time.Second): + t.Fatal("timed out waiting for multiplexer close") + } + + _, ok := <-mux.Output() + assert.False(t, ok) +} + +func TestWalkMultiplexer_AddWalksAfterCloseIsNoop(t *testing.T) { + mux := NewWalkMultiplexer(context.Background(), 1) + mux.Close() + + input := make(chan *StreamPathResult, 1) + input <- &StreamPathResult{Path: "/ignored"} + close(input) + + waitForInput := mux.AddWalks([]<-chan *StreamPathResult{input}) + waitForInput() + + _, ok := <-mux.Output() + assert.False(t, ok) +} + +func TestWalkMultiplexer_ContextCancellationStopsInputs(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + mux := NewWalkMultiplexer(ctx, 1) + input := make(chan *StreamPathResult) + waitForInput := mux.AddWalks([]<-chan *StreamPathResult{input}) + + cancel() + + waitDone := make(chan struct{}) + go func() { + waitForInput() + close(waitDone) + }() + + select { + case <-waitDone: + case <-time.After(time.Second): + t.Fatal("timed out waiting for input goroutine after context cancellation") + } + + closeDone := make(chan struct{}) + go func() { + mux.Close() + close(closeDone) + }() + + select { + case <-closeDone: + case <-time.After(time.Second): + t.Fatal("timed out waiting for close after context cancellation") + } +} + +func TestWalkMultiplexer_MinimumOutputBufferSize(t *testing.T) { + mux := NewWalkMultiplexer(context.Background(), 0) + input := make(chan *StreamPathResult, 1) + waitForInput := mux.AddWalks([]<-chan *StreamPathResult{input}) + + input <- &StreamPathResult{Path: "/buffered"} + close(input) + + waitDone := make(chan struct{}) + go func() { + waitForInput() + close(waitDone) + }() + + select { + case <-waitDone: + case <-time.After(time.Second): + t.Fatal("timed out waiting for input with minimum output buffer") + } + + mux.Close() + result, ok := <-mux.Output() + require.True(t, ok) + require.NotNil(t, result) + assert.Equal(t, "/buffered", result.Path) +} + +func assertNotClosed(t *testing.T, ch <-chan struct{}) { + t.Helper() + + select { + case <-ch: + t.Fatal("channel closed before expected") + case <-time.After(10 * time.Millisecond): + } +} diff --git a/common/rst/builder.go b/common/rst/builder.go index d6f302f5..236ce0f9 100644 --- a/common/rst/builder.go +++ b/common/rst/builder.go @@ -4,9 +4,9 @@ import ( "context" "errors" "fmt" - "math" "path/filepath" "runtime" + "strings" "sync" "time" @@ -14,9 +14,16 @@ import ( "github.com/thinkparq/beegfs-go/ctl/pkg/ctl/entry" "github.com/thinkparq/protobuf/go/beeremote" "github.com/thinkparq/protobuf/go/flex" - "golang.org/x/sync/errgroup" ) +// TODO: Add to remote a global builder section that has a maxRequests. Also, allow per remote storage target overrides +// in case some targets next more or less or it doesn't matter. +// - 0 should be as many as possible. +// - Add something like only-count-active-submissions-against-max-requests flag to remote-storage-target? +// This would ensure that only jobs that were submitted with a non-terminal state would be counted against maxRequests. + +const maxRequests = 1000 + // JobBuilderClient is a special RST client that builders new job requests based on the information // provided via flex.JobRequestCfg. type JobBuilderClient struct { @@ -54,24 +61,104 @@ func (c *JobBuilderClient) GetJobRequest(cfg *flex.JobRequestCfg) *beeremote.Job } // GenerateWorkRequests for JobBuilderClient should simply pass a single -func (c *JobBuilderClient) GenerateWorkRequests(ctx context.Context, lastJob *beeremote.Job, job *beeremote.Job, availableWorkers int) (requests []*flex.WorkRequest, err error) { +func (c *JobBuilderClient) GenerateWorkRequests(ctx context.Context, lastJob *beeremote.Job, job *beeremote.Job, availableWorkers int) (workRequests []*flex.WorkRequest, err error) { if !job.Request.HasBuilder() { return nil, ErrReqAndRSTTypeMismatch } - workRequests := RecreateWorkRequests(job, nil) - return workRequests, nil + workRequests = RecreateWorkRequests(job, nil) + return } -func (c *JobBuilderClient) ExecuteJobBuilderRequest(ctx context.Context, workRequest *flex.WorkRequest, jobSubmissionChan chan<- *beeremote.JobRequest) (reschedule bool, err error) { +func (c *JobBuilderClient) ExecuteJobBuilderRequest(ctx context.Context, workRequest *flex.WorkRequest, jobSubmissionCh chan<- *beeremote.JobRequest, workerSaturation []func() float64) *SchedulingResult { if !workRequest.HasBuilder() { - err = ErrReqAndRSTTypeMismatch - return + return &SchedulingResult{Err: ErrReqAndRSTTypeMismatch} } + return c.executeBuilderRequest(ctx, workRequest, jobSubmissionCh, workerSaturation) +} + +func (c *JobBuilderClient) executeBuilderRequest(ctx context.Context, workRequest *flex.WorkRequest, jobSubmissionCh chan<- *beeremote.JobRequest, workerSaturation []func() float64) (result *SchedulingResult) { builder := workRequest.GetBuilder() cfg := builder.GetCfg() + + registry := c.newBulkOperationRegistry(ctx, workRequest.GetJobId(), &builder.BulkOperations) + defer func() { + if closeErr := registry.Close(ctx); closeErr != nil { + result.Err = appendError(result.Err, closeErr) + } + }() + + controller := c.newRequestBuildController(ctx, cfg, jobSubmissionCh, registry.AddRequest, workerSaturation) + abort := func(reason error) *SchedulingResult { + reason = fmt.Errorf("request was aborted: %w", reason) + managers := registry.GetManagersSnapshot() + if len(managers) == 0 { + return &SchedulingResult{Err: MarkBuilderCancelled(reason)} + } + + for _, manager := range managers { + controller.CancelBulkOperation(manager, reason) + } + err := controller.WaitForBulkOperations() + if registry.IsFailedManager() { + return &SchedulingResult{Err: MarkBuilderFailed(reason, err)} + } + return &SchedulingResult{Err: MarkBuilderCancelled(reason, err)} + } + resumeToken := workRequest.GetExternalId() + walkComplete, walkErr := parseResumeToken(resumeToken, workRequest.JobId) + if !walkComplete { + walkSize := min(cap(jobSubmissionCh), maxRequests+1) // +1 is for ResumeToken when there is more work + walkChGenerator, resumeToken, err := c.getNextWalkChGenerator(ctx, workRequest, walkSize) + if err != nil { + if len(registry.GetManagersSnapshot()) == 0 { + return abort(err) + } + walkErr = err + } else { + controller.WalkSourceGenerator(walkChGenerator, resumeToken, maxRequests) + if err = controller.WaitForWalkSources(); err != nil { + if errors.Is(ctx.Err(), context.Canceled) || len(registry.GetManagersSnapshot()) == 0 { + return abort(err) + } + walkErr = err + } + } + } + + for _, manager := range registry.GetManagersSnapshot() { + controller.ExecuteBulkOperation(manager) + } + if err := controller.WaitForBulkOperations(); err != nil { + return abort(err) + } + + result, resumeToken = controller.GetResults() + if resumeToken == "" || walkErr != nil { + resumeToken = buildWalkCompleteSentinel(workRequest.JobId, walkErr) + } + workRequest.SetExternalId(resumeToken) + if result.Reschedule { + return + } + + if registry.IsFailedManager() { + result.Err = MarkBuilderFailed(result.Err) + } else if walkErr != nil { + result.Err = MarkBuilderCancelled(result.Err, walkErr) + } + return +} + +type nextWalkChGenerator func(resumeToken string) (walkCh <-chan *filesystem.StreamPathResult, err error) + +func (c *JobBuilderClient) getNextWalkChGenerator(ctx context.Context, workRequest *flex.WorkRequest, chanSize int) (generator nextWalkChGenerator, resumeToken string, err error) { + maxFiles := maxRequests + builder := workRequest.GetBuilder() + cfg := builder.GetCfg() + resumeToken = workRequest.GetExternalId() var filter filesystem.FileInfoFilter filterExpr := cfg.GetFilterExpr() @@ -82,31 +169,23 @@ func (c *JobBuilderClient) ExecuteJobBuilderRequest(ctx context.Context, workReq } } - // TODO: maxRequests limits the number of requests that can be created at a time before the - // builder job is rescheduled. This should probably be based on the client if possible; - // otherwise, client based metric that are based on builder short/long-term data collection. - // Each client should at least have some input since there may be costs associated with the - // requests as in s3. - maxRequests := 1000 - - walkChanSize := cap(jobSubmissionChan) - var walkChan <-chan *filesystem.StreamPathResult walkPaths := filesystem.StreamPathsLexicographically if cfg.GetUpdate() || cfg.HasCooldownSecs() { walkPaths = filesystem.StreamPathsLexicographicallyWithDirs } - if cfg.Download { + if cfg.GetDownload() { if filter != nil { - return false, fmt.Errorf("filter expressions (--%s) are not supported for downloads yet", filesystem.FilterExprFlag) + err = fmt.Errorf("filter expressions (--%s) are not supported for downloads yet", filesystem.FilterExprFlag) + return } - if walkLocalPathInsteadOfRemote(cfg) { + if WalkLocalPathInsteadOfRemote(cfg) { // Since neither cfg.RemoteStorageTarget nor a remote path is specified, walk the local // path. Create a job for each file that has exactly one rstId or is a stub file. Ignore // files with no rstIds and fail files with multiple rstIds due to ambiguity. - if walkChan, err = walkPaths(ctx, c.mountPoint, workRequest.Path, resumeToken, maxRequests, walkChanSize, nil); err != nil { - return + generator = func(token string) (walkCh <-chan *filesystem.StreamPathResult, err error) { + return walkPaths(ctx, c.mountPoint, workRequest.GetPath(), token, maxFiles, chanSize, nil) } } else { client, ok := c.rstMap[cfg.RemoteStorageTarget] @@ -115,31 +194,67 @@ func (c *JobBuilderClient) ExecuteJobBuilderRequest(ctx context.Context, workReq return } - if walkChan, err = client.GetWalk(ctx, client.SanitizeRemotePath(cfg.RemotePath), walkChanSize, resumeToken, maxRequests); err != nil { - return + generator = func(token string) (walkCh <-chan *filesystem.StreamPathResult, err error) { + return client.GetWalk(ctx, client.SanitizeRemotePath(cfg.GetRemotePath()), chanSize, token, maxFiles) } } } else { - walkChan, err = walkPaths(ctx, c.mountPoint, workRequest.Path, resumeToken, maxRequests, walkChanSize, filter) - if err != nil { - return + generator = func(token string) (walkCh <-chan *filesystem.StreamPathResult, err error) { + return walkPaths(ctx, c.mountPoint, workRequest.Path, token, maxFiles, chanSize, filter) } } - return c.executeJobBuilderRequest(ctx, workRequest, walkChan, jobSubmissionChan, cfg) -} - -func (r *JobBuilderClient) IsWorkRequestReady(ctx context.Context, request *flex.WorkRequest) (bool, time.Duration, error) { - return true, 0, nil + return } // ExecuteWorkRequestPart is not implemented and should never be called. -func (c *JobBuilderClient) ExecuteWorkRequestPart(ctx context.Context, request *flex.WorkRequest, part *flex.Work_Part) error { +func (c *JobBuilderClient) ExecuteWorkRequestPart(ctx context.Context, workRequest *flex.WorkRequest, part *flex.Work_Part) error { return ErrUnsupportedOpForRST } -func (c *JobBuilderClient) CompleteWorkRequests(ctx context.Context, job *beeremote.Job, workResults []*flex.Work, abort bool) error { - return nil +func (c *JobBuilderClient) CompleteWorkRequests(ctx context.Context, job *beeremote.Job, workResults []*flex.Work, abort bool) (err error) { + if abort { + bulkOperations := getBulkOperations(workResults) + if len(bulkOperations) > 0 { + registry := c.newBulkOperationRegistry(ctx, job.GetId(), &bulkOperations) + reason := fmt.Errorf("builder job %q was aborted", job.GetId()) + + cancelWaits := map[*bulkOperationManager]BulkCancelResultFn{} + for _, manager := range registry.GetManagersSnapshot() { + walkCh, wait, cancelErr := manager.Cancel(ctx, reason) + if cancelErr != nil { + err = appendError(err, cancelErr) + continue + } + + cancelWaits[manager] = wait + go func() { + for range walkCh { + } + }() + } + + for manager, cancelWait := range cancelWaits { + if cancelWaitErr := cancelWait(); cancelWaitErr != nil { + err = appendError(err, cancelWaitErr) + } else { + err = appendError(err, manager.Destroy(ctx)) + } + } + } + } + + return +} + +func getBulkOperations(workResults []*flex.Work) []*flex.BulkOperation { + jobBuilderOperations := []*flex.BulkOperation{} + for _, workResult := range workResults { + if workResult.HasJobBuilderInfo() { + jobBuilderOperations = append(jobBuilderOperations, workResult.JobBuilderInfo.BulkOperations...) + } + } + return jobBuilderOperations } // GetConfig is not implemented and should never be called. @@ -167,205 +282,148 @@ func (c *JobBuilderClient) GenerateExternalId(ctx context.Context, cfg *flex.Job return "", ErrUnsupportedOpForRST } -func (c *JobBuilderClient) executeJobBuilderRequest( - ctx context.Context, - request *flex.WorkRequest, - walkChan <-chan *filesystem.StreamPathResult, - jobSubmissionChan chan<- *beeremote.JobRequest, - cfg *flex.JobRequestCfg, -) (bool, error) { - builder := request.GetBuilder() - - var walkingLocalPath bool - var remotePathDir string - var remotePathIsGlob bool - var isPathDir bool - if cfg.Download { - walkingLocalPath = walkLocalPathInsteadOfRemote(cfg) - remotePathDir, remotePathIsGlob = GetDownloadRemotePathDirectory(cfg.RemotePath) - stat, err := c.mountPoint.Lstat(cfg.Path) - isPathDir = err == nil && stat.IsDir() +func (c *JobBuilderClient) IsWorkRequestReady(ctx context.Context, workRequest *flex.WorkRequest) (ready bool, delay time.Duration, err error) { + return true, 0, nil +} + +func (c *JobBuilderClient) IncludeRequestInBulkOperation(ctx context.Context, request *beeremote.JobRequest) (include bool, operation string) { + return false, "" +} + +func (c *JobBuilderClient) OpenBulkOperation(ctx context.Context, stateMountPath string, operation string) (clientBulkOperation, error) { + return nil, ErrUnsupportedOpForRST +} + +func (c *JobBuilderClient) newBulkOperationRegistry(ctx context.Context, builderJobId string, builderBulkOperations *[]*flex.BulkOperation) *bulkOperationRegistry { + manager := &bulkOperationRegistry{ + managers: make(map[string]*bulkOperationManager), + managersMu: sync.Mutex{}, + rstMap: c.rstMap, + builderBulkOperations: builderBulkOperations, + builderJobId: builderJobId, } - reschedule := false - builderStateMu := sync.Mutex{} - maxWorkers := runtime.GOMAXPROCS(0) - walkDoneChan := make(chan struct{}, maxWorkers) - defer close(walkDoneChan) - createJobRequests := func() error { - var err error - var inMountPath string - var remotePath string - for { - select { - case <-ctx.Done(): - return ctx.Err() - case walkResp, ok := <-walkChan: - if !ok { - select { - case walkDoneChan <- struct{}{}: - default: - } - return nil - } + for _, bulkOperation := range *builderBulkOperations { + key := bulkOperationKey(bulkOperation.RstId, bulkOperation.Operation) + client, _ := manager.rstMap[bulkOperation.RstId] + manager.managers[key] = newBulkOperationManager(ctx, client, builderJobId, bulkOperation) + } + return manager +} - if walkResp.Err != nil { - return walkResp.Err - } +const ( + // requestBuildControllerWorkerMultiplier scales GOMAXPROCS to set the maximum number of + // concurrent path-processing goroutines. Each path always blocks on at least one BeeGFS + // metadata operation (lock acquisition via getPathState), making per-path goroutines the right + // model: goroutines are parked during the blocking I/O, freeing OS threads for other work. The + // multiplier must be large enough that enough goroutines are in flight to keep hardware threads + // busy, but small enough to avoid excessive concurrent pressure on the metadata server. + requestBuildControllerWorkerMultiplier = 8.0 + // requestBuildControllerQueueDepthPerWorker controls the job submission backpressure threshold: + // threshold = min(cap(jobSubmissionCh), maxWorkers*queueDepthPerWorker). Once the submission + // queue reaches the threshold, processWalk stops spawning new path goroutines until it drains. + // Higher values allow more in-flight submissions before throttling, which smooths throughput + // but buffers more work in memory. Lower values throttle more tightly and respond faster to a + // slow downstream consumer. + requestBuildControllerQueueDepthPerWorker = 2.0 +) - if walkResp.ResumeToken != "" { - builderStateMu.Lock() - reschedule = true - request.SetExternalId(walkResp.ResumeToken) - builderStateMu.Unlock() - return nil - } +func (c *JobBuilderClient) newRequestBuildController( + ctx context.Context, + builderCfg *flex.JobRequestCfg, + jobSubmissionCh chan<- *beeremote.JobRequest, + addBulkRequest addBulkRequestFn, + workerSaturation []func() float64, +) *requestBuildController { + cpuLimit := max(1, int(requestBuildControllerWorkerMultiplier*float32(runtime.GOMAXPROCS(0)))) + queueLimit := max(1, cap(jobSubmissionCh)) + maxWorkers := min(cpuLimit, queueLimit) + submissionBackpressureThreshold := max(1, min(cap(jobSubmissionCh), int(requestBuildControllerQueueDepthPerWorker*float32(maxWorkers)))) + requestBuilder := c.newJobRequestBuilder(builderCfg, jobSubmissionCh, addBulkRequest) + return &requestBuildController{ + ctx: ctx, + requestBuilder: requestBuilder, + backpressureThreshold: submissionBackpressureThreshold, + getPaths: c.getPathsFn(builderCfg), + maxWorkersCh: make(chan struct{}, maxWorkers), + workerSaturation: workerSaturation, + } +} - if cfg.Download { - if walkingLocalPath { - // Walking cfg.Path to support stub file download and files with a defined rst. - inMountPath = walkResp.Path - } else { - remotePath = walkResp.Path - inMountPath, err = GetDownloadInMountPath(cfg.Path, remotePath, remotePathDir, remotePathIsGlob, isPathDir, cfg.Flatten) - if err != nil { - // This should never happen since both remotePath and remotePathDir - // come directly from cfg.RemotePath, so any error here indicates a - // bug in the walking logic. - return err - } - - // Ensure the local directory structure supports the object downloads - if err := c.mountPoint.CreateDir(filepath.Dir(inMountPath), 0755); err != nil { - return err - } - } - } else { - inMountPath = walkResp.Path - remotePath = inMountPath - } - } +func (c *JobBuilderClient) newJobRequestBuilder( + builderCfg *flex.JobRequestCfg, + jobSubmissionCh chan<- *beeremote.JobRequest, + addBulkRequest addBulkRequestFn, +) *jobRequestBuilder { + requestBuilder := &jobRequestBuilder{ + mountPoint: c.mountPoint, + RstMap: c.rstMap, + jobSubmissionCh: jobSubmissionCh, + builderCfg: builderCfg, + getPathState: GetPathState, + planFileState: PlanFileStateForWorkRequests, + clearAccessFlags: entry.ClearAccessFlags, + addBulkRequest: addBulkRequest, + } + requestBuilder.init() - if cfg.GetUpdate() || cfg.HasCooldownSecs() { - if stat, statErr := c.mountPoint.Lstat(inMountPath); statErr == nil && stat.IsDir() { - var rstIds []uint32 - if cfg.GetUpdate() && IsValidRstId(cfg.RemoteStorageTarget) { - rstIds = []uint32{cfg.RemoteStorageTarget} - } - var cooldownSecs *uint16 - if cfg.HasCooldownSecs() { - v := uint16(math.MaxUint16) - if cfg.GetCooldownSecs() <= math.MaxUint16 { - v = uint16(cfg.GetCooldownSecs()) - } - cooldownSecs = &v - } - dirErr := entry.SetDirRstPattern(ctx, inMountPath, rstIds, cooldownSecs) - builderStateMu.Lock() - builder.Submitted++ - if dirErr != nil { - builder.Errors++ - } - builderStateMu.Unlock() - continue - } - } + return requestBuilder +} - jobRequests, err := BuildJobRequests(ctx, c.rstMap, c.mountPoint, inMountPath, remotePath, cfg) - if err != nil { - // BuildJobRequest should only return fatal errors, or if there are no RSTs - // specified/configured on an entry and there is no other way to return the - // error other then aborting the builder job entirely. - return err +func (c *JobBuilderClient) getPathsFn(cfg *flex.JobRequestCfg) requestPathResolverFn { + if cfg.Download { + if WalkLocalPathInsteadOfRemote(cfg) { + // Walking cfg.Path to support stub file download and files with a defined rst. + return func(walkPath string) (string, string, error) { + return walkPath, "", nil } + } - errorCount := 0 - for _, jobRequest := range jobRequests { - status := jobRequest.GetGenerationStatus() - if status != nil && (status.State == beeremote.JobRequest_GenerationStatus_ERROR || status.State == beeremote.JobRequest_GenerationStatus_FAILED_PRECONDITION) { - errorCount++ - } - select { - case <-ctx.Done(): - case jobSubmissionChan <- jobRequest: - } + return func(walkPath string) (string, string, error) { + // GetDownloadInMountPath should never return an error happen since remotePath and + // remotePathDir are derived from cfg.RemotePath, so any error here indicates a bug + // in the walking logic. + remotePathDir, remotePathIsGlob := GetDownloadRemotePathDirectory(cfg.RemotePath) + stat, err := c.mountPoint.Lstat(cfg.Path) + isPathDir := err == nil && stat.IsDir() + + remotePath := walkPath + inMountPath, err := GetDownloadInMountPath(cfg.Path, remotePath, remotePathDir, remotePathIsGlob, isPathDir, cfg.Flatten) + if err == nil { + // Ensure the local directory structure supports the object downloads + err = c.mountPoint.CreateDir(filepath.Dir(inMountPath), 0755) } - - builderStateMu.Lock() - builder.Submitted += int32(len(jobRequests)) - builder.Errors += int32(errorCount) - builderStateMu.Unlock() + return inMountPath, remotePath, err } } - // Start worker(s) that process walk paths and enqueue job requests. Begin with one and add more - // (up to GOMAXPROCS) when the job submission channel stays near empty, indicating the consumer is - // draining faster than we can fill it. This keeps throughput balanced without over saturating - // the system. - g, ctx := errgroup.WithContext(ctx) - g.Go(func() error { - workers := 1 - lowThresholdTicks := 0 - g.Go(createJobRequests) - for { - select { - case <-ctx.Done(): - return nil - case <-walkDoneChan: - return nil - case <-time.After(100 * time.Millisecond): - size := len(jobSubmissionChan) - if workers < maxWorkers && size <= 2*workers { - if size <= workers { - lowThresholdTicks += 3 - } else { - lowThresholdTicks++ - } + return func(walkPath string) (string, string, error) { + return walkPath, walkPath, nil + } +} - if lowThresholdTicks >= 3 { - g.Go(createJobRequests) - workers++ - lowThresholdTicks = 0 - } - } else { - lowThresholdTicks = 0 - } - } - } - }) - if err := g.Wait(); err != nil { - return false, fmt.Errorf("job builder request was aborted: %w", err) +func buildWalkCompleteSentinel(jobId string, err error) string { + if err != nil { + return fmt.Sprintf("sentinel:%s:%s", jobId, err.Error()) } - if reschedule { - return true, nil + return fmt.Sprintf("sentinel:%s:", jobId) +} + +func parseResumeToken(token string, jobId string) (walkComplete bool, sentinelErr error) { + if token == "" { + return } var errMessage string - totalSubmitted := builder.GetSubmitted() - totalErrors := builder.GetErrors() - if totalSubmitted == 0 { - if cfg.Download { - if walkingLocalPath { - errMessage = fmt.Sprintf("walking local path since --%s was not provided; No matches found in path: %s", RemotePathFlag, cfg.Path) - } else { - errMessage = fmt.Sprintf("no matches found in remote path: %s", cfg.RemotePath) - } - } else { - errMessage = fmt.Sprintf("no matches found in local path: %s", cfg.Path) - } - } else if totalErrors > 0 { - errMessage = fmt.Sprintf("%d of %d requests were submitted with errors", totalErrors, totalSubmitted) + if errMessage, walkComplete = strings.CutPrefix(token, fmt.Sprintf("sentinel:%s:", jobId)); !walkComplete { + return } - if errMessage != "" { - if !IsValidRstId(cfg.RemoteStorageTarget) { - errMessage += fmt.Sprintf("; --%s was not provided so relying on configured rstIds and stub urls", RemoteTargetFlag) - } - return false, errors.New(errMessage) + sentinelErr = errors.New(errMessage) } - return false, nil + return } -func walkLocalPathInsteadOfRemote(cfg *flex.JobRequestCfg) bool { +func WalkLocalPathInsteadOfRemote(cfg *flex.JobRequestCfg) bool { return cfg.RemotePath == "" } diff --git a/common/rst/builder_test.go b/common/rst/builder_test.go new file mode 100644 index 00000000..9fcb35ce --- /dev/null +++ b/common/rst/builder_test.go @@ -0,0 +1,119 @@ +package rst + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/thinkparq/beegfs-go/common/filesystem" + "github.com/thinkparq/protobuf/go/beeremote" + "github.com/thinkparq/protobuf/go/flex" +) + +// trackingBulkOperation is a clientBulkOperation stand-in that records whether Cancel was invoked +// and waited on, so tests can assert CompleteWorkRequests' abort path drives cancellation through +// to completion. +type trackingBulkOperation struct { + cancelCalled bool + cancelReason error + waitCalled bool + destroyCalled bool +} + +func (t *trackingBulkOperation) AddRequest(ctx context.Context, request *beeremote.JobRequest) error { + return nil +} + +func (t *trackingBulkOperation) Execute(ctx context.Context) (<-chan *BulkStreamPathResult, BulkExecuteResultFn, error) { + walkCh := make(chan *BulkStreamPathResult) + close(walkCh) + return walkCh, func() *SchedulingResult { return &SchedulingResult{} }, nil +} + +func (t *trackingBulkOperation) Cancel(ctx context.Context, reason error) (<-chan *BulkStreamPathResult, BulkCancelResultFn, error) { + walkCh := make(chan *BulkStreamPathResult) + return walkCh, func() error { + t.cancelCalled = true + t.cancelReason = reason + t.waitCalled = true + close(walkCh) + return nil + }, nil +} + +func (t *trackingBulkOperation) Close(ctx context.Context) error { + return nil +} + +func (t *trackingBulkOperation) Destroy(ctx context.Context) error { + t.destroyCalled = true + return nil +} + +func TestGetBulkOperationsReturnsAllStartedBulkOperations(t *testing.T) { + expected := []*flex.BulkOperation{ + flex.BulkOperation_builder{ + StateMountPath: ".beegfs-rst/job/job-1/1", + RstId: 1, + Operation: "retrieve", + }.Build(), + flex.BulkOperation_builder{ + StateMountPath: ".beegfs-rst/job/job-1/2", + RstId: 2, + Operation: "archive", + Errors: func() *string { s := "resume failed"; return &s }(), + }.Build(), + } + + workResults := []*flex.Work{ + flex.Work_builder{ + JobBuilderInfo: flex.Work_JobBuilderInfo_builder{ + BulkOperations: expected, + }.Build(), + }.Build(), + flex.Work_builder{}.Build(), + } + + assert.Equal(t, expected, getBulkOperations(workResults)) +} + +func TestCompleteWorkRequestsAbortCancelsAllStartedBulkOperations(t *testing.T) { + mockRST := &MockClient{} + client := NewJobBuilderClient(context.Background(), map[uint32]Provider{1: mockRST}, filesystem.NewMockFS()) + + job := beeremote.Job_builder{ + Id: "builder-job", + Request: beeremote.JobRequest_builder{ + Path: "/test/builder", + RemoteStorageTarget: JobBuilderRstId, + Builder: flex.BuilderJob_builder{}.Build(), + }.Build(), + }.Build() + + tracker := &trackingBulkOperation{} + mockRST.On("OpenBulkOperation", mock.Anything, ".beegfs-rst/job/builder-job/1", "retrieve").Return(tracker, nil).Once() + + workResults := []*flex.Work{ + flex.Work_builder{ + JobBuilderInfo: flex.Work_JobBuilderInfo_builder{ + BulkOperations: []*flex.BulkOperation{ + flex.BulkOperation_builder{ + StateMountPath: ".beegfs-rst/job/builder-job/1", + RstId: 1, + Operation: "retrieve", + }.Build(), + }, + }.Build(), + }.Build(), + } + + err := client.CompleteWorkRequests(context.Background(), job, workResults, true) + require.NoError(t, err) + require.True(t, tracker.cancelCalled) + require.True(t, tracker.waitCalled) + require.ErrorContains(t, tracker.cancelReason, `builder job "builder-job" was aborted`) + require.True(t, tracker.destroyCalled) + mockRST.AssertExpectations(t) +} diff --git a/common/rst/builderbulk.go b/common/rst/builderbulk.go new file mode 100644 index 00000000..38757814 --- /dev/null +++ b/common/rst/builderbulk.go @@ -0,0 +1,230 @@ +package rst + +import ( + "context" + "errors" + "fmt" + "maps" + "path" + "sync" + + "github.com/thinkparq/protobuf/go/beeremote" + "github.com/thinkparq/protobuf/go/flex" +) + +type BulkStreamPathResult struct { + BulkInfo *flex.BulkJobRequestInfo + RstId uint32 + Path string + Err error +} + +type bulkOperationRegistry struct { + managers map[string]*bulkOperationManager + managersMu sync.Mutex + rstMap map[uint32]Provider + builderBulkOperations *[]*flex.BulkOperation + builderJobId string +} + +func (m *bulkOperationRegistry) GetManagersSnapshot() map[string]*bulkOperationManager { + m.managersMu.Lock() + defer m.managersMu.Unlock() + + snapshot := make(map[string]*bulkOperationManager, len(m.managers)) + maps.Copy(snapshot, m.managers) + return snapshot +} + +func (m *bulkOperationRegistry) IsFailedManager() bool { + m.managersMu.Lock() + defer m.managersMu.Unlock() + + for _, manager := range m.managers { + if manager.IsFailed() { + return true + } + } + return false +} + +func (m *bulkOperationRegistry) AddRequest(ctx context.Context, request *beeremote.JobRequest) (skipSubmit bool, err error) { + if request.GetGenerationStatus() != nil { + return + } + + rstId := request.GetRemoteStorageTarget() + client := m.rstMap[rstId] + include, operation := client.IncludeRequestInBulkOperation(ctx, request) + if include { + m.managersMu.Lock() + defer m.managersMu.Unlock() + + manager, ok := m.managers[bulkOperationKey(rstId, operation)] + if !ok { + if _, manager, err = m.addManagerUnlocked(ctx, client, rstId, operation); err != nil { + return + } + } + + if manager.IsFailed() { + request.SetGenerationStatus(&beeremote.JobRequest_GenerationStatus{ + State: beeremote.JobRequest_GenerationStatus_FAILED_PRECONDITION, + Message: fmt.Sprintf("remote storage target %d's %q bulk operation previously failed and will not be retried: %s", rstId, operation, manager.GetErrors()), + }) + return + } + + if err = manager.AddRequest(ctx, request); err != nil { + return + } + skipSubmit = true + } + return +} + +func (m *bulkOperationRegistry) addManagerUnlocked(ctx context.Context, client Provider, rstId uint32, operation string) (key string, manager *bulkOperationManager, err error) { + key = bulkOperationKey(rstId, operation) + bulkOperation := &flex.BulkOperation{RstId: rstId, Operation: operation} + manager = newBulkOperationManager(ctx, client, m.builderJobId, bulkOperation) + + *m.builderBulkOperations = append(*m.builderBulkOperations, bulkOperation) + m.managers[key] = manager + return +} + +// Close closes all bulk operation managers and returns any errors encountered. +func (m *bulkOperationRegistry) Close(ctx context.Context) (err error) { + for managerKey, manager := range m.GetManagersSnapshot() { + if closeErr := manager.Close(ctx); closeErr != nil { + err = errors.Join(err, fmt.Errorf("failed to close bulk operation manager, %s: %w", managerKey, closeErr)) + } + } + return +} + +const ( + stateRoot = ".beegfs-rst" + bulkManagerPath = "job" +) + +type bulkOperationManager struct { + clientBulkOperation + StateMountPath string + rstId uint32 + operation string + mu sync.Mutex + errors *string + failed *bool +} + +func newBulkOperationManager(ctx context.Context, client Provider, jobId string, bulkOperation *flex.BulkOperation) *bulkOperationManager { + stateMountPath := path.Join(stateRoot, bulkManagerPath, jobId, fmt.Sprint(bulkOperation.RstId)) + if bulkOperation.Errors == nil { + bulkOperation.Errors = new(string) + } + + manager := &bulkOperationManager{ + StateMountPath: stateMountPath, + rstId: bulkOperation.RstId, + operation: bulkOperation.Operation, + errors: bulkOperation.Errors, + failed: &bulkOperation.Failed, + } + + if !bulkOperation.Failed { + if client == nil { + err := fmt.Errorf("unable to create bulk operation manager: remote storage target ID %d does not exist in the configuration", bulkOperation.RstId) + manager.AppendError(err) + manager.SetFailed() + } else if clientBulkOperation, err := client.OpenBulkOperation(ctx, stateMountPath, bulkOperation.Operation); err != nil { + manager.AppendError(err) + manager.SetFailed() + } else { + manager.clientBulkOperation = clientBulkOperation + } + } + + return manager +} + +func (m *bulkOperationManager) IsFailed() bool { + return *m.failed +} + +func (m *bulkOperationManager) SetFailed() { + *m.failed = true +} + +// AddRequest attaches the bulk operation's StateMountPath and Operation to the request. JobIndex is +// intentionally left unset here; the provider-specific clientBulkOperation assigns it based on its +// own persisted state (e.g. the count of requests already recorded on disk) since it's the one that +// must be able to reconstruct a correct index after a builder reschedule reopens the operation. +func (m *bulkOperationManager) AddRequest(ctx context.Context, request *beeremote.JobRequest) error { + if m.clientBulkOperation == nil { + return fmt.Errorf("cannot add request to bulk operation %s: it previously failed permanently and has no provider handle", m.Key()) + } + + m.mu.Lock() + defer m.mu.Unlock() + + request.SetBulkInfo(&flex.BulkJobRequestInfo{ + StateMountPath: m.StateMountPath, + Operation: m.operation, + }) + + return m.clientBulkOperation.AddRequest(ctx, request) +} + +func (m *bulkOperationManager) Key() string { + return bulkOperationKey(m.rstId, m.operation) +} + +func (m *bulkOperationManager) Execute(ctx context.Context) (walkCh <-chan *BulkStreamPathResult, getResults BulkExecuteResultFn, err error) { + if m.clientBulkOperation == nil { + err = fmt.Errorf("cannot execute bulk operation %s: it previously failed permanently and has no provider handle", m.Key()) + return + } + return m.clientBulkOperation.Execute(ctx) +} + +func (m *bulkOperationManager) Cancel(ctx context.Context, reason error) (walkCh <-chan *BulkStreamPathResult, wait BulkCancelResultFn, err error) { + if m.clientBulkOperation == nil { + err = fmt.Errorf("cannot cancel bulk operation %s: it previously failed permanently and has no provider handle", m.Key()) + return + } + return m.clientBulkOperation.Cancel(ctx, reason) +} + +func (m *bulkOperationManager) Close(ctx context.Context) error { + if m.clientBulkOperation == nil { + return nil + } + return m.clientBulkOperation.Close(ctx) +} + +func (m *bulkOperationManager) AppendError(err error) { + if err == nil { + return + } + + m.mu.Lock() + defer m.mu.Unlock() + if *m.errors == "" { + *m.errors = err.Error() + } else { + *m.errors = fmt.Sprintf("%s. %s", *m.errors, err.Error()) + } +} + +func (m *bulkOperationManager) GetErrors() error { + if *m.errors == "" { + return nil + } + + return fmt.Errorf("bulk operation %s: (%s)", m.operation, *m.errors) +} + +func bulkOperationKey(rstId uint32, operation string) string { + return fmt.Sprintf("%d-%s", rstId, operation) +} diff --git a/common/rst/builderbulk_test.go b/common/rst/builderbulk_test.go new file mode 100644 index 00000000..eda77f39 --- /dev/null +++ b/common/rst/builderbulk_test.go @@ -0,0 +1,150 @@ +package rst + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/thinkparq/protobuf/go/beeremote" + "github.com/thinkparq/protobuf/go/flex" +) + +// fakeBulkOperation is a minimal clientBulkOperation stand-in for exercising bulkOperationRegistry +// and bulkOperationManager in isolation, without going through a real provider. +type fakeBulkOperation struct { + addRequestErr error + closeErr error +} + +func (f *fakeBulkOperation) AddRequest(ctx context.Context, request *beeremote.JobRequest) error { + return f.addRequestErr +} + +func (f *fakeBulkOperation) Execute(ctx context.Context) (<-chan *BulkStreamPathResult, BulkExecuteResultFn, error) { + return nil, nil, fmt.Errorf("fakeBulkOperation.Execute not implemented") +} + +func (f *fakeBulkOperation) Cancel(ctx context.Context, reason error) (<-chan *BulkStreamPathResult, BulkCancelResultFn, error) { + return nil, nil, fmt.Errorf("fakeBulkOperation.Cancel not implemented") +} + +func (f *fakeBulkOperation) Close(ctx context.Context) error { + return f.closeErr +} + +func (m *fakeBulkOperation) Destroy(ctx context.Context) error { + return nil +} + +// newTestBulkOperationRegistry builds a bulkOperationRegistry backed by a single rstId (1) mapped to +// client, mirroring what JobBuilderClient.newBulkOperationRegistry produces but without requiring a +// pre-populated builderBulkOperations slice. +func newTestBulkOperationRegistry(client Provider) *bulkOperationRegistry { + bulkOperations := []*flex.BulkOperation{} + return &bulkOperationRegistry{ + managers: make(map[string]*bulkOperationManager), + rstMap: map[uint32]Provider{1: client}, + builderBulkOperations: &bulkOperations, + builderJobId: "job-1", + } +} + +func TestBulkOperationRegistry_AddRequestSkipsRequestsWithGenerationStatus(t *testing.T) { + registry := newTestBulkOperationRegistry(&MockClient{}) + request := &beeremote.JobRequest{} + request.SetGenerationStatus(&beeremote.JobRequest_GenerationStatus{}) + + skipSubmit, err := registry.AddRequest(context.Background(), request) + require.NoError(t, err) + assert.False(t, skipSubmit) + assert.Empty(t, registry.managers) +} + +func TestBulkOperationRegistry_AddRequestNotIncludedDoesNotCreateManager(t *testing.T) { + client := &MockClient{} + client.On("IncludeRequestInBulkOperation", mock.Anything, mock.Anything).Return(false, "") + registry := newTestBulkOperationRegistry(client) + + request := &beeremote.JobRequest{} + request.SetRemoteStorageTarget(1) + + skipSubmit, err := registry.AddRequest(context.Background(), request) + require.NoError(t, err) + assert.False(t, skipSubmit) + assert.Empty(t, registry.managers) +} + +// TestBulkOperationRegistry_AddRequestCreatesManagerOnDemandAndReusesIt asserts that the first +// request for a given rstId+operation lazily creates its manager (recording it on +// builderBulkOperations), and that subsequent requests for the same key reuse it rather than +// creating a second manager. +func TestBulkOperationRegistry_AddRequestCreatesManagerOnDemandAndReusesIt(t *testing.T) { + client := &MockClient{} + client.On("IncludeRequestInBulkOperation", mock.Anything, mock.Anything).Return(true, "retrieve") + registry := newTestBulkOperationRegistry(client) + + for i := 0; i < 2; i++ { + request := &beeremote.JobRequest{} + request.SetRemoteStorageTarget(1) + + skipSubmit, err := registry.AddRequest(context.Background(), request) + require.NoError(t, err) + assert.True(t, skipSubmit) + } + + assert.Len(t, registry.managers, 1) + assert.Len(t, *registry.builderBulkOperations, 1) + + manager := registry.managers["1-retrieve"] + require.NotNil(t, manager) + assert.Equal(t, "retrieve", manager.operation) +} + +func TestBulkOperationRegistry_AddRequestPropagatesManagerAddRequestError(t *testing.T) { + addErr := fmt.Errorf("disk full") + client := &MockClient{} + client.On("IncludeRequestInBulkOperation", mock.Anything, mock.Anything).Return(true, "retrieve") + client.On("OpenBulkOperation", mock.Anything, mock.Anything, mock.Anything).Return(&fakeBulkOperation{addRequestErr: addErr}, nil) + registry := newTestBulkOperationRegistry(client) + + request := &beeremote.JobRequest{} + request.SetRemoteStorageTarget(1) + + _, err := registry.AddRequest(context.Background(), request) + require.ErrorIs(t, err, addErr) +} + +func TestBulkOperationRegistry_CloseAggregatesManagerCloseErrors(t *testing.T) { + closeErr := fmt.Errorf("failed to unmount") + registry := &bulkOperationRegistry{ + managers: map[string]*bulkOperationManager{ + "1-retrieve": { + clientBulkOperation: &fakeBulkOperation{closeErr: closeErr}, + operation: "retrieve", + errors: new(string), + failed: new(bool), + }, + }, + } + + err := registry.Close(context.Background()) + require.Error(t, err) + assert.ErrorIs(t, err, closeErr) + assert.Contains(t, err.Error(), "1-retrieve") +} + +func TestBulkOperationManager_AppendErrorAccumulatesAndGetErrorsFormats(t *testing.T) { + manager := &bulkOperationManager{operation: "archive", errors: new(string), failed: new(bool)} + assert.NoError(t, manager.GetErrors()) + + manager.AppendError(fmt.Errorf("first")) + manager.AppendError(fmt.Errorf("second")) + manager.AppendError(nil) + + err := manager.GetErrors() + require.Error(t, err) + assert.Equal(t, "bulk operation archive: (first. second)", err.Error()) +} diff --git a/common/rst/buildercontroller.go b/common/rst/buildercontroller.go new file mode 100644 index 00000000..e3d1cca9 --- /dev/null +++ b/common/rst/buildercontroller.go @@ -0,0 +1,428 @@ +package rst + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + + "golang.org/x/sync/errgroup" +) + +const ( + // avgProcessTimeAlpha is the EWMA weight for each new sample in workThroughput. Higher values + // react faster and lower smooths more. + avgProcessTimeAlpha = 0.15 + // processTimeStaleWindow ensures gaps longer than this reset the average instead of blending + // into it. + processTimeStaleWindow = 10 * time.Second + // processTimeOutlierCapMultiplier caps a single sample at this multiple of the current average, + // so one stuck completion can't dominate the estimate in one update. Sustained slowdowns still + // get through over a few samples as the cap rises with the average. + processTimeOutlierCapMultiplier = 3 + // submissionQueueBaseDrainTime is the acceptable buffered latency in jobSubmissionCh when the + // worker pool has no work. submissionQueueTargetDrainTime shrinks this toward + // submissionQueueMinDrainTime as worker saturation rises, tightening backpressure so the + // builder doesn't keep generating submissions faster than the cluster has capacity to execute. + submissionQueueBaseDrainTime = 2 * time.Second + // submissionQueueMinDrainTime floors how far the target can shrink under heavy worker + // saturation, so the builder always retains some forward progress instead of stalling outright. + submissionQueueMinDrainTime = 50 * time.Millisecond + // submissionQueueRampStartSaturation is the saturation percentage below which the builder + // doesn't throttle at all. The goal is 100% saturation (one job per worker), not zero, so there's + // no reason to start shrinking the drain time target while comfortably under that; the target + // only ramps down from submissionQueueBaseDrainTime to submissionQueueMinDrainTime across the + // range from this value up to 100%. + submissionQueueRampStartSaturation = 50.0 +) + +// requestBuildController uses bounded fan-out stage instead of a worker pool where each each path +// accepted from the walk stream is processed in its own goroutine, while the errgroup limit caps +// the number of paths processed concurrently. These goroutines are also throttled by calls to +// waitForSubmissionCapacity. +// +// A fixed worker pool and bounded per-path goroutines both park during blocking I/O and have +// equivalent throughput at the same concurrency. The advantage here is simpler backpressure: +// processWalk stops launching new path processors when the downstream job submission queue reaches +// its threshold, so queue pressure propagates back through the pipeline. +// +// Benchmarks showed goroutine spawn overhead (~1-2us) is negligible compared with BeeGFS metadata +// operations. The throughput matched a fixed pool across the measured I/O delay ranges. +type requestBuildController struct { + ctx context.Context + maxWorkersCh chan struct{} + getPaths requestPathResolverFn + requestBuilder *jobRequestBuilder + backpressureThreshold int + workerSaturation []func() float64 + processTimeMu sync.Mutex + processTimeAvg float64 + processTimeCounter atomic.Int64 + lastProcessTime time.Time + + sourceGroup *errgroup.Group + sourceGroupCtx context.Context + sourceProducerGroup *errgroup.Group + activeJobSubmissions atomic.Int64 // All non-terminal submitted requests. Requests add to a bulk operation will not counter. + + bulkGroup *errgroup.Group + bulkGroupCtx context.Context + bulkCallbacks []func() + + result *SchedulingResult + resumeToken string +} + +func (c *requestBuildController) WalkSourceGenerator(nextWalkCh nextWalkChGenerator, resumeToken string, activeJobSubmissionsTarget int) { + if c.sourceGroup == nil { + c.sourceGroup, c.sourceGroupCtx = errgroup.WithContext(c.ctx) + c.sourceProducerGroup = new(errgroup.Group) + } + + c.sourceGroup.Go(func() error { + walkCh, err := nextWalkCh(resumeToken) + if err != nil { + return err + } + + for { + select { + case <-c.sourceGroupCtx.Done(): + return c.sourceGroupCtx.Err() + case result, ok := <-walkCh: + if !ok { + return nil + } + + var failedPrecondition error + if result.Err != nil { + if cancelErr, ok := errors.AsType[*RequestCancelError](result.Err); ok { + failedPrecondition = cancelErr.Reason + } else { + return result.Err + } + } + + if result.ResumeToken != "" { + // Allow any active source processors to finish so activeJobSubmissions will be + // correct before deciding whether to start another walk. + if err := c.sourceProducerGroup.Wait(); err != nil { + return err + } + if c.activeJobSubmissions.Load() < int64(activeJobSubmissionsTarget) { + if walkCh, err = nextWalkCh(result.ResumeToken); err != nil { + return err + } + continue + } + + if c.resumeToken != "" { + return fmt.Errorf("conflicting walk resume tokens: [%s, %s]", c.resumeToken, result.ResumeToken) + } + c.resumeToken = result.ResumeToken + c.result = &SchedulingResult{Reschedule: true} + return nil + } + + inMountPath, remotePath, err := c.getPaths(result.Path) + if err != nil { + return err + } + + c.addWorker() + start := time.Now() + c.sourceProducerGroup.Go(func() error { + defer func() { c.releaseWorker(time.Since(start)) }() + // c.ctx, not c.sourceGroupCtx: sourceGroupCtx belongs to c.sourceGroup, whose + // Wait() cancels it as soon as the outer walk-driving goroutine returns + // (including on success) which can race ahead of sourceProducerGroup.Wait() and + // cancel still-in-flight workers here out from under them (WaitForWalkSources + // waits on sourceGroup before sourceProducerGroup). This goroutine belongs to + // sourceProducerGroup, so it should only stop when the controller's own context + // says so, not as a side effect of a sibling group's lifecycle. + submitted, err := c.requestBuilder.ProcessFromSource(c.ctx, inMountPath, remotePath, failedPrecondition) + c.activeJobSubmissions.Add(submitted) + return err + }) + + } + + if err := c.waitForSubmissionCapacity(); err != nil { + return err + } + } + }) +} + +func (c *requestBuildController) ExecuteBulkOperation(manager *bulkOperationManager) { + if manager.IsFailed() { + return + } + + if c.bulkGroup == nil { + c.bulkGroup, c.bulkGroupCtx = errgroup.WithContext(c.ctx) + } + + walkCh, getResult, err := manager.Execute(c.bulkGroupCtx) + if err != nil { + c.CancelBulkOperation(manager, err) + return + } + + c.bulkCallbacks = append(c.bulkCallbacks, func() { + result := getResult() + if result.Err != nil { + manager.AppendError(result.Err) + c.CancelBulkOperation(manager, result.Err) + } + if result.Reschedule && (c.result == nil || !c.result.Reschedule || result.Delay < c.result.Delay) { + c.result = result + } + }) + + processWalkCh(c.bulkGroupCtx, c.bulkGroup, c.bulkProcess, c.waitForSubmissionCapacity, walkCh) +} + +func (c *requestBuildController) CancelBulkOperation(manager *bulkOperationManager, reason error) { + if manager.IsFailed() { + return + } + + if c.bulkGroup == nil { + c.bulkGroup, c.bulkGroupCtx = errgroup.WithContext(c.ctx) + } + + walkCh, getResult, err := manager.Cancel(c.bulkGroupCtx, reason) + if err != nil { + manager.AppendError(err) + manager.SetFailed() + return + } + + c.bulkCallbacks = append(c.bulkCallbacks, func() { + if err := getResult(); err != nil { + manager.AppendError(err) + manager.SetFailed() + } + }) + + processWalkCh(c.bulkGroupCtx, c.bulkGroup, c.bulkProcess, c.waitForSubmissionCapacity, walkCh) +} + +func processWalkCh[T any](ctx context.Context, group *errgroup.Group, process func(T) error, check func() error, walkCh <-chan T) { + group.Go(func() error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case result, ok := <-walkCh: + if !ok { + return nil + } + if err := process(result); err != nil { + return err + } + } + + if err := check(); err != nil { + return err + } + } + }) +} + +// GetResults returns the merged scheduling result and resume token accumulated so far. It does not +// wait for outstanding source or bulk walks; callers must first call WaitForWalkSources and +// WaitForBulkOperations to ensure all in-flight work has completed. It is safe to call more than +// once but will only return the results accumulated since requestBuildController's instantiation or +// the previous GetResults call. +func (c *requestBuildController) GetResults() (result *SchedulingResult, resumeToken string) { + result = &SchedulingResult{} + if c.result != nil { + result.Reschedule = c.result.Reschedule + result.Delay = c.result.Delay + result.Err = c.result.Err + c.result = nil + } + resumeToken = c.resumeToken + return +} + +// WaitForBulkOperations waits for all bulk operations to finish. Any returned errors should not be +// considered fatal so any bulk operations can finish. +func (c *requestBuildController) WaitForWalkSources() error { + if c.sourceGroup == nil { + return nil + } + + err := c.sourceGroup.Wait() + c.sourceGroup = nil + + err = errors.Join(err, c.sourceProducerGroup.Wait()) + if err != nil { + return fmt.Errorf("error walking source paths: %w", err) + } + return nil +} + +// WaitForBulkOperations waits for all bulk operations to finish. Any returned errors should be +// considered fatal and stop builder job. +func (c *requestBuildController) WaitForBulkOperations() (err error) { + if c.bulkGroup == nil { + return + } + + // Additional callbacks can be appended to c.bulkCallbacks by individual callbacks. So it's + // important to execute each from a copy after emptying c.bulkCallbacks. + for len(c.bulkCallbacks) > 0 { + if c.bulkGroup != nil { + err = errors.Join(err, c.bulkGroup.Wait()) + c.bulkGroup = nil + } + callbacks := c.bulkCallbacks + c.bulkCallbacks = nil + for _, callback := range callbacks { + callback() + } + } + return err +} + +func (c *requestBuildController) bulkProcess(result *BulkStreamPathResult) error { + var failedPrecondition error + if result.Err != nil { + if cancelErr, ok := errors.AsType[*RequestCancelError](result.Err); ok { + failedPrecondition = cancelErr.Reason + } else { + return result.Err + } + } + + inMountPath, remotePath, err := c.getPaths(result.Path) + if err != nil { + return err + } + + c.addWorker() + start := time.Now() + c.bulkGroup.Go(func() error { + defer func() { c.releaseWorker(time.Since(start)) }() + return c.requestBuilder.ProcessFromBulkOperation(c.ctx, inMountPath, remotePath, result.RstId, result.BulkInfo, failedPrecondition) + }) + + return nil +} + +func (c *requestBuildController) waitForSubmissionCapacity() error { + if !c.submissionQueueOverCapacity() { + return nil + } + + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + + for c.submissionQueueOverCapacity() { + select { + case <-c.ctx.Done(): + return c.ctx.Err() + case <-ticker.C: + } + } + return nil +} + +func (c *requestBuildController) submissionQueueOverCapacity() bool { + occupancy := len(c.requestBuilder.jobSubmissionCh) + throughput := c.workThroughput() + if throughput <= 0 { + // There's no samples yet so use the static threshold. + return occupancy >= c.backpressureThreshold + } + estimatedDrainTime := time.Duration(float64(occupancy) / throughput * float64(time.Second)) + return estimatedDrainTime > c.submissionQueueTargetDrainTime() +} + +// submissionQueueTargetDrainTime derives the acceptable jobSubmissionCh buffered latency from +// current worker saturation instead of a fixed constant. The goal is 100% saturation (one job per +// worker), not zero, so the target stays at submissionQueueBaseDrainTime (no throttling) while +// saturation is under submissionQueueRampStartSaturation, then shrinks toward +// submissionQueueMinDrainTime as saturation climbs the rest of the way to 100%: the busier the +// worker pool actually executing real jobs, the less sense it makes to let the builder keep piling +// submissions in ahead of it. +// +// This combines workerSaturation's windows (shortest to longest) with a max rather than just +// reading workerSaturation[0], so the response is asymmetric: a spike on the short window tightens +// the target immediately (fast reaction to overload), but relaxing it back down requires the +// longest window to also confirm sustained low load, since the max stays pulled up by whichever +// window is slowest to decay. That mirrors the fast-tighten/slow-loosen shape of AIMD-style +// congestion control, and avoids the target flapping loose again during a brief lull mid-burst only +// to immediately re-trigger backpressure. +func (c *requestBuildController) submissionQueueTargetDrainTime() time.Duration { + if len(c.workerSaturation) == 0 { + return submissionQueueBaseDrainTime + } + + saturation := c.workerSaturation[0]() + if longWindow := c.workerSaturation[len(c.workerSaturation)-1](); longWindow > saturation { + saturation = longWindow + } + + if saturation <= submissionQueueRampStartSaturation { + return submissionQueueBaseDrainTime + } + + factor := 1 - (saturation-submissionQueueRampStartSaturation)/(100-submissionQueueRampStartSaturation) + if factor < 0 { + factor = 0 + } + + span := submissionQueueBaseDrainTime - submissionQueueMinDrainTime + target := submissionQueueMinDrainTime + time.Duration(factor*float64(span)) + return target +} + +func (c *requestBuildController) addWorker() { + c.maxWorkersCh <- struct{}{} +} + +func (c *requestBuildController) releaseWorker(processTime time.Duration) { + <-c.maxWorkersCh + + now := time.Now() + + c.processTimeMu.Lock() + defer c.processTimeMu.Unlock() + if c.lastProcessTime.IsZero() || now.Sub(c.lastProcessTime) > processTimeStaleWindow { + // There was no recent sample to average with so (re)start the average. + c.processTimeAvg = float64(processTime) + c.processTimeCounter.Store(1) + } else { + sample := float64(processTime) + if outlierCap := c.processTimeAvg * processTimeOutlierCapMultiplier; sample > outlierCap { + // Bound how far this single completion can pull the average, so one atypically slow + // path (e.g. stuck behind a network retry) doesn't dominate the estimate in one update. + // A genuine sustained slowdown still comes through over the next few samples, since + // avgProcessTime itself rises each time, raising the cap along with it. + sample = outlierCap + } + alpha := max(avgProcessTimeAlpha, 1/float64(c.processTimeCounter.Add(1))) + c.processTimeAvg = alpha*sample + (1-alpha)*c.processTimeAvg + } + c.lastProcessTime = now +} + +// workThroughput returns the current estimated path completions-per-second across all in-flight +// goroutines. The number of goroutines currently processing a path is divided by the rolling +// average per-path processing duration, since each goroutine is completing paths in parallel at +// roughly the same average rate. Returns 0 if no path has completed yet. +func (c *requestBuildController) workThroughput() float64 { + c.processTimeMu.Lock() + defer c.processTimeMu.Unlock() + + if c.processTimeAvg <= 0 { + return 0 + } + return float64(len(c.maxWorkersCh)) * float64(time.Second) / c.processTimeAvg +} diff --git a/common/rst/buildercontroller_test.go b/common/rst/buildercontroller_test.go new file mode 100644 index 00000000..3d5fd4f4 --- /dev/null +++ b/common/rst/buildercontroller_test.go @@ -0,0 +1,457 @@ +package rst + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/thinkparq/beegfs-go/common/beegfs" + "github.com/thinkparq/beegfs-go/common/beemsg/msg" + "github.com/thinkparq/beegfs-go/common/filesystem" + "github.com/thinkparq/beegfs-go/ctl/pkg/ctl/entry" + "github.com/thinkparq/protobuf/go/beeremote" + "github.com/thinkparq/protobuf/go/flex" +) + +func TestRequestBuildController_WalkSourceProcessesPathsAndSubmitsRequests(t *testing.T) { + ctx := context.Background() + jobSubmissionCh := make(chan *beeremote.JobRequest, 10) + controller := newTestRequestBuildController(ctx, jobSubmissionCh) + + walkCh := make(chan *filesystem.StreamPathResult, 2) + walkCh <- &filesystem.StreamPathResult{Path: "/a"} + walkCh <- &filesystem.StreamPathResult{Path: "/b"} + close(walkCh) + + controller.WalkSourceGenerator(testWalkChGenerator(walkCh), "", 0) + require.NoError(t, controller.WaitForWalkSources()) + + result, resumeToken := controller.GetResults() + assert.Empty(t, resumeToken) + assert.False(t, result.Reschedule) + assert.ElementsMatch(t, []string{"/a", "/b"}, submittedPaths(jobSubmissionCh)) +} + +func TestRequestBuildController_WalkSourceSetsResumeTokenAndReschedules(t *testing.T) { + ctx := context.Background() + jobSubmissionCh := make(chan *beeremote.JobRequest, 10) + controller := newTestRequestBuildController(ctx, jobSubmissionCh) + + walkCh := make(chan *filesystem.StreamPathResult, 1) + walkCh <- &filesystem.StreamPathResult{ResumeToken: "resume-token"} + close(walkCh) + + controller.WalkSourceGenerator(testWalkChGenerator(walkCh), "", 0) + require.NoError(t, controller.WaitForWalkSources()) + + result, resumeToken := controller.GetResults() + assert.Equal(t, "resume-token", resumeToken) + assert.True(t, result.Reschedule) +} + +func TestRequestBuildController_WalkSourceRejectsConflictingResumeTokens(t *testing.T) { + ctx := context.Background() + jobSubmissionCh := make(chan *beeremote.JobRequest, 10) + controller := newTestRequestBuildController(ctx, jobSubmissionCh) + controller.resumeToken = "existing-token" + + walkCh := make(chan *filesystem.StreamPathResult, 1) + walkCh <- &filesystem.StreamPathResult{ResumeToken: "new-token"} + close(walkCh) + + controller.WalkSourceGenerator(testWalkChGenerator(walkCh), "", 0) + err := controller.WaitForWalkSources() + require.Error(t, err) + assert.Contains(t, err.Error(), "conflicting walk resume tokens") +} + +func TestRequestBuildController_WalkSourceReturnsWalkErrors(t *testing.T) { + ctx := context.Background() + jobSubmissionCh := make(chan *beeremote.JobRequest, 10) + controller := newTestRequestBuildController(ctx, jobSubmissionCh) + + walkErr := fmt.Errorf("walk failed") + walkCh := make(chan *filesystem.StreamPathResult, 1) + walkCh <- &filesystem.StreamPathResult{Err: walkErr} + close(walkCh) + + controller.WalkSourceGenerator(testWalkChGenerator(walkCh), "", 0) + err := controller.WaitForWalkSources() + require.ErrorIs(t, err, walkErr) +} + +// TestRequestBuildController_WalkSourceConvertsRequestCancelErrorToFailedPrecondition asserts that a +// RequestCancelError on the walk result does not fail the builder job. Instead it is submitted as a +// FAILED_PRECONDITION request carrying the cancellation reason. +func TestRequestBuildController_WalkSourceConvertsRequestCancelErrorToFailedPrecondition(t *testing.T) { + ctx := context.Background() + jobSubmissionCh := make(chan *beeremote.JobRequest, 10) + controller := newTestRequestBuildController(ctx, jobSubmissionCh) + + walkCh := make(chan *filesystem.StreamPathResult, 1) + walkCh <- &filesystem.StreamPathResult{Path: "/a", Err: &RequestCancelError{Reason: errors.New("cancelled")}} + close(walkCh) + + controller.WalkSourceGenerator(testWalkChGenerator(walkCh), "", 0) + require.NoError(t, controller.WaitForWalkSources()) + + requests := drainRequests(jobSubmissionCh) + require.Len(t, requests, 1) + assert.Equal(t, beeremote.JobRequest_GenerationStatus_FAILED_PRECONDITION, requests[0].GetGenerationStatus().GetState()) + assert.Equal(t, "cancelled", requests[0].GetGenerationStatus().GetMessage()) +} + +func TestRequestBuildController_ExecuteBulkOperationProcessesPathsAndSubmitsRequests(t *testing.T) { + ctx := context.Background() + jobSubmissionCh := make(chan *beeremote.JobRequest, 10) + controller := newTestRequestBuildController(ctx, jobSubmissionCh) + + bulkCh := make(chan *BulkStreamPathResult, 1) + bulkCh <- &BulkStreamPathResult{ + Path: "/bulk-a", + RstId: 1, + BulkInfo: &flex.BulkJobRequestInfo{Operation: "retrieve"}, + } + close(bulkCh) + + manager := newTestBulkManager("mgr", func(ctx context.Context) (<-chan *BulkStreamPathResult, BulkExecuteResultFn, error) { + return bulkCh, func() *SchedulingResult { return &SchedulingResult{} }, nil + }, nil) + controller.ExecuteBulkOperation(manager) + require.NoError(t, controller.WaitForBulkOperations()) + + result, resumeToken := controller.GetResults() + assert.Empty(t, resumeToken) + assert.False(t, result.Reschedule) + assert.Equal(t, []string{"/bulk-a"}, submittedPaths(jobSubmissionCh)) +} + +func TestRequestBuildController_ExecuteBulkOperationReturnsWalkErrors(t *testing.T) { + ctx := context.Background() + jobSubmissionCh := make(chan *beeremote.JobRequest, 10) + controller := newTestRequestBuildController(ctx, jobSubmissionCh) + + walkErr := fmt.Errorf("bulk walk failed") + bulkCh := make(chan *BulkStreamPathResult, 1) + bulkCh <- &BulkStreamPathResult{Err: walkErr} + close(bulkCh) + + manager := newTestBulkManager("mgr", func(ctx context.Context) (<-chan *BulkStreamPathResult, BulkExecuteResultFn, error) { + return bulkCh, func() *SchedulingResult { return &SchedulingResult{} }, nil + }, nil) + controller.ExecuteBulkOperation(manager) + + err := controller.WaitForBulkOperations() + require.ErrorIs(t, err, walkErr) +} + +func TestRequestBuildController_ExecuteBulkOperationNoopWhenManagerAlreadyFailed(t *testing.T) { + ctx := context.Background() + jobSubmissionCh := make(chan *beeremote.JobRequest, 10) + controller := newTestRequestBuildController(ctx, jobSubmissionCh) + + manager := newTestBulkManager("mgr", func(ctx context.Context) (<-chan *BulkStreamPathResult, BulkExecuteResultFn, error) { + t.Fatal("Execute should not be called for an already-failed manager") + return nil, nil, nil + }, nil) + manager.SetFailed() + + controller.ExecuteBulkOperation(manager) + require.NoError(t, controller.WaitForBulkOperations()) +} + +// TestRequestBuildController_ExecuteBulkOperationExecuteErrorCancelsAndSurfacesReason covers the case +// where a registered bulk manager fails before it ever produces a walk channel (e.g. it could not be +// opened). ExecuteBulkOperation must fall back to cancelling the manager with that error as the +// reason rather than silently dropping it. A well-behaved clientBulkOperation forwards the reason +// for any paths it can't complete via its own Cancel walk channel, so the failure still surfaces +// through WaitForBulkOperations. +func TestRequestBuildController_ExecuteBulkOperationExecuteErrorCancelsAndSurfacesReason(t *testing.T) { + ctx := context.Background() + jobSubmissionCh := make(chan *beeremote.JobRequest, 10) + controller := newTestRequestBuildController(ctx, jobSubmissionCh) + + openErr := fmt.Errorf("failed to open bulk operation") + manager := newTestBulkManager("mgr", + func(ctx context.Context) (<-chan *BulkStreamPathResult, BulkExecuteResultFn, error) { + return nil, nil, openErr + }, + func(ctx context.Context, reason error) (<-chan *BulkStreamPathResult, BulkCancelResultFn, error) { + walkCh := make(chan *BulkStreamPathResult, 1) + walkCh <- &BulkStreamPathResult{Err: reason} + close(walkCh) + return walkCh, func() error { return nil }, nil + }, + ) + controller.ExecuteBulkOperation(manager) + + err := controller.WaitForBulkOperations() + require.ErrorIs(t, err, openErr) +} + +// TestRequestBuildController_ExecuteBulkOperationMergesRescheduleAcrossManagers asserts that when +// multiple bulk managers report a reschedule, the merged result keeps the smallest delay while the +// error from the manager that produced it is preserved on the merged result. +func TestRequestBuildController_ExecuteBulkOperationMergesRescheduleAcrossManagers(t *testing.T) { + ctx := context.Background() + jobSubmissionCh := make(chan *beeremote.JobRequest, 10) + controller := newTestRequestBuildController(ctx, jobSubmissionCh) + + boomErr := fmt.Errorf("boom") + emptyBulkExecuteFn := func(delay time.Duration, err error) BulkExecuteFn { + return func(ctx context.Context) (<-chan *BulkStreamPathResult, BulkExecuteResultFn, error) { + walkCh := make(chan *BulkStreamPathResult) + close(walkCh) + return walkCh, func() *SchedulingResult { + return &SchedulingResult{Reschedule: true, Delay: delay, Err: err} + }, nil + } + } + // A result.Err from getResult() triggers an automatic cancel of the manager, so every manager + // here needs a Cancel implementation even though the test isn't exercising cancellation itself. + noopCancel := func(ctx context.Context, reason error) (<-chan *BulkStreamPathResult, BulkCancelResultFn, error) { + walkCh := make(chan *BulkStreamPathResult) + close(walkCh) + return walkCh, func() error { return nil }, nil + } + + slowManager := newTestBulkManager("slow", emptyBulkExecuteFn(5*time.Second, nil), noopCancel) + fastManager := newTestBulkManager("fast", emptyBulkExecuteFn(2*time.Second, boomErr), noopCancel) + controller.ExecuteBulkOperation(slowManager) + controller.ExecuteBulkOperation(fastManager) + + require.NoError(t, controller.WaitForBulkOperations()) + + result, _ := controller.GetResults() + assert.True(t, result.Reschedule) + assert.Equal(t, 2*time.Second, result.Delay) + assert.ErrorIs(t, result.Err, boomErr) +} + +func TestRequestBuildController_CancelBulkOperationNoopWhenManagerAlreadyFailed(t *testing.T) { + ctx := context.Background() + jobSubmissionCh := make(chan *beeremote.JobRequest, 10) + controller := newTestRequestBuildController(ctx, jobSubmissionCh) + + manager := newTestBulkManager("mgr", nil, func(ctx context.Context, reason error) (<-chan *BulkStreamPathResult, BulkCancelResultFn, error) { + t.Fatal("Cancel should not be called for an already-failed manager") + return nil, nil, nil + }) + manager.SetFailed() + + controller.CancelBulkOperation(manager, fmt.Errorf("reason")) + require.NoError(t, controller.WaitForBulkOperations()) +} + +func TestRequestBuildController_CancelBulkOperationSetsManagerFailedWhenCancelErrors(t *testing.T) { + ctx := context.Background() + jobSubmissionCh := make(chan *beeremote.JobRequest, 10) + controller := newTestRequestBuildController(ctx, jobSubmissionCh) + + cancelErr := fmt.Errorf("cannot cancel") + manager := newTestBulkManager("mgr", nil, func(ctx context.Context, reason error) (<-chan *BulkStreamPathResult, BulkCancelResultFn, error) { + return nil, nil, cancelErr + }) + + controller.CancelBulkOperation(manager, fmt.Errorf("reason")) + + assert.True(t, manager.IsFailed()) + require.Error(t, manager.GetErrors()) + assert.Contains(t, manager.GetErrors().Error(), cancelErr.Error()) + // Cancel failed before any walk goroutine was spawned, so there's nothing left to wait for. + require.NoError(t, controller.WaitForBulkOperations()) +} + +func TestRequestBuildController_CancelBulkOperationSetsManagerFailedOnWaitError(t *testing.T) { + ctx := context.Background() + jobSubmissionCh := make(chan *beeremote.JobRequest, 10) + controller := newTestRequestBuildController(ctx, jobSubmissionCh) + + waitErr := fmt.Errorf("cancel wait failed") + manager := newTestBulkManager("mgr", nil, func(ctx context.Context, reason error) (<-chan *BulkStreamPathResult, BulkCancelResultFn, error) { + walkCh := make(chan *BulkStreamPathResult) + close(walkCh) + return walkCh, func() error { return waitErr }, nil + }) + + controller.CancelBulkOperation(manager, fmt.Errorf("reason")) + require.NoError(t, controller.WaitForBulkOperations()) + + assert.True(t, manager.IsFailed()) + require.Error(t, manager.GetErrors()) + assert.Contains(t, manager.GetErrors().Error(), waitErr.Error()) +} + +func TestRequestBuildController_WaitForWalkSourcesReturnsImmediatelyWhenNoSourceWalk(t *testing.T) { + controller := &requestBuildController{} + require.NoError(t, controller.WaitForWalkSources()) +} + +func TestRequestBuildController_WaitForBulkOperationsReturnsImmediatelyWhenNoBulkWalk(t *testing.T) { + controller := &requestBuildController{} + require.NoError(t, controller.WaitForBulkOperations()) +} + +// TestRequestBuildController_PathProcessingConcurrencyIsBounded asserts that maxWorkersCh actually +// bounds how many paths are processed concurrently: with a single worker slot, a second path must +// not start processing until the first releases its slot. +func TestRequestBuildController_PathProcessingConcurrencyIsBounded(t *testing.T) { + ctx := context.Background() + jobSubmissionCh := make(chan *beeremote.JobRequest, 10) + controller := newTestRequestBuildController(ctx, jobSubmissionCh) + controller.maxWorkersCh = make(chan struct{}, 1) + + started := make(chan struct{}, 3) + release := make(chan struct{}) + var mu sync.Mutex + var maxInFlight, inFlight int + + baseGetPathState := controller.requestBuilder.getPathState + controller.requestBuilder.getPathState = func(ctx context.Context, mountPoint filesystem.Provider, inMountPath string, mode PathStateMode) (PathState, error) { + mu.Lock() + inFlight++ + if inFlight > maxInFlight { + maxInFlight = inFlight + } + mu.Unlock() + + started <- struct{}{} + <-release + + mu.Lock() + inFlight-- + mu.Unlock() + return baseGetPathState(ctx, mountPoint, inMountPath, mode) + } + + walkCh := make(chan *filesystem.StreamPathResult, 3) + walkCh <- &filesystem.StreamPathResult{Path: "/a"} + walkCh <- &filesystem.StreamPathResult{Path: "/b"} + walkCh <- &filesystem.StreamPathResult{Path: "/c"} + close(walkCh) + + controller.WalkSourceGenerator(testWalkChGenerator(walkCh), "", 0) + + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("first path never started processing") + } + + select { + case <-started: + t.Fatal("a second path started processing before the first released its worker slot") + case <-time.After(100 * time.Millisecond): + } + + close(release) + + require.NoError(t, controller.WaitForWalkSources()) + + mu.Lock() + defer mu.Unlock() + assert.Equal(t, 1, maxInFlight) + assert.Equal(t, 0, inFlight) +} + +// testWalkChGenerator returns a nextWalkChGenerator that always hands back walkCh, for tests that +// only need a single walk channel and never expect nextWalkCh to be called with a follow-up resume +// token. +func testWalkChGenerator(walkCh <-chan *filesystem.StreamPathResult) nextWalkChGenerator { + return func(resumeToken string) (<-chan *filesystem.StreamPathResult, error) { + return walkCh, nil + } +} + +// submittedPaths closes and drains jobSubmissionCh, returning the path of every submitted request. +// Callers must only invoke this once no further sends can occur, e.g. after +// requestBuildController.WaitForWalkSources()/WaitForBulkOperations(). +func submittedPaths(jobSubmissionCh chan *beeremote.JobRequest) []string { + var paths []string + for _, req := range drainRequests(jobSubmissionCh) { + paths = append(paths, req.GetPath()) + } + return paths +} + +// drainRequests closes and drains jobSubmissionCh, returning every submitted request. Callers must +// only invoke this once no further sends can occur. +func drainRequests(jobSubmissionCh chan *beeremote.JobRequest) []*beeremote.JobRequest { + close(jobSubmissionCh) + var requests []*beeremote.JobRequest + for req := range jobSubmissionCh { + requests = append(requests, req) + } + return requests +} + +func newTestRequestBuildController(ctx context.Context, jobSubmissionCh chan *beeremote.JobRequest) *requestBuildController { + client := NewJobBuilderClient(ctx, map[uint32]Provider{1: &MockClient{}}, filesystem.NewMockFS()) + cfg := &flex.JobRequestCfg{RemoteStorageTarget: 1} + controller := client.newRequestBuildController(ctx, cfg, jobSubmissionCh, func(ctx context.Context, request *beeremote.JobRequest) (bool, error) { + return false, nil + }, nil) + + controller.requestBuilder.getPathState = func(ctx context.Context, mountPoint filesystem.Provider, inMountPath string, mode PathStateMode) (PathState, error) { + return PathState{ + LockedInfo: &flex.JobLockedInfo{}, + LockAcquired: true, + EntryInfo: &entry.GetEntryCombinedInfo{}, + RstCfg: msg.RemoteStorageTarget{ + RSTIDs: []uint32{1}, + }, + }, nil + } + controller.requestBuilder.planFileState = func(ctx context.Context, mountPoint filesystem.Provider, cfg *flex.JobRequestCfg) (applyPlanFn, error) { + return func(*PathState) (undoFn, error) { return func() error { return nil }, nil }, nil + } + controller.requestBuilder.clearAccessFlags = func(ctx context.Context, path string, flags beegfs.AccessFlags) error { + return nil + } + + return controller +} + +// stubBulkOperation lets tests inject Execute/Cancel behavior directly into a *bulkOperationManager +// without going through a real clientBulkOperation implementation. +type stubBulkOperation struct { + executeFn BulkExecuteFn + cancelFn BulkCancelFn +} + +func (s *stubBulkOperation) AddRequest(ctx context.Context, request *beeremote.JobRequest) error { + return nil +} + +func (s *stubBulkOperation) Execute(ctx context.Context) (<-chan *BulkStreamPathResult, BulkExecuteResultFn, error) { + return s.executeFn(ctx) +} + +func (s *stubBulkOperation) Cancel(ctx context.Context, reason error) (<-chan *BulkStreamPathResult, BulkCancelResultFn, error) { + return s.cancelFn(ctx, reason) +} + +func (s *stubBulkOperation) Close(ctx context.Context) error { + return nil +} + +func (m *stubBulkOperation) Destroy(ctx context.Context) error { + return nil +} + +// newTestBulkManager builds a *bulkOperationManager backed by a stub clientBulkOperation, so tests +// can inject Execute/Cancel behavior without a real RST client. executeFn/cancelFn may be nil if the +// test never exercises that method. +func newTestBulkManager(operation string, executeFn BulkExecuteFn, cancelFn BulkCancelFn) *bulkOperationManager { + return &bulkOperationManager{ + clientBulkOperation: &stubBulkOperation{executeFn: executeFn, cancelFn: cancelFn}, + operation: operation, + errors: new(string), + failed: new(bool), + } +} diff --git a/common/rst/builderjobrequest.go b/common/rst/builderjobrequest.go new file mode 100644 index 00000000..1b43a7b0 --- /dev/null +++ b/common/rst/builderjobrequest.go @@ -0,0 +1,365 @@ +package rst + +import ( + "context" + "errors" + "fmt" + "math" + "time" + + "github.com/thinkparq/beegfs-go/common/beegfs" + "github.com/thinkparq/beegfs-go/common/filesystem" + "github.com/thinkparq/beegfs-go/ctl/pkg/ctl/entry" + "github.com/thinkparq/protobuf/go/beeremote" + "github.com/thinkparq/protobuf/go/flex" + "google.golang.org/protobuf/proto" +) + +type requestPathResolverFn func(walkPath string) (inMountPath string, remotePath string, err error) +type addBulkRequestFn func(ctx context.Context, request *beeremote.JobRequest) (skipSubmit bool, err error) +type getPathStateFn func(ctx context.Context, mountPoint filesystem.Provider, inMountPath string, mode PathStateMode) (PathState, error) +type planFileStateForWorkRequestsFn func(ctx context.Context, mountPoint filesystem.Provider, cfg *flex.JobRequestCfg) (applyPlanFn, error) +type clearAccessFlagsFn func(ctx context.Context, path string, flags beegfs.AccessFlags) error +type setDirRstConfigFn func(ctx context.Context, inMountPath string) (isDir bool, err error) + +type jobRequestBuilder struct { + mountPoint filesystem.Provider + RstMap map[uint32]Provider + jobSubmissionCh chan<- *beeremote.JobRequest + builderCfg *flex.JobRequestCfg + addBulkRequest addBulkRequestFn + getPathState getPathStateFn + planFileState planFileStateForWorkRequestsFn + clearAccessFlags clearAccessFlagsFn + setDirRstConfig setDirRstConfigFn +} + +func (w *jobRequestBuilder) init() { + w.initSetRstConfig() +} + +func (w *jobRequestBuilder) initSetRstConfig() { + if !(w.builderCfg.GetUpdate() || w.builderCfg.HasCooldownSecs()) { + // When neither builder config Update nor CooldownSec are set then directories are not + // included in the walk and we can safely ignore directory configuration updates. + w.setDirRstConfig = func(context.Context, string) (bool, error) { return false, nil } + return + } + + var rstIds []uint32 + if w.builderCfg.GetUpdate() && IsValidRstId(w.builderCfg.RemoteStorageTarget) { + rstIds = []uint32{w.builderCfg.RemoteStorageTarget} + } + + var cooldownSecs *uint16 + if w.builderCfg.HasCooldownSecs() { + v := uint16(math.MaxUint16) + if w.builderCfg.GetCooldownSecs() <= math.MaxUint16 { + v = uint16(w.builderCfg.GetCooldownSecs()) + } + cooldownSecs = &v + } + + w.setDirRstConfig = func(ctx context.Context, inMountPath string) (bool, error) { + stat, err := w.mountPoint.Lstat(inMountPath) + if err != nil { + return false, err + } + + return stat.IsDir(), entry.SetDirRstPattern(ctx, inMountPath, rstIds, cooldownSecs) + } +} + +func (w *jobRequestBuilder) ProcessFromSource(ctx context.Context, inMountPath string, remotePath string, failedPrecondition error) (activeJobSubmissions int64, err error) { + if isDir, err := w.setDirRstConfig(ctx, inMountPath); isDir || err != nil { + // Abort the builder job since the beegfs was unable to set the directory's rst + // configuration. The issue is likely systemic. + return 0, err + } + + var pathState PathState + var skip bool + var pathIssue error + if pathState, skip, pathIssue, err = w.resolvePathStateForRequest(ctx, inMountPath); err != nil || skip { + return + } else if pathIssue != nil { + failedPrecondition = appendError(failedPrecondition, pathIssue) + } + + var keepLock bool + if !pathState.LockAcquired && FileExists(pathState.LockedInfo) && !IsFileOffloaded(pathState.LockedInfo) { + keepLock = true + failedPrecondition = appendError(failedPrecondition, fmt.Errorf("file access lock is already held")) + } + defer func() { + if !keepLock { + if clearErr := w.clearAccessFlags(ctx, inMountPath, beegfs.LockedContentAccessFlags); clearErr != nil { + err = errors.Join(err, fmt.Errorf("unable to clear lock: %w", clearErr)) + } + } + }() + + for _, cfg := range w.buildJobRequestCfgs(inMountPath, remotePath, pathState.RstCfg.RSTIDs, pathState.LockedInfo, w.builderCfg) { + request := w.buildJobRequest(ctx, cfg, failedPrecondition) + canReleaseLock, processErr := w.processJobRequestCfg(ctx, cfg, pathState, request) + if !canReleaseLock { + keepLock = true + } + if processErr != nil { + err = processErr + return + } + + if !request.HasGenerationStatus() { + activeJobSubmissions++ + } + } + + return +} + +func (w *jobRequestBuilder) ProcessFromBulkOperation( + ctx context.Context, + inMountPath string, + remotePath string, + rstId uint32, + BulkInfo *flex.BulkJobRequestInfo, + failedPrecondition error, +) (err error) { + pathState, pathStateErr := w.getPathState(ctx, w.mountPoint, inMountPath, PathStateWithLock) + if errors.Is(pathStateErr, ErrGetPathStateFatal) { + // Returning err from this function aborts the entire builder job, so only fatal path + // state errors are returned here. Non-fatal path state errors are attached to the + // generated request when an rstId is available, allowing the builder job to continue. + err = pathStateErr + return + } + + // The file access lock must be acquired by this builder job before adding it to a bulk + // operation so releasing it is acceptable. + var keepLock bool + defer func() { + if !keepLock { + if clearErr := w.clearAccessFlags(ctx, inMountPath, beegfs.LockedContentAccessFlags); clearErr != nil { + err = errors.Join(err, fmt.Errorf("unable to clear lock: %w", clearErr)) + } + } + }() + + cfg := w.buildJobRequestCfg(inMountPath, remotePath, rstId, pathState.LockedInfo, w.builderCfg) + request := w.buildJobRequest(ctx, cfg, failedPrecondition) + request.SetRemoteStorageTarget(rstId) + request.SetBulkInfo(BulkInfo) + canReleaseLock, processErr := w.processJobRequestCfg(ctx, cfg, pathState, request) + if !canReleaseLock { + keepLock = true + } + if processErr != nil { + err = processErr + return + } + + return +} + +func (w *jobRequestBuilder) resolvePathStateForRequest(ctx context.Context, inMountPath string) (pathState PathState, skip bool, pathIssue error, err error) { + var pathStateErr error + pathState, pathStateErr = w.getPathState(ctx, w.mountPoint, inMountPath, PathStateWithLock) + if errors.Is(pathStateErr, ErrGetPathStateFatal) { + // Returning err from this function aborts the entire builder job, so only fatal path + // state errors are returned here. Non-fatal path state errors are attached to the + // generated request when an rstId is available, allowing the builder job to continue. + err = pathStateErr + return + } + + // If the caller specified a valid remote storage target, use it as the request's rstId. + // Otherwise, rely on any rstIds discovered from the file state. If none are available, + // skip the path so no request is created and the builder job does not fail. This is valid + // because callers can trigger jobs from configured file rstIds without specifying a target. + if IsValidRstId(w.builderCfg.RemoteStorageTarget) { + if IsFileOffloaded(pathState.LockedInfo) && w.builderCfg.RemoteStorageTarget != pathState.LockedInfo.StubUrlRstId && !w.builderCfg.GetOverwrite() { + pathIssue = fmt.Errorf("supplied --%s does not match stub file", RemoteTargetFlag) + } + pathState.RstCfg.RSTIDs = []uint32{w.builderCfg.RemoteStorageTarget} + } else if len(pathState.RstCfg.RSTIDs) == 0 && pathStateErr == nil { + skip = true + return + } + + if pathStateErr != nil { + pathIssue = pathStateErr + } else if len(pathState.RstCfg.RSTIDs) > 1 && (w.builderCfg.Download || w.builderCfg.StubLocal) { + pathIssue = ErrFileHasAmbiguousRSTs + } + + return +} + +// buildJobRequestCfgs returns a jobRequestCfg list for each rstId. Each jobRequestCfg is a clone of +// cfg updated with the provided information. +func (w *jobRequestBuilder) buildJobRequestCfgs( + inMountPath string, + remotePath string, + rstIds []uint32, + lockedInfo *flex.JobLockedInfo, + cfg *flex.JobRequestCfg, +) []*flex.JobRequestCfg { + var requests []*flex.JobRequestCfg + for _, rstId := range rstIds { + request := w.buildJobRequestCfg(inMountPath, remotePath, rstId, lockedInfo, cfg) + requests = append(requests, request) + } + return requests +} + +// buildJobRequestCfgs returns a jobRequestCfg for each rstId. jobRequestCfg is a clone of cfg +// updated with the provided information. +func (w *jobRequestBuilder) buildJobRequestCfg( + inMountPath string, + remotePath string, + rstId uint32, + lockedInfo *flex.JobLockedInfo, + cfg *flex.JobRequestCfg, +) *flex.JobRequestCfg { + request := proto.Clone(cfg).(*flex.JobRequestCfg) + request.SetPath(inMountPath) + request.SetRemotePath(remotePath) + request.SetRemoteStorageTarget(rstId) + request.SetLockedInfo(proto.Clone(lockedInfo).(*flex.JobLockedInfo)) + return request +} + +// processJobRequestCfg builds, prepares, and submits the job request for cfg. canReleaseLock is +// returned true only when this path produced no in-flight work that still depends on the lock; once +// the request is routed, prepared, or submitted for real work, the lock must remain held. +func (w *jobRequestBuilder) processJobRequestCfg( + ctx context.Context, + cfg *flex.JobRequestCfg, + pathState PathState, + request *beeremote.JobRequest, +) (canReleaseLock bool, err error) { + lockedInfo := cfg.GetLockedInfo() + var applyPlan applyPlanFn + if request.HasGenerationStatus() { + canReleaseLock = true + } else { + var planErr error + if applyPlan, planErr = w.planFileState(ctx, w.mountPoint, cfg); planErr != nil { + canReleaseLock = true + request.SetGenerationStatus(&beeremote.JobRequest_GenerationStatus{ + State: beeremote.JobRequest_GenerationStatus_FAILED_PRECONDITION, + Message: fmt.Sprintf("failed to prepare file state: %s", planErr.Error()), + }) + } + } + + if !request.HasGenerationStatus() { + if !request.HasBulkInfo() { + var skipSubmission bool + if skipSubmission, err = w.addBulkRequest(ctx, request); err != nil { + // addBulkRequest failed and since the file access lock was newly acquired, it may + // be released. + canReleaseLock = true + return + } else if skipSubmission { + return + } else if request.HasGenerationStatus() { + // addBulkRequest may reject inclusion outright (eg the target bulk operation + // previously failed permanently) by attaching a GenerationStatus rather than + // deferring to the bulk operation's own walk. Honor it the same way as if it had + // been set from the start, so the request is submitted as-is instead of being + // planned and applied as though nothing happened. + canReleaseLock = true + } + } + } + + if !request.HasGenerationStatus() { + applyUndo, applyErr := applyPlan(&pathState) + if applyErr != nil { + if errors.Is(applyErr, ErrJobAlreadyComplete) { + canReleaseLock = true + request.GenerationStatus = &beeremote.JobRequest_GenerationStatus{ + State: beeremote.JobRequest_GenerationStatus_ALREADY_COMPLETE, + Message: lockedInfo.Mtime.AsTime().Format(time.RFC3339), + } + } else if errors.Is(applyErr, ErrJobAlreadyOffloaded) { + canReleaseLock = false + request.GenerationStatus = &beeremote.JobRequest_GenerationStatus{ + State: beeremote.JobRequest_GenerationStatus_ALREADY_OFFLOADED, + } + } else if errors.Is(applyErr, ErrJobFailedPrecondition) { + canReleaseLock = true + request.SetGenerationStatus(&beeremote.JobRequest_GenerationStatus{ + State: beeremote.JobRequest_GenerationStatus_FAILED_PRECONDITION, + Message: fmt.Sprintf("failed to prepare file state: %s", applyErr.Error()), + }) + } else { + canReleaseLock = false + request.SetGenerationStatus(&beeremote.JobRequest_GenerationStatus{ + State: beeremote.JobRequest_GenerationStatus_ERROR, + Message: fmt.Sprintf("failed to prepare file state: %s", applyErr.Error()), + }) + } + } else { + // Generating the externalId must be the last possible error to avoid situations where, once the + // externalId is generated, it would be lost as a result of a subsequent preconditional failure. + client := w.RstMap[request.GetRemoteStorageTarget()] + externalId, externalIdErr := client.GenerateExternalId(ctx, cfg) + if externalIdErr != nil { + if undoErr := applyUndo(); undoErr != nil { + canReleaseLock = false + request.SetGenerationStatus(&beeremote.JobRequest_GenerationStatus{ + State: beeremote.JobRequest_GenerationStatus_ERROR, + Message: fmt.Sprintf("failed to generate external id: %s; rollback also failed: %s", externalIdErr.Error(), undoErr.Error()), + }) + } else { + canReleaseLock = true + request.SetGenerationStatus(&beeremote.JobRequest_GenerationStatus{ + State: beeremote.JobRequest_GenerationStatus_FAILED_PRECONDITION, + Message: fmt.Sprintf("failed to generate external id: %s", externalIdErr.Error()), + }) + } + } else { + lockedInfo.SetExternalId(externalId) + canReleaseLock = false + } + } + } + + w.submitJobRequest(ctx, request) + return +} + +func (w *jobRequestBuilder) buildJobRequest(ctx context.Context, cfg *flex.JobRequestCfg, failedPrecondition error) *beeremote.JobRequest { + rstId := cfg.GetRemoteStorageTarget() + client, ok := w.RstMap[rstId] + if !ok { + // The rstId is from the file's RST config but has no matching client. This means it was + // either removed after the file was configured, or was set incorrectly. Return a + // FAILED_PRECONDITION so remote submits the job with the error message rather than + // rejecting the job. + return &beeremote.JobRequest{ + Path: cfg.Path, + RemoteStorageTarget: cfg.GetRemoteStorageTarget(), + GenerationStatus: &beeremote.JobRequest_GenerationStatus{ + State: beeremote.JobRequest_GenerationStatus_FAILED_PRECONDITION, + Message: fmt.Sprintf("failed to build job request: %s: rstId %d", ErrConfigRSTTypeIsUnknown.Error(), rstId), + }, + } + } + + if failedPrecondition != nil { + return BuildJobRequestWithFailedPrecondition(client, cfg, failedPrecondition.Error()) + } + return BuildJobRequest(ctx, client, cfg) +} + +func (w *jobRequestBuilder) submitJobRequest(ctx context.Context, request *beeremote.JobRequest) { + select { + case <-ctx.Done(): + return + case w.jobSubmissionCh <- request: + } +} diff --git a/common/rst/builderjobrequest_test.go b/common/rst/builderjobrequest_test.go new file mode 100644 index 00000000..ef969532 --- /dev/null +++ b/common/rst/builderjobrequest_test.go @@ -0,0 +1,605 @@ +package rst + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/thinkparq/beegfs-go/common/beegfs" + "github.com/thinkparq/beegfs-go/common/beemsg/msg" + "github.com/thinkparq/beegfs-go/common/filesystem" + "github.com/thinkparq/beegfs-go/ctl/pkg/ctl/entry" + "github.com/thinkparq/protobuf/go/beeremote" + "github.com/thinkparq/protobuf/go/flex" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestJobRequestBuilder_InitSetDirRstConfigNoopWhenDirsNotWalked(t *testing.T) { + w := &jobRequestBuilder{builderCfg: &flex.JobRequestCfg{}} + w.initSetRstConfig() + + // mountPoint is intentionally left nil; if the no-op short circuit didn't take effect this + // would panic when the real implementation tries to Lstat. + isDir, err := w.setDirRstConfig(context.Background(), "/some/path") + assert.False(t, isDir) + assert.NoError(t, err) +} + +func TestJobRequestBuilder_ResolvePathStateForRequest(t *testing.T) { + fixedMtime := timestamppb.Now() + + tests := []struct { + name string + builderCfg *flex.JobRequestCfg + pathState PathState + pathStateErr error + wantErr bool + wantSkip bool + wantPathIssue bool + wantRstIds []uint32 + }{ + { + name: "fatal path state error is returned as err", + builderCfg: &flex.JobRequestCfg{}, + pathStateErr: fmt.Errorf("%w: %w", ErrGetPathStateFatal, errors.New("boom")), + wantErr: true, + }, + { + name: "no valid rstId and no discovered rstIds skips the path", + builderCfg: &flex.JobRequestCfg{}, + pathState: PathState{RstCfg: msg.RemoteStorageTarget{RSTIDs: nil}}, + wantSkip: true, + }, + { + name: "explicit valid rstId overrides discovered rstIds", + builderCfg: &flex.JobRequestCfg{RemoteStorageTarget: 5}, + pathState: PathState{ + LockedInfo: &flex.JobLockedInfo{Mtime: fixedMtime}, + RstCfg: msg.RemoteStorageTarget{RSTIDs: []uint32{1, 2}}, + }, + wantRstIds: []uint32{5}, + }, + { + name: "explicit rstId mismatching offloaded stub without overwrite records a path issue", + builderCfg: &flex.JobRequestCfg{RemoteStorageTarget: 5, Overwrite: false}, + pathState: PathState{ + LockedInfo: &flex.JobLockedInfo{Mtime: fixedMtime, StubUrlRstId: 9}, + RstCfg: msg.RemoteStorageTarget{RSTIDs: []uint32{9}}, + }, + wantPathIssue: true, + wantRstIds: []uint32{5}, + }, + { + name: "explicit rstId mismatching offloaded stub with overwrite records no issue", + builderCfg: &flex.JobRequestCfg{RemoteStorageTarget: 5, Overwrite: true}, + pathState: PathState{ + LockedInfo: &flex.JobLockedInfo{Mtime: fixedMtime, StubUrlRstId: 9}, + RstCfg: msg.RemoteStorageTarget{RSTIDs: []uint32{9}}, + }, + wantRstIds: []uint32{5}, + }, + { + name: "non-fatal path state error is recorded as a path issue", + builderCfg: &flex.JobRequestCfg{}, + pathState: PathState{ + LockedInfo: &flex.JobLockedInfo{Mtime: fixedMtime}, + RstCfg: msg.RemoteStorageTarget{RSTIDs: []uint32{1}}, + }, + pathStateErr: errors.New("non-fatal issue"), + wantPathIssue: true, + wantRstIds: []uint32{1}, + }, + { + name: "multiple discovered rstIds with download set is ambiguous", + builderCfg: &flex.JobRequestCfg{Download: true}, + pathState: PathState{ + LockedInfo: &flex.JobLockedInfo{Mtime: fixedMtime}, + RstCfg: msg.RemoteStorageTarget{RSTIDs: []uint32{1, 2}}, + }, + wantPathIssue: true, + wantRstIds: []uint32{1, 2}, + }, + { + name: "multiple discovered rstIds without download or stub-local is fine", + builderCfg: &flex.JobRequestCfg{}, + pathState: PathState{ + LockedInfo: &flex.JobLockedInfo{Mtime: fixedMtime}, + RstCfg: msg.RemoteStorageTarget{RSTIDs: []uint32{1, 2}}, + }, + wantRstIds: []uint32{1, 2}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + w := &jobRequestBuilder{ + builderCfg: tt.builderCfg, + getPathState: func(ctx context.Context, mountPoint filesystem.Provider, inMountPath string, mode PathStateMode) (PathState, error) { + return tt.pathState, tt.pathStateErr + }, + } + + pathState, skip, pathIssue, err := w.resolvePathStateForRequest(context.Background(), "/some/path") + + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantSkip, skip) + if tt.wantSkip { + return + } + if tt.wantPathIssue { + assert.Error(t, pathIssue) + } else { + assert.NoError(t, pathIssue) + } + assert.Equal(t, tt.wantRstIds, pathState.RstCfg.RSTIDs) + }) + } +} + +func TestJobRequestBuilder_BuildJobRequestCfgs(t *testing.T) { + w := &jobRequestBuilder{} + cfg := &flex.JobRequestCfg{RemoteStorageTarget: 99, Path: "/original", Priority: new(int32(3))} + lockedInfo := &flex.JobLockedInfo{Size: 42} + + requests := w.buildJobRequestCfgs("/in-mount/path", "remote/path", []uint32{1, 2}, lockedInfo, cfg) + + require.Len(t, requests, 2) + for i, wantRstId := range []uint32{1, 2} { + assert.Equal(t, "/in-mount/path", requests[i].GetPath()) + assert.Equal(t, "remote/path", requests[i].GetRemotePath()) + assert.Equal(t, wantRstId, requests[i].GetRemoteStorageTarget()) + require.NotNil(t, requests[i].GetLockedInfo()) + assert.Equal(t, int64(42), requests[i].GetLockedInfo().GetSize()) + // Each generated cfg must own an independent clone of lockedInfo. + assert.NotSame(t, lockedInfo, requests[i].GetLockedInfo()) + } + // The original cfg passed in must not be mutated by cloning. + assert.Equal(t, "/original", cfg.Path) + assert.Equal(t, uint32(99), cfg.RemoteStorageTarget) +} + +func TestJobRequestBuilder_BuildJobRequest(t *testing.T) { + t.Run("unknown rstId returns failed precondition without a client", func(t *testing.T) { + w := &jobRequestBuilder{RstMap: map[uint32]Provider{}} + cfg := &flex.JobRequestCfg{Path: "/foo", RemoteStorageTarget: 7} + + request := w.buildJobRequest(context.Background(), cfg, nil) + + require.True(t, request.HasGenerationStatus()) + assert.Equal(t, beeremote.JobRequest_GenerationStatus_FAILED_PRECONDITION, request.GetGenerationStatus().GetState()) + assert.Equal(t, "/foo", request.GetPath()) + assert.Equal(t, uint32(7), request.GetRemoteStorageTarget()) + }) + + t.Run("failedPrecondition produces a failed precondition request", func(t *testing.T) { + client := &MockClient{} + w := &jobRequestBuilder{RstMap: map[uint32]Provider{1: client}} + cfg := &flex.JobRequestCfg{Path: "/foo", RemoteStorageTarget: 1} + + request := w.buildJobRequest(context.Background(), cfg, errors.New("precondition failed")) + + require.True(t, request.HasGenerationStatus()) + assert.Equal(t, beeremote.JobRequest_GenerationStatus_FAILED_PRECONDITION, request.GetGenerationStatus().GetState()) + assert.Contains(t, request.GetGenerationStatus().GetMessage(), "precondition failed") + }) + + t.Run("no failedPrecondition builds a normal request", func(t *testing.T) { + client := &MockClient{} + w := &jobRequestBuilder{RstMap: map[uint32]Provider{1: client}} + cfg := &flex.JobRequestCfg{ + Path: "/foo", + RemoteStorageTarget: 1, + LockedInfo: &flex.JobLockedInfo{}, + } + + request := w.buildJobRequest(context.Background(), cfg, nil) + + assert.False(t, request.HasGenerationStatus()) + assert.Equal(t, "/foo", request.GetPath()) + }) +} + +func TestJobRequestBuilder_ProcessJobRequestCfg(t *testing.T) { + t.Run("request with generation status releases lock and is submitted", func(t *testing.T) { + w := &jobRequestBuilder{ + RstMap: map[uint32]Provider{}, + jobSubmissionCh: make(chan *beeremote.JobRequest, 1), + } + cfg := &flex.JobRequestCfg{Path: "/foo", RemoteStorageTarget: 99} // no matching client -> FAILED_PRECONDITION + request := w.buildJobRequest(context.Background(), cfg, nil) + + canReleaseLock, err := w.processJobRequestCfg(context.Background(), cfg, PathState{}, request) + + require.NoError(t, err) + assert.True(t, canReleaseLock) + require.Len(t, w.jobSubmissionCh, 1) + }) + + t.Run("bulk request failure releases lock, returns err, and does not submit", func(t *testing.T) { + client := &MockClient{} + w := &jobRequestBuilder{ + RstMap: map[uint32]Provider{1: client}, + jobSubmissionCh: make(chan *beeremote.JobRequest, 1), + addBulkRequest: func(ctx context.Context, request *beeremote.JobRequest) (bool, error) { + return false, errors.New("bulk add failed") + }, + planFileState: func(ctx context.Context, mountPoint filesystem.Provider, cfg *flex.JobRequestCfg) (applyPlanFn, error) { + return func(*PathState) (undoFn, error) { return func() error { return nil }, nil }, nil + }, + } + cfg := &flex.JobRequestCfg{Path: "/foo", RemoteStorageTarget: 1, LockedInfo: &flex.JobLockedInfo{}} + request := w.buildJobRequest(context.Background(), cfg, nil) + + canReleaseLock, err := w.processJobRequestCfg(context.Background(), cfg, PathState{}, request) + + require.Error(t, err) + assert.True(t, canReleaseLock) + assert.Empty(t, w.jobSubmissionCh) + }) + + t.Run("bulk request skip keeps the lock and does not submit", func(t *testing.T) { + client := &MockClient{} + w := &jobRequestBuilder{ + RstMap: map[uint32]Provider{1: client}, + jobSubmissionCh: make(chan *beeremote.JobRequest, 1), + addBulkRequest: func(ctx context.Context, request *beeremote.JobRequest) (bool, error) { + return true, nil + }, + planFileState: func(ctx context.Context, mountPoint filesystem.Provider, cfg *flex.JobRequestCfg) (applyPlanFn, error) { + return func(*PathState) (undoFn, error) { return func() error { return nil }, nil }, nil + }, + } + cfg := &flex.JobRequestCfg{Path: "/foo", RemoteStorageTarget: 1, LockedInfo: &flex.JobLockedInfo{}} + request := w.buildJobRequest(context.Background(), cfg, nil) + + canReleaseLock, err := w.processJobRequestCfg(context.Background(), cfg, PathState{}, request) + + require.NoError(t, err) + assert.False(t, canReleaseLock) + assert.Empty(t, w.jobSubmissionCh) + }) + + t.Run("successful preparation submits the request and keeps the lock", func(t *testing.T) { + client := &MockClient{} + client.On("GenerateExternalId", mock.Anything, mock.Anything).Return("external-id", nil) + submissionCh := make(chan *beeremote.JobRequest, 1) + w := &jobRequestBuilder{ + RstMap: map[uint32]Provider{1: client}, + jobSubmissionCh: submissionCh, + addBulkRequest: func(ctx context.Context, request *beeremote.JobRequest) (bool, error) { + return false, nil + }, + planFileState: func(ctx context.Context, mountPoint filesystem.Provider, cfg *flex.JobRequestCfg) (applyPlanFn, error) { + return func(*PathState) (undoFn, error) { return func() error { return nil }, nil }, nil + }, + } + cfg := &flex.JobRequestCfg{Path: "/foo", RemoteStorageTarget: 1, LockedInfo: &flex.JobLockedInfo{}} + request := w.buildJobRequest(context.Background(), cfg, nil) + + canReleaseLock, err := w.processJobRequestCfg(context.Background(), cfg, PathState{EntryInfo: &entry.GetEntryCombinedInfo{}}, request) + + require.NoError(t, err) + assert.False(t, canReleaseLock) + require.Len(t, submissionCh, 1) + // The externalId is generated after the request is built, so it lands on cfg's + // LockedInfo rather than the already-built submitted request. + assert.Equal(t, "external-id", cfg.GetLockedInfo().GetExternalId()) + }) +} + +func TestJobRequestBuilder_SubmitJobRequest(t *testing.T) { + t.Run("submits when the channel has capacity", func(t *testing.T) { + submissionCh := make(chan *beeremote.JobRequest, 1) + w := &jobRequestBuilder{jobSubmissionCh: submissionCh} + request := &beeremote.JobRequest{Path: "/foo"} + + w.submitJobRequest(context.Background(), request) + + require.Len(t, submissionCh, 1) + assert.Equal(t, request, <-submissionCh) + }) + + t.Run("returns without blocking when the context is already cancelled", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + // Unbuffered channel with no reader would block forever if submitJobRequest didn't + // respect ctx.Done(). + w := &jobRequestBuilder{jobSubmissionCh: make(chan *beeremote.JobRequest)} + + done := make(chan struct{}) + go func() { + w.submitJobRequest(ctx, &beeremote.JobRequest{}) + close(done) + }() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("submitJobRequest blocked despite a cancelled context") + } + }) +} + +func TestJobRequestBuilder_ProcessFromSource(t *testing.T) { + newBuilder := func() *jobRequestBuilder { + return &jobRequestBuilder{ + RstMap: map[uint32]Provider{}, + jobSubmissionCh: make(chan *beeremote.JobRequest, 2), + builderCfg: &flex.JobRequestCfg{}, + } + } + + t.Run("directories are skipped without clearing the lock", func(t *testing.T) { + w := newBuilder() + w.setDirRstConfig = func(ctx context.Context, inMountPath string) (bool, error) { return true, nil } + w.clearAccessFlags = func(ctx context.Context, path string, flags beegfs.AccessFlags) error { + t.Fatal("clearAccessFlags should not be called for directories") + return nil + } + + activeJobSubmissions, err := w.ProcessFromSource(context.Background(), "/some/dir", "", nil) + require.NoError(t, err) + assert.Zero(t, activeJobSubmissions) + }) + + t.Run("setDirRstConfig error is propagated without clearing the lock", func(t *testing.T) { + w := newBuilder() + wantErr := errors.New("dir config failed") + w.setDirRstConfig = func(ctx context.Context, inMountPath string) (bool, error) { return false, wantErr } + w.clearAccessFlags = func(ctx context.Context, path string, flags beegfs.AccessFlags) error { + t.Fatal("clearAccessFlags should not be called") + return nil + } + + activeJobSubmissions, err := w.ProcessFromSource(context.Background(), "/some/path", "", nil) + require.ErrorIs(t, err, wantErr) + assert.Zero(t, activeJobSubmissions) + }) + + t.Run("skip from resolvePathStateForRequest returns without clearing the lock", func(t *testing.T) { + w := newBuilder() + w.setDirRstConfig = func(ctx context.Context, inMountPath string) (bool, error) { return false, nil } + w.getPathState = func(ctx context.Context, mountPoint filesystem.Provider, inMountPath string, mode PathStateMode) (PathState, error) { + return PathState{}, nil // No RSTIDs and no explicit target -> skip. + } + w.clearAccessFlags = func(ctx context.Context, path string, flags beegfs.AccessFlags) error { + t.Fatal("clearAccessFlags should not be called when the path is skipped") + return nil + } + + activeJobSubmissions, err := w.ProcessFromSource(context.Background(), "/some/path", "", nil) + require.NoError(t, err) + assert.Zero(t, activeJobSubmissions) + }) + + t.Run("lock is cleared once processing completes without in-flight work", func(t *testing.T) { + w := newBuilder() + w.setDirRstConfig = func(ctx context.Context, inMountPath string) (bool, error) { return false, nil } + w.getPathState = func(ctx context.Context, mountPoint filesystem.Provider, inMountPath string, mode PathStateMode) (PathState, error) { + return PathState{ + LockedInfo: &flex.JobLockedInfo{Mtime: timestamppb.Now()}, + LockAcquired: true, + RstCfg: msg.RemoteStorageTarget{RSTIDs: []uint32{1}}, // No client registered -> FAILED_PRECONDITION request. + }, nil + } + var cleared bool + w.clearAccessFlags = func(ctx context.Context, path string, flags beegfs.AccessFlags) error { + cleared = true + assert.Equal(t, beegfs.LockedContentAccessFlags, flags) + return nil + } + + activeJobSubmissions, err := w.ProcessFromSource(context.Background(), "/some/path", "/remote/path", nil) + + require.NoError(t, err) + assert.True(t, cleared) + require.Len(t, w.jobSubmissionCh, 1) + assert.Zero(t, activeJobSubmissions) + }) + + t.Run("existing lock is reported as failed precondition", func(t *testing.T) { + client := &MockClient{} + jobSubmissionCh := make(chan *beeremote.JobRequest, 2) + w := newBuilder() + w.jobSubmissionCh = jobSubmissionCh + w.RstMap = map[uint32]Provider{1: client} + w.setDirRstConfig = func(ctx context.Context, inMountPath string) (bool, error) { return false, nil } + w.getPathState = func(ctx context.Context, mountPoint filesystem.Provider, inMountPath string, mode PathStateMode) (PathState, error) { + return PathState{ + LockedInfo: &flex.JobLockedInfo{Exists: true, Mtime: timestamppb.Now()}, + RstCfg: msg.RemoteStorageTarget{RSTIDs: []uint32{1}}, + }, nil + } + w.clearAccessFlags = func(ctx context.Context, path string, flags beegfs.AccessFlags) error { + t.Fatal("clearAccessFlags should not be called for a lock this builder did not acquire") + return nil + } + + activeJobSubmissions, err := w.ProcessFromSource(context.Background(), "/some/path", "/remote/path", nil) + + require.NoError(t, err) + require.Len(t, jobSubmissionCh, 1) + request := <-jobSubmissionCh + require.NotNil(t, request.GetGenerationStatus()) + assert.Equal(t, beeremote.JobRequest_GenerationStatus_FAILED_PRECONDITION, request.GetGenerationStatus().GetState()) + assert.Equal(t, "file access lock is already held", request.GetGenerationStatus().GetMessage()) + assert.Zero(t, activeJobSubmissions) + }) + + t.Run("lock is held when any generated request has in-flight work", func(t *testing.T) { + client := &MockClient{} + client.On("GenerateExternalId", mock.Anything, mock.Anything).Return("external-id", nil) + w := newBuilder() + w.RstMap = map[uint32]Provider{1: client, 2: client} + w.setDirRstConfig = func(ctx context.Context, inMountPath string) (bool, error) { return false, nil } + w.getPathState = func(ctx context.Context, mountPoint filesystem.Provider, inMountPath string, mode PathStateMode) (PathState, error) { + return PathState{ + LockedInfo: &flex.JobLockedInfo{Mtime: timestamppb.Now()}, + EntryInfo: &entry.GetEntryCombinedInfo{}, + RstCfg: msg.RemoteStorageTarget{RSTIDs: []uint32{1, 2}}, + }, nil + } + w.addBulkRequest = func(ctx context.Context, request *beeremote.JobRequest) (bool, error) { + return false, nil + } + w.planFileState = func(ctx context.Context, mountPoint filesystem.Provider, cfg *flex.JobRequestCfg) (applyPlanFn, error) { + return func(*PathState) (undoFn, error) { return func() error { return nil }, nil }, nil + } + w.clearAccessFlags = func(ctx context.Context, path string, flags beegfs.AccessFlags) error { + t.Fatal("clearAccessFlags should not be called while work is in flight") + return nil + } + + activeJobSubmissions, err := w.ProcessFromSource(context.Background(), "/some/path", "/remote/path", nil) + + require.NoError(t, err) + require.Len(t, w.jobSubmissionCh, 2) + assert.EqualValues(t, 2, activeJobSubmissions) + }) + + t.Run("clearAccessFlags error is joined into the returned error", func(t *testing.T) { + w := newBuilder() + w.setDirRstConfig = func(ctx context.Context, inMountPath string) (bool, error) { return false, nil } + w.getPathState = func(ctx context.Context, mountPoint filesystem.Provider, inMountPath string, mode PathStateMode) (PathState, error) { + return PathState{ + LockedInfo: &flex.JobLockedInfo{Mtime: timestamppb.Now()}, + LockAcquired: true, + RstCfg: msg.RemoteStorageTarget{RSTIDs: []uint32{1}}, + }, nil + } + wantErr := errors.New("clear failed") + w.clearAccessFlags = func(ctx context.Context, path string, flags beegfs.AccessFlags) error { + return wantErr + } + + activeJobSubmissions, err := w.ProcessFromSource(context.Background(), "/some/path", "/remote/path", nil) + + require.ErrorIs(t, err, wantErr) + assert.Zero(t, activeJobSubmissions) + }) +} + +func TestJobRequestBuilder_ProcessFromBulkOperation(t *testing.T) { + newBuilder := func() *jobRequestBuilder { + return &jobRequestBuilder{ + RstMap: map[uint32]Provider{}, + jobSubmissionCh: make(chan *beeremote.JobRequest, 2), + builderCfg: &flex.JobRequestCfg{}, + } + } + bulkInfo := &flex.BulkJobRequestInfo{Operation: "retrieve"} + + t.Run("fatal path state error is returned as err without clearing the lock", func(t *testing.T) { + w := newBuilder() + wantErr := fmt.Errorf("%w: %w", ErrGetPathStateFatal, errors.New("boom")) + w.getPathState = func(ctx context.Context, mountPoint filesystem.Provider, inMountPath string, mode PathStateMode) (PathState, error) { + return PathState{}, wantErr + } + w.clearAccessFlags = func(ctx context.Context, path string, flags beegfs.AccessFlags) error { + t.Fatal("clearAccessFlags should not be called on a fatal path state error") + return nil + } + + err := w.ProcessFromBulkOperation(context.Background(), "/some/path", "/remote/path", 1, bulkInfo, nil) + + require.ErrorIs(t, err, ErrGetPathStateFatal) + }) + + t.Run("non-fatal path state error does not block building the request or clearing the lock", func(t *testing.T) { + // Unlike ProcessFromSource, the rstId for a bulk operation is supplied directly by the + // caller rather than discovered from path state, so a non-fatal path state error has + // nothing to attach to and is dropped. + w := newBuilder() + submissionCh := make(chan *beeremote.JobRequest, 2) + w.jobSubmissionCh = submissionCh + w.getPathState = func(ctx context.Context, mountPoint filesystem.Provider, inMountPath string, mode PathStateMode) (PathState, error) { + return PathState{}, errors.New("non-fatal issue") + } + var cleared bool + w.clearAccessFlags = func(ctx context.Context, path string, flags beegfs.AccessFlags) error { + cleared = true + return nil + } + + err := w.ProcessFromBulkOperation(context.Background(), "/some/path", "/remote/path", 1, bulkInfo, nil) + + require.NoError(t, err) + assert.True(t, cleared) + require.Len(t, submissionCh, 1) + request := <-submissionCh + assert.NotContains(t, request.GetGenerationStatus().GetMessage(), "non-fatal issue") + }) + + t.Run("lock is cleared once processing completes without in-flight work", func(t *testing.T) { + w := newBuilder() + submissionCh := make(chan *beeremote.JobRequest, 2) + w.jobSubmissionCh = submissionCh + w.getPathState = func(ctx context.Context, mountPoint filesystem.Provider, inMountPath string, mode PathStateMode) (PathState, error) { + return PathState{}, nil // No client registered for rstId 1 -> FAILED_PRECONDITION request. + } + var cleared bool + w.clearAccessFlags = func(ctx context.Context, path string, flags beegfs.AccessFlags) error { + cleared = true + assert.Equal(t, beegfs.LockedContentAccessFlags, flags) + return nil + } + + err := w.ProcessFromBulkOperation(context.Background(), "/some/path", "/remote/path", 1, bulkInfo, nil) + + require.NoError(t, err) + assert.True(t, cleared) + require.Len(t, submissionCh, 1) + request := <-submissionCh + assert.Equal(t, uint32(1), request.GetRemoteStorageTarget()) + assert.Equal(t, bulkInfo, request.GetBulkInfo()) + }) + + t.Run("lock is held when processing produces in-flight work", func(t *testing.T) { + client := &MockClient{} + client.On("GenerateExternalId", mock.Anything, mock.Anything).Return("external-id", nil) + w := newBuilder() + w.RstMap = map[uint32]Provider{1: client} + w.getPathState = func(ctx context.Context, mountPoint filesystem.Provider, inMountPath string, mode PathStateMode) (PathState, error) { + return PathState{ + LockedInfo: &flex.JobLockedInfo{Mtime: timestamppb.Now()}, + EntryInfo: &entry.GetEntryCombinedInfo{}, + }, nil + } + w.planFileState = func(ctx context.Context, mountPoint filesystem.Provider, cfg *flex.JobRequestCfg) (applyPlanFn, error) { + return func(*PathState) (undoFn, error) { return func() error { return nil }, nil }, nil + } + w.clearAccessFlags = func(ctx context.Context, path string, flags beegfs.AccessFlags) error { + t.Fatal("clearAccessFlags should not be called while work is in flight") + return nil + } + + err := w.ProcessFromBulkOperation(context.Background(), "/some/path", "/remote/path", 1, bulkInfo, nil) + + require.NoError(t, err) + require.Len(t, w.jobSubmissionCh, 1) + }) + + t.Run("clearAccessFlags error is joined into the returned error", func(t *testing.T) { + w := newBuilder() + w.getPathState = func(ctx context.Context, mountPoint filesystem.Provider, inMountPath string, mode PathStateMode) (PathState, error) { + return PathState{}, nil // No client registered for rstId 1 -> FAILED_PRECONDITION request. + } + wantErr := errors.New("clear failed") + w.clearAccessFlags = func(ctx context.Context, path string, flags beegfs.AccessFlags) error { + return wantErr + } + + err := w.ProcessFromBulkOperation(context.Background(), "/some/path", "/remote/path", 1, bulkInfo, nil) + + require.ErrorIs(t, err, wantErr) + }) +} diff --git a/common/rst/errors.go b/common/rst/errors.go index 506e27db..d4244ac4 100644 --- a/common/rst/errors.go +++ b/common/rst/errors.go @@ -30,7 +30,8 @@ var ( ErrOffloadFileUrlMismatch = errors.New("offload file url does not match") ErrOffloadFileNotReadable = errors.New("unable to read stub file") ErrRSTUnavailable = errors.New("remote target is unavailable") - ErrGetLockedInfoFatal = errors.New("fatal error collecting locked info") + ErrGetPathStateFatal = errors.New("fatal error collecting state path info") + ErrBulkOperationCancelRequest = errors.New("bulk operation cancel request") ) func IsErrJobTerminalSentinel(err error) bool { @@ -50,3 +51,13 @@ func (m *MtimeErr) Unwrap() error { return m.Err } func GetErrJobAlreadyCompleteWithMtime(mtime time.Time) *MtimeErr { return &MtimeErr{Err: ErrJobAlreadyComplete, Time: mtime} } + +// Bulk operations may need to be cancelled and in some cases, the unsent job requests should be +// cancelled. Pass &RequestCancelError{Reason: err} to the filesystem.StreamPathResult Err to +// ensure the request is submitted as a failed-precondition rather than an error. +type RequestCancelError struct { + Reason error +} + +func (e *RequestCancelError) Error() string { return e.Reason.Error() } +func (e *RequestCancelError) Unwrap() error { return ErrBulkOperationCancelRequest } diff --git a/common/rst/mock.go b/common/rst/mock.go index f404b498..c059345a 100644 --- a/common/rst/mock.go +++ b/common/rst/mock.go @@ -3,12 +3,15 @@ package rst import ( "context" "fmt" + "os" + "sync" "time" "github.com/stretchr/testify/mock" "github.com/thinkparq/beegfs-go/common/filesystem" "github.com/thinkparq/protobuf/go/beeremote" "github.com/thinkparq/protobuf/go/flex" + "google.golang.org/protobuf/proto" ) // MockClient can be used to mock RST client behavior. This is mostly useful when testing other @@ -45,33 +48,69 @@ import ( // - You CANNOT use `Mock.On` with the `MockJob` request type. type MockClient struct { mock.Mock + bulkMu sync.Mutex + completedBulkPaths map[string]struct{} } var _ Provider = &MockClient{} -func (r *MockClient) GetJobRequest(cfg *flex.JobRequestCfg) *beeremote.JobRequest { - return nil +func (m *MockClient) GetJobRequest(cfg *flex.JobRequestCfg) *beeremote.JobRequest { + if m.hasExpectedCall("GetJobRequest") { + args := m.Called(cfg) + return args.Get(0).(*beeremote.JobRequest) + } + + mockJob := &flex.MockJob{ + NumTestSegments: 1, + Cfg: proto.Clone(cfg).(*flex.JobRequestCfg), + } + if cfg.LockedInfo != nil { + mockJob.LockedInfo = proto.Clone(cfg.LockedInfo).(*flex.JobLockedInfo) + if cfg.Download { + mockJob.FileSize = cfg.LockedInfo.GetRemoteSize() + } else { + mockJob.FileSize = cfg.LockedInfo.GetSize() + } + mockJob.ExternalId = cfg.LockedInfo.GetExternalId() + } + + return &beeremote.JobRequest{ + Path: cfg.Path, + RemoteStorageTarget: cfg.RemoteStorageTarget, + StubLocal: cfg.StubLocal, + Priority: cfg.GetPriority(), + Force: cfg.Force, + Type: &beeremote.JobRequest_Mock{ + Mock: mockJob, + }, + Update: cfg.Update, + } } -func (rst *MockClient) GenerateWorkRequests(ctx context.Context, lastJob *beeremote.Job, job *beeremote.Job, availableWorkers int) (requests []*flex.WorkRequest, err error) { +func (m *MockClient) GenerateWorkRequests(ctx context.Context, lastJob *beeremote.Job, job *beeremote.Job, availableWorkers int) (requests []*flex.WorkRequest, err error) { if job.Request.GetMock() != nil { if job.Request.GetMock().ShouldFail { return nil, fmt.Errorf("test requested an error") } - workRequests := RecreateWorkRequests(job, generateSegments(job.Request.GetMock().FileSize, int64(job.Request.GetMock().NumTestSegments), 1)) + numSegments := int64(job.Request.GetMock().NumTestSegments) + if numSegments <= 0 { + numSegments = 1 + } + + workRequests := RecreateWorkRequests(job, generateSegments(job.Request.GetMock().FileSize, numSegments, 1)) return workRequests, nil } - args := rst.Called(job, availableWorkers) + args := m.Called(job, availableWorkers) if args.Error(2) != nil { return nil, args.Error(2) } return args.Get(0).([]*flex.WorkRequest), nil } -func (rst *MockClient) ExecuteWorkRequestPart(ctx context.Context, request *flex.WorkRequest, part *flex.Work_Part) error { +func (m *MockClient) ExecuteWorkRequestPart(ctx context.Context, request *flex.WorkRequest, part *flex.Work_Part) error { if request.GetMock() != nil { if request.GetMock().ShouldFail { @@ -81,7 +120,7 @@ func (rst *MockClient) ExecuteWorkRequestPart(ctx context.Context, request *flex return nil } - args := rst.Called(ctx, request, part) + args := m.Called(ctx, request, part) err := args.Error(0) if err == nil { part.Completed = true @@ -89,12 +128,56 @@ func (rst *MockClient) ExecuteWorkRequestPart(ctx context.Context, request *flex return err } -// ExecuteJobBuilderRequest is not implemented and should never be called. -func (r *MockClient) ExecuteJobBuilderRequest(ctx context.Context, workRequest *flex.WorkRequest, jobSubmissionChan chan<- *beeremote.JobRequest) (bool, error) { - return false, ErrUnsupportedOpForRST +func (m *MockClient) ExecuteJobBuilderRequest(ctx context.Context, workRequest *flex.WorkRequest, jobSubmissionCh chan<- *beeremote.JobRequest, workerSaturation []func() float64) *SchedulingResult { + if !m.hasExpectedCall("ExecuteJobBuilderRequest") { + return &SchedulingResult{Err: ErrUnsupportedOpForRST} + } + + args := m.Called(ctx, workRequest, jobSubmissionCh) + delay, _ := args.Get(1).(time.Duration) + return &SchedulingResult{ + Reschedule: args.Bool(0), + Delay: delay, + Err: args.Error(2), + } } -func (rst *MockClient) CompleteWorkRequests(ctx context.Context, job *beeremote.Job, workResults []*flex.Work, abort bool) error { +func (m *MockClient) IncludeRequestInBulkOperation(ctx context.Context, request *beeremote.JobRequest) (include bool, operation string) { + if m.hasExpectedCall("IncludeRequestInBulkOperation") { + args := m.Called(ctx, request) + return args.Bool(0), args.String(1) + } + + lockedInfo := getMockRequestLockedInfo(request) + if lockedInfo == nil || !lockedInfo.GetIsArchived() { + return false, "" + } + + operation = "retrieve" + if m.isBulkPathCompleted(operation, getMockBulkReplayPath(request)) { + return false, "" + } + + return true, operation +} + +func (m *MockClient) OpenBulkOperation(ctx context.Context, stateMountPath string, operation string) (clientBulkOperation, error) { + if m.hasExpectedCall("OpenBulkOperation") { + args := m.Called(ctx, stateMountPath, operation) + if args.Error(1) != nil { + return nil, args.Error(1) + } + return args.Get(0).(clientBulkOperation), nil + } + + return &mockBulkOperation{ + client: m, + stateMountPath: stateMountPath, + operation: operation, + }, nil +} + +func (m *MockClient) CompleteWorkRequests(ctx context.Context, job *beeremote.Job, workResults []*flex.Work, abort bool) error { if job.Request.GetMock() != nil { if job.Request.GetMock().ShouldFail { @@ -103,32 +186,159 @@ func (rst *MockClient) CompleteWorkRequests(ctx context.Context, job *beeremote. return nil } - args := rst.Called(job, workResults, abort) + args := m.Called(job, workResults, abort) return args.Error(0) } -func (rst *MockClient) GetConfig() *flex.RemoteStorageTarget { - args := rst.Called() +func (m *MockClient) GetConfig() *flex.RemoteStorageTarget { + args := m.Called() return args.Get(0).(*flex.RemoteStorageTarget) } -func (r *MockClient) GetWalk(ctx context.Context, path string, chanSize int, resumeToken string, maxRequests int) (<-chan *filesystem.StreamPathResult, error) { +func (m *MockClient) GetWalk(ctx context.Context, path string, chanSize int, resumeToken string, maxRequests int) (<-chan *filesystem.StreamPathResult, error) { return nil, ErrUnsupportedOpForRST } -func (r *MockClient) SanitizeRemotePath(remotePath string) string { +func (m *MockClient) SanitizeRemotePath(remotePath string) string { return remotePath } -func (r *MockClient) GetRemotePathInfo(ctx context.Context, cfg *flex.JobRequestCfg) (int64, time.Time, bool, bool, error) { - return 0, time.Time{}, false, false, ErrUnsupportedOpForRST +func (m *MockClient) GetRemotePathInfo(ctx context.Context, cfg *flex.JobRequestCfg) (int64, time.Time, bool, bool, error) { + if m.hasExpectedCall("GetRemotePathInfo") { + args := m.Called(ctx, cfg) + return args.Get(0).(int64), args.Get(1).(time.Time), args.Bool(2), args.Bool(3), args.Error(4) + } + + lockedInfo := cfg.GetLockedInfo() + if lockedInfo == nil { + return 0, time.Time{}, false, false, os.ErrNotExist + } + + remoteMtime := time.Time{} + if lockedInfo.GetRemoteMtime() != nil { + remoteMtime = lockedInfo.GetRemoteMtime().AsTime() + } + + return lockedInfo.GetRemoteSize(), remoteMtime, lockedInfo.GetIsArchived(), true, nil } -func (r *MockClient) GenerateExternalId(ctx context.Context, cfg *flex.JobRequestCfg) (string, error) { - return "", ErrUnsupportedOpForRST +func (m *MockClient) GenerateExternalId(ctx context.Context, cfg *flex.JobRequestCfg) (string, error) { + if m.hasExpectedCall("GenerateExternalId") { + args := m.Called(ctx, cfg) + return args.String(0), args.Error(1) + } + + if cfg.GetLockedInfo() != nil && cfg.GetLockedInfo().GetExternalId() != "" { + return cfg.GetLockedInfo().GetExternalId(), nil + } + if cfg.GetRemotePath() != "" { + return cfg.GetRemotePath(), nil + } + return cfg.GetPath(), nil } -func (r *MockClient) IsWorkRequestReady(ctx context.Context, request *flex.WorkRequest) (bool, time.Duration, error) { - args := r.Called(request) +func (m *MockClient) IsWorkRequestReady(ctx context.Context, request *flex.WorkRequest) (bool, time.Duration, error) { + args := m.Called(request) return args.Bool(0), args.Get(1).(time.Duration), args.Error(2) } + +func (m *MockClient) hasExpectedCall(method string) bool { + for _, call := range m.ExpectedCalls { + if call.Method == method { + return true + } + } + return false +} + +func (m *MockClient) isBulkPathCompleted(operation string, path string) bool { + m.bulkMu.Lock() + defer m.bulkMu.Unlock() + _, ok := m.completedBulkPaths[m.getCompletedBulkPathKey(operation, path)] + return ok +} + +func (m *MockClient) markBulkPathCompleted(operation string, path string) { + m.bulkMu.Lock() + defer m.bulkMu.Unlock() + if m.completedBulkPaths == nil { + m.completedBulkPaths = map[string]struct{}{} + } + m.completedBulkPaths[m.getCompletedBulkPathKey(operation, path)] = struct{}{} +} + +func (m *MockClient) getCompletedBulkPathKey(operation string, path string) string { + return fmt.Sprintf("%s\x00%s", operation, path) +} + +type mockBulkOperation struct { + client *MockClient + stateMountPath string + operation string + requests []*beeremote.JobRequest +} + +func (x *mockBulkOperation) Close(ctx context.Context) error { + return nil +} + +func (m *mockBulkOperation) AddRequest(ctx context.Context, request *beeremote.JobRequest) error { + m.requests = append(m.requests, proto.Clone(request).(*beeremote.JobRequest)) + return nil +} + +func (m *mockBulkOperation) Execute(ctx context.Context) (<-chan *BulkStreamPathResult, BulkExecuteResultFn, error) { + walkCh := make(chan *BulkStreamPathResult, len(m.requests)) + for _, request := range m.requests { + path := getMockBulkReplayPath(request) + m.client.markBulkPathCompleted(m.operation, path) + walkCh <- &BulkStreamPathResult{Path: path} + } + close(walkCh) + return walkCh, func() *SchedulingResult { return &SchedulingResult{} }, nil +} + +func (m *mockBulkOperation) Cancel(ctx context.Context, reason error) (<-chan *BulkStreamPathResult, BulkCancelResultFn, error) { + walkCh := make(chan *BulkStreamPathResult) + close(walkCh) + return walkCh, func() error { return nil }, nil +} + +func (m *mockBulkOperation) Destroy(ctx context.Context) error { + return nil +} + +func getMockRequestLockedInfo(request *beeremote.JobRequest) *flex.JobLockedInfo { + switch request.WhichType() { + case beeremote.JobRequest_Sync_case: + return request.GetSync().GetLockedInfo() + case beeremote.JobRequest_Mock_case: + if request.GetMock().GetLockedInfo() != nil { + return request.GetMock().GetLockedInfo() + } + if request.GetMock().GetCfg() != nil { + return request.GetMock().GetCfg().GetLockedInfo() + } + } + return nil +} + +func getMockBulkReplayPath(request *beeremote.JobRequest) string { + switch request.WhichType() { + case beeremote.JobRequest_Sync_case: + if request.GetSync().GetOperation() == flex.SyncJob_DOWNLOAD && request.GetSync().GetRemotePath() != "" { + return request.GetSync().GetRemotePath() + } + case beeremote.JobRequest_Mock_case: + cfg := request.GetMock().GetCfg() + if cfg != nil { + if cfg.GetDownload() && cfg.GetRemotePath() != "" { + return cfg.GetRemotePath() + } + if cfg.GetPath() != "" { + return cfg.GetPath() + } + } + } + return request.GetPath() +} diff --git a/common/rst/mock_test.go b/common/rst/mock_test.go new file mode 100644 index 00000000..48fdfe03 --- /dev/null +++ b/common/rst/mock_test.go @@ -0,0 +1,110 @@ +package rst + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/thinkparq/protobuf/go/beeremote" + "github.com/thinkparq/protobuf/go/flex" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestMockClientGetJobRequestBuildsExecutableMockRequest(t *testing.T) { + client := &MockClient{} + cfg := &flex.JobRequestCfg{ + Path: "/mnt/file", + RemotePath: "remote/file", + RemoteStorageTarget: 7, + Download: true, + LockedInfo: &flex.JobLockedInfo{ + ExternalId: "restore-123", + RemoteSize: 4096, + }, + } + + request := client.GetJobRequest(cfg) + require.NotNil(t, request) + require.True(t, request.HasMock()) + assert.Equal(t, "/mnt/file", request.GetPath()) + assert.Equal(t, uint32(7), request.GetRemoteStorageTarget()) + assert.Equal(t, int32(1), request.GetMock().GetNumTestSegments()) + assert.Equal(t, int64(4096), request.GetMock().GetFileSize()) + assert.Equal(t, "restore-123", request.GetMock().GetExternalId()) + require.NotNil(t, request.GetMock().GetCfg()) + assert.Equal(t, "remote/file", request.GetMock().GetCfg().GetRemotePath()) + + job := &beeremote.Job{ + Id: "job-1", + ExternalId: "ext-1", + Request: request, + } + workRequests, err := client.GenerateWorkRequests(context.Background(), nil, job, 1) + require.NoError(t, err) + require.Len(t, workRequests, 1) + assert.True(t, workRequests[0].HasMock()) + assert.Equal(t, int64(4096), workRequests[0].GetMock().GetFileSize()) +} + +func TestMockClientBulkOperationReplaysArchivedRequestsOnce(t *testing.T) { + client := &MockClient{} + cfg := &flex.JobRequestCfg{ + Path: "/mnt/file", + RemotePath: "remote/file", + RemoteStorageTarget: 7, + Download: true, + LockedInfo: &flex.JobLockedInfo{ + IsArchived: true, + }, + } + + request := client.GetJobRequest(cfg) + include, operation := client.IncludeRequestInBulkOperation(context.Background(), request) + require.True(t, include) + assert.Equal(t, "retrieve", operation) + + bulkOp, err := client.OpenBulkOperation(context.Background(), ".beegfs-rst/job/job-1/7", operation) + require.NoError(t, err) + require.NoError(t, bulkOp.AddRequest(context.Background(), request)) + + walkCh, getResults, err := bulkOp.Execute(context.Background()) + require.NoError(t, err) + + var replayPaths []string + for walkResp := range walkCh { + require.NoError(t, walkResp.Err) + replayPaths = append(replayPaths, walkResp.Path) + } + assert.Equal(t, []string{"remote/file"}, replayPaths) + + result := getResults() + require.NoError(t, result.Err) + assert.False(t, result.Reschedule) + assert.Equal(t, time.Duration(0), result.Delay) + + replayedRequest := client.GetJobRequest(cfg) + include, operation = client.IncludeRequestInBulkOperation(context.Background(), replayedRequest) + assert.False(t, include) + assert.Empty(t, operation) +} + +func TestMockClientGetRemotePathInfoUsesLockedInfo(t *testing.T) { + client := &MockClient{} + expectedMtime := time.Date(2026, 5, 22, 12, 0, 0, 0, time.UTC) + cfg := &flex.JobRequestCfg{ + LockedInfo: &flex.JobLockedInfo{ + RemoteSize: 8192, + RemoteMtime: timestamppb.New(expectedMtime), + IsArchived: true, + }, + } + + remoteSize, remoteMtime, isArchived, allowRestore, err := client.GetRemotePathInfo(context.Background(), cfg) + require.NoError(t, err) + assert.Equal(t, int64(8192), remoteSize) + assert.Equal(t, expectedMtime, remoteMtime) + assert.True(t, isArchived) + assert.True(t, allowRestore) +} diff --git a/common/rst/rst.go b/common/rst/rst.go index a0f12ad8..f991bf04 100644 --- a/common/rst/rst.go +++ b/common/rst/rst.go @@ -41,6 +41,45 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) +var ( + // ErrBuilderFailed marks a builder-level termination that must leave the work request in the + // FAILED state. Use it when the builder can no longer continue and the builder/provider state + // may require cleanup or manual attention. + ErrBuilderFailed = errors.New("builder failed") + // ErrBuilderCancelled marks a builder-level termination that must leave the work request in + // the CANCELLED state. Use it when the builder can no longer continue, but it has not entered + // a failed/invalid state that requires failed-job cleanup semantics. + ErrBuilderCancelled = errors.New("builder cancelled") +) + +func MarkBuilderFailed(errs ...error) error { + return markBuilderWithSentinel(ErrBuilderFailed, errs...) +} + +func MarkBuilderCancelled(errs ...error) error { + return markBuilderWithSentinel(ErrBuilderCancelled, errs...) +} + +func markBuilderWithSentinel(sentinel error, errs ...error) (err error) { + for _, nextErr := range errs { + if nextErr == nil { + continue + } + if err == nil { + err = nextErr + } else { + err = fmt.Errorf("%w; %w", err, nextErr) + } + } + + if err == nil { + return sentinel + } else if errors.Is(err, sentinel) { + return err + } + return fmt.Errorf("%w: %w", sentinel, err) +} + // SupportedRSTTypes is used with SetRSTTypeHook in the config package to allows configuring with // multiple RST types without writing repetitive code. The map contains the all lowercase string // identifier of the prefix key of the TOML table used to indicate the configuration options for a @@ -52,6 +91,11 @@ import ( // initialized but empty struct of the correct type. var SupportedRSTTypes = map[string]func() (any, any){ "s3": func() (any, any) { t := new(flex.RemoteStorageTarget_S3_); return t, &t.S3 }, + // XtreemStore is S3-compatible and uses the existing S3 implementation. + "xtreemstore": func() (any, any) { + t := &flex.RemoteStorageTarget_Xtreemstore{Xtreemstore: &flex.RemoteStorageTarget_XtreemStore{}} + return t, &t.Xtreemstore.S3 + }, // Azure is not currently supported, but this is how an Azure type could be added: // "azure": func() (any, any) { t := new(flex.RemoteStorageTarget_Azure_); return t, &t.Azure }, // Mock could be included here if it ever made sense to allow configuration using a file. @@ -68,10 +112,32 @@ type Provider interface { // job.StartMtime should be set. GenerateWorkRequests(ctx context.Context, lastJob *beeremote.Job, job *beeremote.Job, availableWorkers int) (requests []*flex.WorkRequest, err error) // ExecuteJobBuilderRequest is for providers that need to submit additional job requests. Stream - // any new requests into jobSubmissionChan. If building jobs is long running, return - // rescheduled==true to reschedule the remaining work for later which allows other work time to - // complete. - ExecuteJobBuilderRequest(ctx context.Context, workRequest *flex.WorkRequest, jobSubmissionChan chan<- *beeremote.JobRequest) (reschedule bool, err error) + // any new requests into jobSubmissionCh. Set SchedulingResult.Reschedule when there's more work + // (e.g. the walk was cut off by its per round batch limit or a bulk operation isn't done yet). + // Use SchedulingResult.Delay to back off before the next round. + // + // Builder reporting is split across three layers: + // + // - Individual job request outcomes should be reported on the generated JobRequest via + // GenerationStatus whenever the builder can continue generating more requests. + // - Builder progress should be persisted on the builder itself via any resume token stored in + // workRequest.ExternalId, so a later call can pick up where this one left off. + // - Builder termination must be reported through SchedulingResult.Err when the builder can no + // longer safely or usefully continue generating additional requests. + // + // A non-nil SchedulingResult.Err means the builder execution is over. It must be classified + // with one of the builder sentinels: + // + // - ErrBuilderCancelled means the builder must stop early because continued submissions are + // likely to fail or are otherwise unsafe, but the builder/provider state is not known to be + // failed or invalid in a way that requires failed-job cleanup semantics. + // - ErrBuilderFailed means the builder must stop and the builder/provider state must be + // treated as failed. Use it when cleanup may be required or state is invalid, + // inconsistent, incomplete, or otherwise requires manual attention. + // + // Unclassified errors are treated as failed by callers. + // + ExecuteJobBuilderRequest(ctx context.Context, workRequest *flex.WorkRequest, jobSubmissionCh chan<- *beeremote.JobRequest, workerSaturation []func() float64) *SchedulingResult // ExecuteWorkRequestPart accepts a request and which part of the request it should carry out. // It blocks until the request is complete, but the caller can cancel the provided context to // return early. It determines and executes the requested operation (if supported) then directly @@ -114,6 +180,67 @@ type Provider interface { // start work requests that have been placed into a wait queue. This is useful for providers // that need the ability to wait for resources to be made available before continuing. IsWorkRequestReady(ctx context.Context, request *flex.WorkRequest) (ready bool, delay time.Duration, err error) + // IncludeRequestInBulkOperation indicates whether the request should be included in a provider-defined + // bulk operation. operation is an arbitrary provider-defined identifier that groups compatible + // requests within provider bulk request. + IncludeRequestInBulkOperation(ctx context.Context, request *beeremote.JobRequest) (include bool, operation string) + // OpenBulkOperation opens or creates the provider-defined bulk operation identified by + // stateMountPath, operation, and the provider itself, and returns a handle that manages that + // operation for the current builder execution. + // + // The builder calls this once per tracked bulk operation, including when resuming a builder job + // that already persisted metadata from an earlier execution. Implementations should therefore + // recover any provider-side state needed to continue appending requests, executing, or + // cancelling the operation. + // + // stateMountPath is reserved for provider state that must survive builder reschedules or + // retries. Return an error only when the bulk operation cannot be opened in a usable state. + OpenBulkOperation(ctx context.Context, stateMountPath string, operation string) (clientBulkOperation, error) +} + +type SchedulingResult struct { + Reschedule bool + Delay time.Duration + Err error +} + +type BulkExecuteResultFn func() *SchedulingResult +type BulkExecuteFn func(ctx context.Context) (walkCh <-chan *BulkStreamPathResult, getResults BulkExecuteResultFn, err error) +type BulkCancelResultFn func() error +type BulkCancelFn func(ctx context.Context, reason error) (walkCh <-chan *BulkStreamPathResult, getResults BulkCancelResultFn, err error) +type clientBulkOperation interface { + // AddRequest adds a single request to the bulk operation state. Calls are serialized by the + // caller. The implementation owns request.BulkInfo.JobIndex: it must assign a JobIndex based on + // its own persisted state (not on any value already set on the request) so the index stays + // correct across builder reschedules that reopen the same bulk operation. Return an error only + // for failures that should stop the parent builder job. + AddRequest(ctx context.Context, request *beeremote.JobRequest) error + // Execute starts a bulk operation for the currently accumulated requests. The returned + // getResults function must not return until walkCh has been closed, and it returns the + // reschedule details and any errors that occurred. err should only be returned when the builder + // job itself should fail. All other errors should be reported on walkCh with the relevant path + // so the request can reflect the failure. + // + // + // Any paths that are ready may be sent to walkCh immediately so their requests can be + // submitted. + Execute(ctx context.Context) (walkCh <-chan *BulkStreamPathResult, getResults BulkExecuteResultFn, err error) + // Cancel stops the bulk operation and sends any unsent paths along with reason error to walkCh. + // Any bulk operation specific errors should be reported from the returned wait function, which + // must not return until walkCh has been closed. + // + // When failed builder job are cancelled, walkCh paths will be discarded which is consistent + // with normal builder job behavior. So it is the responsibility of the provider to cancel the + // bulk operation and handle any cleanup. If any manual cleanup is require, the user must be + // notified. + Cancel(ctx context.Context, reason error) (walkCh <-chan *BulkStreamPathResult, wait BulkCancelResultFn, err error) + // Close releases any resources that were opened. + Close(ctx context.Context) error + // Destroy permanently removes this bulk operation's on-disk state. It must only be called once the + // operation will never be reopened again (e.g. when the builder job that owns it is being torn down + // for good), since AddRequest, Execute, and Cancel all assume these files exist for as long as the + // operation is live. + Destroy(ctx context.Context) error } // New initializes a provider client based on the provided config. It accepts a context that can be @@ -128,6 +255,8 @@ func New(ctx context.Context, config *flex.RemoteStorageTarget, mountPoint files switch config.Type.(type) { case *flex.RemoteStorageTarget_S3_: return newS3(ctx, config, mountPoint) + case *flex.RemoteStorageTarget_Xtreemstore: + return newXtreemstore(ctx, config, mountPoint) case *flex.RemoteStorageTarget_Mock: // This handles setting up a Mock RST for testing from external packages like WorkerMgr. See // the documentation ion `MockClient` in mock.go for how to setup expectations. @@ -202,6 +331,10 @@ func RecreateWorkRequests(job *beeremote.Job, segments []*flex.WorkRequest_Segme Priority: new(request.GetPriority()), } + if request.HasBulkInfo() { + wr.BulkInfo = proto.Clone(request.GetBulkInfo()).(*flex.BulkJobRequestInfo) + } + switch request.WhichType() { case beeremote.JobRequest_Sync_case: wr.Type = &flex.WorkRequest_Sync{ @@ -247,98 +380,75 @@ func generateSegments(fileSize int64, segCount int64, partsPerSegment int32) []* return segments } -// BuildJobRequests returns a list of job requests, one for each remote target. Unless -// skipPrepareJob=true then remote resource information will be added to the request's lockedInfo -// and common checks and tasks will be preformed. -// -// A returned error indicates that one or more job request were not able to be built. However, if -// a request was able to be built, the error will be specified in the request's GenerationStatus. -func BuildJobRequests(ctx context.Context, rstMap map[uint32]Provider, mountPoint filesystem.Provider, inMountPath string, remotePath string, cfg *flex.JobRequestCfg) ([]*beeremote.JobRequest, error) { - keepLock := false - lockedInfo, writeLockSet, rstIds, currentRSTCfg, entryInfoMsg, ownerNode, err := GetLockedInfo(ctx, mountPoint, cfg, inMountPath, false) - - defer func() { - if !keepLock && writeLockSet { - if clearWriteLockErr := entry.ClearAccessFlags(ctx, inMountPath, beegfs.LockedContentAccessFlags); clearWriteLockErr != nil { - err = errors.Join(err, fmt.Errorf("unable to write lock: %w", clearWriteLockErr)) - } +// BuildJobRequestWithFailedPrecondition returns a job request with failed precondition +// GenerationStatus with the specified message. +func BuildJobRequestWithFailedPrecondition(client Provider, cfg *flex.JobRequestCfg, message string) *beeremote.JobRequest { + request := client.GetJobRequest(cfg) + status := &beeremote.JobRequest_GenerationStatus{ + State: beeremote.JobRequest_GenerationStatus_FAILED_PRECONDITION, + Message: message, + } + request.SetGenerationStatus(status) + return request +} + +// BuildJobRequest creates a provider-specific job request for cfg if the request is valid; +// otherwise, the request will be returned with a failed precondition status for the issue. +func BuildJobRequest(ctx context.Context, client Provider, cfg *flex.JobRequestCfg) *beeremote.JobRequest { + lockedInfo := cfg.GetLockedInfo() + if !IsFileLocked(lockedInfo) && FileExists(lockedInfo) { + return BuildJobRequestWithFailedPrecondition(client, cfg, "path lock has not been acquired") + } + + cfg.SetRemotePath(client.SanitizeRemotePath(cfg.RemotePath)) + if IsFileOffloaded(lockedInfo) { + // Use rst url from the stub file when a remote-path wasn't provided. + if cfg.RemotePath == "" { + cfg.SetRemotePath(client.SanitizeRemotePath(lockedInfo.StubUrlPath)) + } else if !cfg.Overwrite && cfg.RemotePath != lockedInfo.StubUrlPath { + return BuildJobRequestWithFailedPrecondition(client, cfg, "unexpected stub file path") } - }() - if err != nil { - // If the user didn't specify any RSTs and the entry doesn't have any RSTs configured, just - // silently ignore it. Otherwise pushing a subset of files based on their configured RST IDs - // would always fail, whenever there is a file with no RSTs set on its entry info. - if errors.Is(err, ErrFileHasNoRSTs) { - return nil, nil - } - // If this function returns an error but it will also abort the entire builder job, which we - // generally want to avoid outside fatal errors. Outside fatal errors, if there are any RST - // IDs available for this inMountPath (either specified by the user, or determined - // automatically), then report any errors as part of the generated requests for each file. - // For non-fatal errors on paths that have no RSTs we must just return the error anyway to - // avoid it being silently dropped. - if errors.Is(err, ErrGetLockedInfoFatal) || len(rstIds) == 0 { - return nil, err - } - } else if len(rstIds) > 1 && (cfg.Download || cfg.StubLocal) { - err = errors.Join(err, ErrFileHasAmbiguousRSTs) - } - - var errs []error - var requests []*beeremote.JobRequest - for _, rstId := range rstIds { - client, ok := rstMap[rstId] - if !ok { - errs = append(errs, errors.Join(err, fmt.Errorf("%w: rstId %d", ErrConfigRSTTypeIsUnknown, rstId))) - continue + if !cfg.Overwrite && cfg.RemoteStorageTarget != lockedInfo.StubUrlRstId { + return BuildJobRequestWithFailedPrecondition(client, cfg, "unexpected stub file rst id") } + } - requestCfg := proto.Clone(cfg).(*flex.JobRequestCfg) - requestCfg.SetPath(inMountPath) - requestCfg.SetRemotePath(client.SanitizeRemotePath(remotePath)) - requestCfg.SetRemoteStorageTarget(rstId) - requestLockedInfo := proto.Clone(lockedInfo).(*flex.JobLockedInfo) - requestCfg.SetLockedInfo(requestLockedInfo) + if cfg.Download && cfg.RemotePath == "" { + if !FileExists(lockedInfo) { + return BuildJobRequestWithFailedPrecondition(client, cfg, fmt.Sprintf("unable to determine remote path: %s", fs.ErrNotExist.Error())) + } - if err != nil { - request := client.GetJobRequest(requestCfg) - status := &beeremote.JobRequest_GenerationStatus{ - State: beeremote.JobRequest_GenerationStatus_FAILED_PRECONDITION, - Message: fmt.Sprintf("failed to build job request: %s", err.Error()), + // Attempt to retrieve remote path from a previously completed job request. + if lastJob, err := GetLastCompletedJobFromRst(ctx, cfg.Path, cfg.RemoteStorageTarget); err != nil { + return BuildJobRequestWithFailedPrecondition(client, cfg, fmt.Sprintf("failed to determine last completed job request to determine remote path: %s", err.Error())) + } else if lastJob != nil { + switch lastJob.Request.WhichType() { + case beeremote.JobRequest_Sync_case: + cfg.SetRemotePath(client.SanitizeRemotePath(lastJob.Request.GetSync().RemotePath)) + default: + return BuildJobRequestWithFailedPrecondition(client, cfg, fmt.Sprintf("unable to determine remote path: %s", ErrConfigRSTTypeIsUnknown.Error())) } - request.SetGenerationStatus(status) - requests = append(requests, request) - continue } + } - request := BuildJobRequest(ctx, client, mountPoint, requestCfg) - if request.GetGenerationStatus() == nil { - if err = PrepareFileStateForWorkRequests(ctx, client, mountPoint, currentRSTCfg, entryInfoMsg, ownerNode, requestCfg); err != nil { - if errors.Is(err, ErrJobAlreadyComplete) { - request.GenerationStatus = &beeremote.JobRequest_GenerationStatus{ - State: beeremote.JobRequest_GenerationStatus_ALREADY_COMPLETE, - Message: lockedInfo.Mtime.AsTime().Format(time.RFC3339), - } - } else if errors.Is(err, ErrJobAlreadyOffloaded) { - keepLock = true - request.GenerationStatus = &beeremote.JobRequest_GenerationStatus{State: beeremote.JobRequest_GenerationStatus_ALREADY_OFFLOADED} - } else { - request.SetGenerationStatus(&beeremote.JobRequest_GenerationStatus{ - State: beeremote.JobRequest_GenerationStatus_FAILED_PRECONDITION, - Message: fmt.Sprintf("failed to prepare file state: %s", err.Error()), - }) - } - } else { - // This request will execute so ensure the lock is kept. - keepLock = true - } - } // If we couldn't build a runnable job request, there would be no active job to drive the normal unlock path so don't keep the lock. + remoteSize, remoteMtime, isArchived, isArchiveRestoreAllowed, err := client.GetRemotePathInfo(ctx, cfg) + if err != nil && (cfg.Download || !errors.Is(err, os.ErrNotExist)) { + return BuildJobRequestWithFailedPrecondition(client, cfg, fmt.Sprintf("unable to retrieve remote path information: %s", err.Error())) + } + if cfg.Download && isArchived && !isArchiveRestoreAllowed { + return BuildJobRequestWithFailedPrecondition(client, cfg, fmt.Sprintf("remote object is archived and restore is not permitted; rerun with --%s to continue", AllowRestoreFlag)) + } - requests = append(requests, request) + // Only update remote information when the object exists so lockedInfo.RemoteMtime is nil when + // the remote object does not exist. + if !errors.Is(err, os.ErrNotExist) { + lockedInfo.SetRemoteSize(remoteSize) + lockedInfo.SetRemoteMtime(timestamppb.New(remoteMtime)) + lockedInfo.SetIsArchived(isArchived) } - return requests, errors.Join(errs...) + return client.GetJobRequest(cfg) } // IsFileLocked returns whether the file has acquired a lock. @@ -356,12 +466,6 @@ func IsFileAlreadySynced(lockedInfo *flex.JobLockedInfo) bool { return lockedInfo.Size == lockedInfo.RemoteSize && lockedInfo.Mtime.AsTime().Equal(lockedInfo.RemoteMtime.AsTime()) } -// IsFileSizeMatched returns whether the lockedInfo local and remote file sizes match. It is the -// responsibility of the caller to ensure lockedInfo is already populated and locked. -func IsFileSizeMatched(lockedInfo *flex.JobLockedInfo) bool { - return lockedInfo.Size == lockedInfo.RemoteSize -} - // IsFileOffloaded returns whether the file is offloaded. It is the responsibility of the caller to // ensure lockedInfo is already populated and locked. func IsFileOffloaded(lockedInfo *flex.JobLockedInfo) bool { @@ -375,357 +479,493 @@ func IsFileOffloadedUrlCorrect(rstId uint32, remotePath string, lockedInfo *flex return rstId == lockedInfo.StubUrlRstId && remotePath == lockedInfo.StubUrlPath } -// HasRemotePathInfo indicates whether lockedInfo has been updated with the remote path information. -func HasRemotePathInfo(lockedInfo *flex.JobLockedInfo) bool { - return lockedInfo.RemoteMtime != nil && !lockedInfo.RemoteMtime.AsTime().IsZero() -} - -// BuildJobRequest adds remote resource information to lockedInfo, performs common checks and -// operations, and then returns the job request. It is the responsibility of the caller to ensure -// lockedInfo is already locked. +// PlanFileStateForWorkRequests handles preflight checks and common tasks based on collected +// lockedInfo. // -// Be aware that cfg's remote information will be updated. -func BuildJobRequest(ctx context.Context, client Provider, mountPoint filesystem.Provider, cfg *flex.JobRequestCfg) *beeremote.JobRequest { - getRequestWithFailedPrecondition := func(message string) *beeremote.JobRequest { - request := client.GetJobRequest(cfg) - status := &beeremote.JobRequest_GenerationStatus{ - State: beeremote.JobRequest_GenerationStatus_FAILED_PRECONDITION, - Message: message, - } - request.SetGenerationStatus(status) - return request - } +// failedPrecondition reports a problem preparing the plan itself, such as an invalid configuration +// or an unrecoverable precondition failure such as a download that would overwrite an existing +// path without --overwrite. When it is non-nil, the returned apply function must not be invoked. +// +// When failedPrecondition is nil, callers should invoke the returned apply function to determine +// the outcome. Its own returned error may be a terminal sentinel when the file is already in the +// expected synced or offloaded state, checked using IsErrJobTerminalSentinel. apply performs the +// planned changes, and its own returned undo function best-effort rolls back any reversible local +// changes when later request generation steps fail after preparation succeeds. +// +// It is the responsibility of the caller to ensure lockedInfo is populated when the file exists. If +// the file does not exist, it will be created and cfg.LockedInfo will be updated. +// +// Be aware that the apply function takes a *PathState argument so it can be updated when the file +// is created. +func PlanFileStateForWorkRequests(ctx context.Context, mountPoint filesystem.Provider, cfg *flex.JobRequestCfg) (apply applyPlanFn, failedPrecondition error) { + addStep, apply := newApplyPlan() + defer addStep(prepareUpdateFileRstPattern(ctx, cfg)) lockedInfo := cfg.LockedInfo - if !IsFileLocked(lockedInfo) && FileExists(lockedInfo) { - return getRequestWithFailedPrecondition("path lock has not been acquired") + originalLockedInfo := proto.Clone(lockedInfo).(*flex.JobLockedInfo) + alreadySynced := IsFileAlreadySynced(lockedInfo) + if cfg.StubLocal { + if (cfg.Download && (cfg.Overwrite || !FileExists(lockedInfo))) || alreadySynced { + addStep(prepareStubLocalOffload(ctx, mountPoint, cfg, alreadySynced)) + return + } - } + if IsFileOffloaded(lockedInfo) { + if !IsFileOffloadedUrlCorrect(cfg.RemoteStorageTarget, cfg.RemotePath, lockedInfo) { + failedPrecondition = ErrOffloadFileUrlMismatch + return + } - cfg.SetRemotePath(client.SanitizeRemotePath(cfg.RemotePath)) - if IsFileOffloaded(lockedInfo) { - // Use rst url from the stub file when a remote-path wasn't provided. - if cfg.RemotePath == "" { - cfg.SetRemotePath(client.SanitizeRemotePath(lockedInfo.StubUrlPath)) - } else if !cfg.Overwrite && cfg.RemotePath != lockedInfo.StubUrlPath { - return getRequestWithFailedPrecondition("unexpected stub file path") - } - if !cfg.Overwrite && cfg.RemoteStorageTarget != lockedInfo.StubUrlRstId { - return getRequestWithFailedPrecondition("unexpected stub file rst id") + addStep(prepareAlreadyOffloaded(ctx, cfg)) + return } - } - if cfg.Download && cfg.RemotePath == "" { - if !FileExists(lockedInfo) { - return getRequestWithFailedPrecondition(fmt.Sprintf("unable to determine remote path: %s", fs.ErrNotExist.Error())) + if cfg.Download && !cfg.Overwrite && FileExists(lockedInfo) { + failedPrecondition = fmt.Errorf("download would overwrite existing path but the overwrite flag was not set: %w", fs.ErrExist) + return + } + } else if FileExists(lockedInfo) { + if alreadySynced { + addStep(prepareAlreadyComplete(cfg)) + return } - // Attempt to retrieve remote path from a previously completed job request. - if lastJob, err := GetLastCompletedJobFromRst(ctx, cfg.Path, cfg.RemoteStorageTarget); err != nil { - return getRequestWithFailedPrecondition(fmt.Sprintf("failed to determine last completed job request to determine remote path: %s", err.Error())) - } else if lastJob != nil { - switch lastJob.Request.WhichType() { - case beeremote.JobRequest_Sync_case: - cfg.SetRemotePath(client.SanitizeRemotePath(lastJob.Request.GetSync().RemotePath)) - default: - return getRequestWithFailedPrecondition(fmt.Sprintf("unable to determine remote path: %s", ErrConfigRSTTypeIsUnknown.Error())) + if cfg.Download { + allowOverwrite := cfg.Overwrite + if IsFileOffloaded(lockedInfo) { + if !allowOverwrite && !IsFileOffloadedUrlCorrect(cfg.RemoteStorageTarget, cfg.RemotePath, lockedInfo) { + failedPrecondition = ErrOffloadFileUrlMismatch + return + } + + addStep(prepareDownloadRestoreDataState(ctx, cfg)) + allowOverwrite = true } - } - } - remoteSize, remoteMtime, isArchived, isArchiveRestoreAllowed, err := client.GetRemotePathInfo(ctx, cfg) - if err != nil && (cfg.Download || !errors.Is(err, os.ErrNotExist)) { - return getRequestWithFailedPrecondition(fmt.Sprintf("unable to retrieve remote path information: %s", err.Error())) - } - if cfg.Download && isArchived && !isArchiveRestoreAllowed { - return getRequestWithFailedPrecondition(fmt.Sprintf("remote object is archived and restore is not permitted; rerun with --%s to continue", AllowRestoreFlag)) + if !allowOverwrite { + failedPrecondition = fmt.Errorf("download would overwrite existing path but the overwrite flag was not set: %w", fs.ErrExist) + return + } + + // Expand the file size if needed. + if lockedInfo.Size < lockedInfo.RemoteSize { + addStep(prepareDownloadExpandFile(mountPoint, cfg, allowOverwrite, originalLockedInfo)) + } + } else if IsFileOffloaded(lockedInfo) { + failedPrecondition = fmt.Errorf("unable to upload stub file: %w", ErrUnsupportedOpForRST) + return + } + } else if cfg.Download { + addStep(prepareDownloadNoFile(ctx, mountPoint, cfg)) + } else { + failedPrecondition = fmt.Errorf("unable to upload file: %w", fs.ErrNotExist) + return } - lockedInfo.SetRemoteSize(remoteSize) - lockedInfo.SetRemoteMtime(timestamppb.New(remoteMtime)) - lockedInfo.SetIsArchived(isArchived) - return client.GetJobRequest(cfg) + return } -func updateFileRstPattern(ctx context.Context, cfg *flex.JobRequestCfg, path string, currentRSTCfg msg.RemoteStorageTarget, entryInfoMsg msg.EntryInfo, ownerNode beegfs.Node) error { - var rstIds []uint32 - if cfg.GetUpdate() { - if !IsValidRstId(cfg.RemoteStorageTarget) { - return fmt.Errorf("--%s requires a valid --%s to be specified", UpdateFlag, RemoteTargetFlag) - } - rstIds = []uint32{cfg.RemoteStorageTarget} +type undoFn func() error +type applyPlanFn func(*PathState) (undoFn, error) +type applyFn func(pathState *PathState, appliedErr error) (undoFn, error) + +var noopUndo = func() error { return nil } + +func newApplyPlan() (add func(applyFn), apply applyPlanFn) { + applySteps := []applyFn{} + add = func(step applyFn) { + applySteps = append(applySteps, step) } - var cooldownSecs *uint16 - if cfg.HasCooldownSecs() { - v := uint16(math.MaxUint16) - if cfg.GetCooldownSecs() <= math.MaxUint16 { - v = uint16(cfg.GetCooldownSecs()) + apply = func(pathState *PathState) (undoFn, error) { + undoSteps := []undoFn{} + undo := func() (undoErr error) { + for i := len(undoSteps) - 1; i >= 0; i-- { + undoErr = errors.Join(undoErr, undoSteps[i]()) + } + return } - cooldownSecs = &v - } - if err := entry.SetFileRstPattern(ctx, path, rstIds, cooldownSecs, currentRSTCfg, entryInfoMsg, ownerNode); err != nil { - return fmt.Errorf("failed to apply RST configuration: %w", err) + + var undoStep undoFn + var applyErr error + for _, applyStep := range applySteps { + undoStep, applyErr = applyStep(pathState, applyErr) + undoSteps = append(undoSteps, undoStep) + } + + if applyErr != nil && !IsErrJobTerminalSentinel(applyErr) { + if undoErr := undo(); undoErr != nil { + applyErr = fmt.Errorf("%w: failed to rollback changes: %w", applyErr, undoErr) + } else { + applyErr = fmt.Errorf("%w: %w", applyErr, ErrJobFailedPrecondition) + } + return noopUndo, applyErr + } + return undo, applyErr + } - return nil + return } -// PrepareFileStateForWorkRequests handles preflight checks and common tasks based on collected -// lockedInfo. Sentinel errors are returned when the file is already in the expected synced or -// offloaded state. Sentinel errors can be checked using IsErrJobTerminalSentinel. -// -// It is the responsibility of the caller to ensure lockedInfo is populated when the file exists. If -// the file does not exist, it will be created and cfg.LockedInfo will be updated. -func PrepareFileStateForWorkRequests(ctx context.Context, client Provider, mountPoint filesystem.Provider, currentRSTCfg msg.RemoteStorageTarget, entryInfo msg.EntryInfo, ownerNode beegfs.Node, cfg *flex.JobRequestCfg) (err error) { +func prepareAlreadyComplete(cfg *flex.JobRequestCfg) applyFn { lockedInfo := cfg.LockedInfo + return func(pathState *PathState, appliedErr error) (undoFn, error) { + return noopUndo, GetErrJobAlreadyCompleteWithMtime(lockedInfo.Mtime.AsTime()) + } +} - fileDataStateCleared := false - // originalDataState is fetched below before fileDataStateCleared is set, and errors are now - // fatal. We still have to pick a default value, use DataStateManualRestore as it is the safest - // choice in case a future bug causes us to not fetch the actual originalDataState. - originalDataState := beegfs.DataState(beegfs.DataStateManualRestore) - filePreallocated := false - fileCreated := false - defer func() { - if err != nil { - if IsFileOffloaded(lockedInfo) { - if fileDataStateCleared { - if restoreErr := entry.SetFileDataState(ctx, cfg.Path, originalDataState); restoreErr != nil { - err = fmt.Errorf("%w: unable to restore offloaded data state: %s", err, restoreErr.Error()) - } - } - if filePreallocated { - // Roll back stub file if there's a failure after the contents have been changed - rstUrl := fmt.Appendf(nil, "rst://%d:%s\n", lockedInfo.StubUrlRstId, lockedInfo.StubUrlPath) - if restoreErr := mountPoint.CreateWriteClose(cfg.Path, rstUrl, 0644, true); restoreErr != nil { - err = fmt.Errorf("%w: unable to restore stub file rst url: %s", err, restoreErr.Error()) - } - } - } else if fileCreated { - // Remove preallocated file since it previously did not exist. - if restoreErr := mountPoint.Remove(cfg.Path); restoreErr != nil { - err = fmt.Errorf("%w: unable to remove preallocated file: %s", err, restoreErr.Error()) - } +func prepareAlreadyOffloaded(ctx context.Context, cfg *flex.JobRequestCfg) applyFn { + return func(pathState *PathState, appliedErr error) (undoFn, error) { + undo := noopUndo + if cfg.HasRestorePolicy() { + state := restorePolicyToDataState(cfg.GetRestorePolicy()) + ownerNode := pathState.OwnerNode + entryInfoMsg := pathState.EntryInfo.GetOrigEntryInfo() + if entryInfoMsg == nil { + return undo, fmt.Errorf("original entry info unavailable") } - } - }() - updateRstCfg := func(sentinel error) error { - if cfg.GetUpdate() || cfg.HasCooldownSecs() { - if err := updateFileRstPattern(ctx, cfg, cfg.Path, currentRSTCfg, entryInfo, ownerNode); err != nil { - if sentinel != nil { - return fmt.Errorf("%w but unable to update RST configuration: %w", sentinel, err) + originalDataState, dataStateErr := entry.GetFileDataStateWithEntryInfo(ctx, cfg.Path, *entryInfoMsg, ownerNode) + if dataStateErr != nil { + return undo, fmt.Errorf("unable to determine original file data state: %w", dataStateErr) + } + + if originalDataState != state { + if err := entry.SetFileDataStateWithEntryInfo(ctx, cfg.Path, state, *entryInfoMsg, ownerNode); err != nil { + return undo, fmt.Errorf("unable to set restore policy: %w", err) + } + + undo = func() error { + return entry.SetFileDataStateWithEntryInfo(ctx, cfg.Path, originalDataState, *entryInfoMsg, ownerNode) } - return err } } - return sentinel + return undo, ErrJobAlreadyOffloaded } +} - alreadySynced := IsFileAlreadySynced(lockedInfo) - if cfg.StubLocal { - if (cfg.Download && (cfg.Overwrite || !FileExists(lockedInfo))) || alreadySynced { - if err = CreateOffloadedDataFile(ctx, mountPoint, cfg.Path, cfg.RemotePath, cfg.RemoteStorageTarget, cfg.Overwrite || alreadySynced, restorePolicyToDataState(cfg.GetRestorePolicy())); err != nil { - err = fmt.Errorf("failed to create stub file: %w", err) - return +// prepareStubLocalOffload creates the stub file for an entry that is either already synced with +// the remote target or about to be created as a download stub, taking over the file's access lock +// if it didn't already exist. It always terminates the plan with ErrJobAlreadyOffloaded since +// nothing else needs to run after it. +func prepareStubLocalOffload(ctx context.Context, mountPoint filesystem.Provider, cfg *flex.JobRequestCfg, alreadySynced bool) applyFn { + lockedInfo := cfg.LockedInfo + return func(pathState *PathState, appliedErr error) (undoFn, error) { + if appliedErr != nil { + return noopUndo, appliedErr + } + + var undo func() error + restorePolicy := restorePolicyToDataState(cfg.GetRestorePolicy()) + rstUrl := fmt.Appendf(nil, "rst://%d:%s\n", cfg.RemoteStorageTarget, cfg.RemotePath) + + if !FileExists(lockedInfo) { + undo = func() error { + if err := mountPoint.Remove(cfg.Path); err != nil { + return fmt.Errorf("failed to remove stub file: %w", err) + } + return nil } - err = entry.SetAccessFlags(ctx, cfg.Path, beegfs.LockedContentAccessFlags) + + // Overwrites via O_TRUNC, which leaves a narrow window where a crash could zero the file. We + // intentionally keep this over atomic-rename: a new inode drops the BeeGFS per-file metadata + // (RST IDs, locks) and silently breaks stub-then-re-push and `--update --remote-target`. Any + // future fix for the O_TRUNC window must reapply that metadata to the new inode. + err := mountPoint.CreateWriteClose(cfg.Path, rstUrl, 0644, false) if err != nil { - return + if errors.Is(err, fs.ErrExist) { + return noopUndo, fmt.Errorf("unable to create stub file: %w", err) + } + return undo, fmt.Errorf("unable to create stub file: %w", err) } - lockedInfo.SetReadWriteLocked(true) - return updateRstCfg(ErrJobAlreadyOffloaded) - } - if IsFileOffloaded(lockedInfo) { - if !IsFileOffloadedUrlCorrect(cfg.RemoteStorageTarget, cfg.RemotePath, lockedInfo) { - err = ErrOffloadFileUrlMismatch - return + if *pathState, err = GetPathState(ctx, mountPoint, cfg.Path, PathStateWithLock); err != nil { + return undo, fmt.Errorf("failed to collect information for stub file: %w", err) } - if cfg.HasRestorePolicy() { - if err = entry.SetFileDataState(ctx, cfg.Path, restorePolicyToDataState(cfg.GetRestorePolicy())); err != nil { - return + info := pathState.LockedInfo + lockedInfo.SetReadWriteLocked(info.ReadWriteLocked) + lockedInfo.SetExists(info.Exists) + lockedInfo.SetSize(info.Size) + lockedInfo.SetMtime(info.Mtime) + lockedInfo.SetMode(info.Mode) + + ownerNode := pathState.OwnerNode + entryInfoMsg := pathState.EntryInfo.GetOrigEntryInfo() + if err := entry.SetFileDataStateWithEntryInfo(ctx, cfg.Path, restorePolicy, *entryInfoMsg, ownerNode); err != nil { + return undo, fmt.Errorf("unable to set restore policy: %w", err) + } + } else { + undo = func() error { + if stat, statErr := mountPoint.Stat(cfg.Path); statErr == nil { + if stat.Size() != lockedInfo.Size || !stat.ModTime().Equal(lockedInfo.Mtime.AsTime()) { + return fmt.Errorf("failed to restore file") + } } + return nil + } + + overwrite := cfg.Overwrite || alreadySynced + ownerNode := pathState.OwnerNode + entryInfoMsg := pathState.EntryInfo.GetOrigEntryInfo() + // Overwrites via O_TRUNC, which leaves a narrow window where a crash could zero the file. We + // intentionally keep this over atomic-rename: a new inode drops the BeeGFS per-file metadata + // (RST IDs, locks) and silently breaks stub-then-re-push and `--update --remote-target`. Any + // future fix for the O_TRUNC window must reapply that metadata to the new inode. + if err := mountPoint.CreateWriteClose(cfg.Path, rstUrl, 0644, overwrite); err != nil { + return undo, fmt.Errorf("failed to create stub file %q: %w", cfg.Path, err) + } + + if err := entry.SetFileDataStateWithEntryInfo(ctx, cfg.Path, restorePolicy, *entryInfoMsg, ownerNode); err != nil { + return undo, fmt.Errorf("unable to set restore policy: %w", err) } - return updateRstCfg(ErrJobAlreadyOffloaded) } - if cfg.Download && !cfg.Overwrite && FileExists(lockedInfo) { - err = fmt.Errorf("download would overwrite existing path but the overwrite flag was not set: %w", fs.ErrExist) - return + return undo, ErrJobAlreadyOffloaded + } +} + +// prepareDownloadRestoreDataState clears the offloaded data state on an existing stub file so a +// download can overwrite its contents, restoring the original data state if a later step fails. +func prepareDownloadRestoreDataState(ctx context.Context, cfg *flex.JobRequestCfg) applyFn { + return func(pathState *PathState, appliedErr error) (undoFn, error) { + ownerNode := pathState.OwnerNode + entryInfoMsg := pathState.EntryInfo.GetOrigEntryInfo() + if entryInfoMsg == nil { + return noopUndo, errors.New("original entry info unavailable: refusing to proceed with restoring file contents") } - } else if FileExists(lockedInfo) { - if alreadySynced { - return updateRstCfg(GetErrJobAlreadyCompleteWithMtime(lockedInfo.Mtime.AsTime())) + + originalDataState, dataStateErr := entry.GetFileDataStateWithEntryInfo(ctx, cfg.Path, *entryInfoMsg, ownerNode) + if dataStateErr != nil { + return noopUndo, fmt.Errorf("unable to determine original file data state: %w", dataStateErr) } - if cfg.Download { - allowOverwrite := cfg.Overwrite + if appliedErr != nil { + return noopUndo, appliedErr + } + + if err := entry.SetFileDataStateWithEntryInfo(ctx, cfg.Path, beegfs.DataStateAvailable, *entryInfoMsg, ownerNode); err != nil { + return noopUndo, fmt.Errorf("unable to set the data state to available: %w", err) + } + + undo := func() error { + return entry.SetFileDataStateWithEntryInfo(ctx, cfg.Path, originalDataState, *entryInfoMsg, ownerNode) + } + return undo, nil + } +} + +// prepareDownloadExpandFile grows an existing file to match the remote object's size before a +// download overwrites its contents, restoring the original stub or file size if a later step +// fails. +func prepareDownloadExpandFile(mountPoint filesystem.Provider, cfg *flex.JobRequestCfg, allowOverwrite bool, originalLockedInfo *flex.JobLockedInfo) applyFn { + lockedInfo := cfg.LockedInfo + return func(pathState *PathState, appliedErr error) (undoFn, error) { + if appliedErr != nil { + return noopUndo, appliedErr + } + + if err := mountPoint.CreateOrResizeFile(cfg.Path, lockedInfo.RemoteSize, allowOverwrite); err != nil { + return noopUndo, fmt.Errorf("unable to preallocate additional space for file: %w", err) + } + + undo := func() error { if IsFileOffloaded(lockedInfo) { - if !allowOverwrite && !IsFileOffloadedUrlCorrect(cfg.RemoteStorageTarget, cfg.RemotePath, lockedInfo) { - err = ErrOffloadFileUrlMismatch - return - } - // Previously if we couldn't fetch the originalDataState we could default to - // DataStateManualRestore, but that is no longer a safe fallback since we support - // other data states now. In the unlikely event we can't get the current file data - // state, treat it as fatal. - originalDataState, err = entry.GetFileDataState(ctx, cfg.Path) - if err != nil { - err = fmt.Errorf("unable to determine original file data state: refusing to proceed with restoring file contents %w", err) - return - } - if err = entry.SetFileDataState(ctx, cfg.Path, beegfs.DataStateAvailable); err != nil { - return - } - fileDataStateCleared = true - allowOverwrite = true + // Restore the original stub file if download preparation overwrote it. + rstUrl := fmt.Appendf(nil, "rst://%d:%s\n", originalLockedInfo.StubUrlRstId, originalLockedInfo.StubUrlPath) + return mountPoint.CreateWriteClose(cfg.Path, rstUrl, 0644, true) + } else if lockedInfo.Size < lockedInfo.RemoteSize { + // Restore the enlarged file to it's original size. + return mountPoint.CreateOrResizeFile(cfg.Path, originalLockedInfo.Size, true) } + return nil + } + return undo, nil + } +} - if !allowOverwrite { - err = fmt.Errorf("download would overwrite existing path but the overwrite flag was not set: %w", fs.ErrExist) - return - } +func prepareDownloadNoFile(ctx context.Context, mountPoint filesystem.Provider, cfg *flex.JobRequestCfg) applyFn { + return func(pathState *PathState, appliedErr error) (undoFn, error) { + if appliedErr != nil { + return noopUndo, appliedErr + } - if !IsFileSizeMatched(lockedInfo) { - if err = mountPoint.CreatePreallocatedFile(cfg.Path, lockedInfo.RemoteSize, allowOverwrite); err != nil { - err = fmt.Errorf("unable to preallocate space for file: %w", err) - return - } - filePreallocated = true + lockedInfo := cfg.LockedInfo + undo := func() error { + if removeErr := mountPoint.Remove(cfg.Path); removeErr != nil && !errors.Is(removeErr, fs.ErrNotExist) { + return fmt.Errorf("unable to remove preallocated file: %w", removeErr) } - } else if IsFileOffloaded(lockedInfo) { - err = fmt.Errorf("unable to upload stub file: %w", ErrUnsupportedOpForRST) - return + return nil } - } else if cfg.Download { - if err = mountPoint.CreatePreallocatedFile(cfg.Path, lockedInfo.RemoteSize, cfg.Overwrite); err != nil { - err = fmt.Errorf("unable to preallocate space for file: %w", err) - return + + err := mountPoint.CreatePreallocatedFile(cfg.Path, lockedInfo.RemoteSize, cfg.Overwrite) + if err != nil { + if errors.Is(err, fs.ErrExist) { + return noopUndo, fmt.Errorf("unable to preallocate space for file: %w", err) + } + return undo, fmt.Errorf("unable to preallocate space for file: %w", err) } - fileCreated = true - filePreallocated = true - var info *flex.JobLockedInfo - if info, _, _, currentRSTCfg, entryInfo, ownerNode, err = GetLockedInfo(ctx, mountPoint, cfg, cfg.Path, false); err != nil { - err = fmt.Errorf("failed to collect information for new file: %w", err) - return + if *pathState, err = GetPathState(ctx, mountPoint, cfg.Path, PathStateWithLock); err != nil { + return undo, fmt.Errorf("failed to collect information for new file: %w", err) } + info := pathState.LockedInfo lockedInfo.SetReadWriteLocked(info.ReadWriteLocked) lockedInfo.SetExists(info.Exists) lockedInfo.SetSize(info.Size) lockedInfo.SetMtime(info.Mtime) lockedInfo.SetMode(info.Mode) - } else { - err = fmt.Errorf("unable to upload file: %w", fs.ErrNotExist) - return - } - // For a download uses currentRSTCfg, entryInfo, and ownerNode from the second GetLockedInfo call. - if err = updateRstCfg(nil); err != nil { - return err + return undo, nil } +} - // Generating the externalId must be the last possible error to avoid situations where, once - // the externalId is generated, it is lost because of a subsequent preconditional failure. - var externalId string - if externalId, err = client.GenerateExternalId(ctx, cfg); err != nil { - return fmt.Errorf("failed to generate external id: %s", err.Error()) - } else { - lockedInfo.SetExternalId(externalId) +func prepareUpdateFileRstPattern(ctx context.Context, cfg *flex.JobRequestCfg) applyFn { + return func(pathState *PathState, appliedErr error) (undo undoFn, err error) { + undo = noopUndo + if !(appliedErr == nil || IsErrJobTerminalSentinel(appliedErr)) { + err = appliedErr + return + } + + defer func() { + if err == nil { + err = appliedErr + } else if IsErrJobTerminalSentinel(appliedErr) { + // Return sentinel as a string so it's message is communicated but still report a + // failed precondition. + err = fmt.Errorf("%s: %w", appliedErr.Error(), err) + } + }() + + path := cfg.Path + entryInfo := pathState.EntryInfo + entryInfoMsg := entryInfo.GetOrigEntryInfo() + currentRSTCfg := entryInfo.Entry.Details.Remote.RemoteStorageTarget + ownerNode := pathState.OwnerNode + + newRSTCfg := currentRSTCfg + var rstIds []uint32 + var revertRstIds []uint32 + if cfg.GetUpdate() { + if !IsValidRstId(cfg.RemoteStorageTarget) { + err = fmt.Errorf("--%s requires a valid --%s to be specified", UpdateFlag, RemoteTargetFlag) + return + } + rstIds = []uint32{cfg.RemoteStorageTarget} + revertRstIds = currentRSTCfg.RSTIDs + newRSTCfg.RSTIDs = rstIds + } + + var cooldownSecs *uint16 + var revertCooldownSecs *uint16 + if cfg.HasCooldownSecs() { + v := uint16(math.MaxUint16) + if cfg.GetCooldownSecs() <= math.MaxUint16 { + v = uint16(cfg.GetCooldownSecs()) + } + cooldownSecs = &v + revertCooldownSecs = ¤tRSTCfg.CoolDownPeriod + newRSTCfg.CoolDownPeriod = v + } + + if setErr := entry.SetFileRstPattern(ctx, path, rstIds, cooldownSecs, currentRSTCfg, *entryInfoMsg, ownerNode); setErr != nil { + err = fmt.Errorf("failed to apply RST configuration: %w", setErr) + return + } + + undo = func() (undoErr error) { + if undoErr = entry.SetFileRstPattern(ctx, path, revertRstIds, revertCooldownSecs, newRSTCfg, *entryInfoMsg, ownerNode); undoErr != nil { + undoErr = fmt.Errorf("failed to revert RST configuration: %w", undoErr) + } + return + } + return undo, nil } +} - return nil +type PathStateMode int + +const ( + PathStateWithLock PathStateMode = iota + PathStateNoLock +) + +type PathState struct { + LockedInfo *flex.JobLockedInfo + LockAcquired bool + EntryInfo *entry.GetEntryCombinedInfo + RstCfg msg.RemoteStorageTarget + OwnerNode beegfs.Node } -// GetLockedInfo acquires the available information for inMountPath. An error will be returned when -// the lock fails to be acquired unless skipAccessLock is true. cfg is used as a configuration -// reference for the inMountPath, so cfg.Path will be ignored; this is necessary to avoid making -// unnecessary cfg clones since the lockedInfo can be used for multiple job requests. writeLockSet -// will be true when the write lock was set. When skipAccessLock is true the access lock state will -// not be changed. skipAccessLock is useful when a point in time read-only copy is needed. -// ErrOffloadFileNotReadable will be returned when the file is offloaded when client is unable to -// read the file. It returns ErrGetLockedInfoFatal if it is likely subsequent calls for other paths -// would likely fail due to some external error or misconfiguration. -func GetLockedInfo( - ctx context.Context, - mountPoint filesystem.Provider, - cfg *flex.JobRequestCfg, - inMountPath string, - skipAccessLock bool, -) (lockedInfo *flex.JobLockedInfo, writeLockSet bool, rstIds []uint32, currentRSTCfg msg.RemoteStorageTarget, entryInfoMsg msg.EntryInfo, ownerNode beegfs.Node, err error) { - - lockedInfo = &flex.JobLockedInfo{} - if IsValidRstId(cfg.RemoteStorageTarget) { - rstIds = []uint32{cfg.RemoteStorageTarget} - } - - entryInfo, err := entry.GetEntry(ctx, nil, entry.GetEntriesCfg{ - Verbose: false, - IncludeOrigMsg: true, - }, inMountPath) +// GetPathState collects existing path state for inMountPath and optionally acquires the file +// access lock. It returns information derived from the current file, stub, and entry metadata for +// the path. +// +// ErrOffloadFileNotReadable is returned when the file is offloaded and the client cannot read the +// stub file. ErrGetLockedInfoFatal wraps entry lookup failures that likely indicate an external +// error or misconfiguration that may also affect other paths. +func GetPathState(ctx context.Context, mountPoint filesystem.Provider, inMountPath string, mode PathStateMode) (PathState, error) { + result := PathState{} + result.LockedInfo = &flex.JobLockedInfo{} + + entryCfg := entry.GetEntriesCfg{Verbose: false, IncludeOrigMsg: true} + entryInfo, err := entry.GetEntry(ctx, nil, entryCfg, inMountPath) if err != nil { if errors.Is(err, os.ErrNotExist) { - return lockedInfo, writeLockSet, rstIds, currentRSTCfg, entryInfoMsg, ownerNode, nil + return result, nil } - return lockedInfo, writeLockSet, rstIds, currentRSTCfg, entryInfoMsg, ownerNode, fmt.Errorf("%w: %w", ErrGetLockedInfoFatal, err) + return result, fmt.Errorf("%w: %w", ErrGetPathStateFatal, err) } - lockedInfo.Exists = true - if entryInfo.Entry.Details == nil { - return lockedInfo, writeLockSet, rstIds, currentRSTCfg, entryInfoMsg, ownerNode, - fmt.Errorf("%w: entry details unavailable (%s)", ErrGetLockedInfoFatal, entryInfo.Entry.EntryInfoPopulated) + entryInfoMsg := entryInfo.GetOrigEntryInfo() + if entryInfoMsg == nil { + return result, fmt.Errorf("original entry info failed to be retrieved: %w", ErrGetPathStateFatal) } - if rstIds == nil { - rstIds = entryInfo.Entry.Details.Remote.RSTIDs - } + result.EntryInfo = entryInfo + result.LockedInfo.Exists = true - if !skipAccessLock { - if !entryInfo.Entry.Details.FileState.IsReadWriteLocked() { - err = entry.SetAccessFlags(ctx, inMountPath, beegfs.LockedContentAccessFlags) - if err != nil { - return - } - writeLockSet = true - } - lockedInfo.SetReadWriteLocked(true) + entryDetails := entryInfo.Entry.Details + if entryDetails == nil { + return result, fmt.Errorf("%w: entry details unavailable (%s)", ErrGetPathStateFatal, entryInfo.Entry.EntryInfoPopulated) } - stat, err := mountPoint.Lstat(inMountPath) - if err != nil { - return + result.OwnerNode = entryInfo.Entry.MetaOwnerNode + result.RstCfg = entryDetails.Remote.RemoteStorageTarget + + isFileLocked := entryDetails.FileState.IsReadWriteLocked() + if !isFileLocked && mode == PathStateWithLock { + if err = entry.SetAccessFlagsWithEntryInfo(ctx, inMountPath, beegfs.LockedContentAccessFlags, *entryInfoMsg, result.OwnerNode); err != nil { + return result, err + } + isFileLocked = true + result.LockAcquired = true } - lockedInfo.Size = stat.Size() - lockedInfo.Mtime = timestamppb.New(stat.ModTime()) - lockedInfo.Mode = uint32(stat.Mode()) + result.LockedInfo.SetReadWriteLocked(isFileLocked) - if beegfs.IsDataStateOffloaded(entryInfo.Entry.Details.FileState.GetDataState()) { - if lockedInfo.StubUrlRstId, lockedInfo.StubUrlPath, err = GetOffloadedUrlPartsFromFile(mountPoint, inMountPath); err != nil { + if beegfs.IsDataStateOffloaded(entryDetails.FileState.GetDataState()) { + stubUrlRstId, stubUrlPath, err := GetOffloadedUrlPartsFromFile(mountPoint, inMountPath) + if err != nil { if errors.Is(err, syscall.EWOULDBLOCK) { - return lockedInfo, writeLockSet, rstIds, currentRSTCfg, entryInfoMsg, ownerNode, ErrOffloadFileNotReadable + return result, ErrOffloadFileNotReadable } - return lockedInfo, writeLockSet, rstIds, currentRSTCfg, entryInfoMsg, ownerNode, fmt.Errorf("unable to retrieve stub file info: %w", err) - } - - if IsValidRstId(cfg.RemoteStorageTarget) && cfg.RemoteStorageTarget != lockedInfo.StubUrlRstId { - return lockedInfo, writeLockSet, nil, currentRSTCfg, entryInfoMsg, ownerNode, fmt.Errorf("supplied --%s does not match stub file", RemoteTargetFlag) + return result, fmt.Errorf("unable to retrieve stub file info: %w", err) } - rstIds = []uint32{lockedInfo.StubUrlRstId} + result.LockedInfo.StubUrlRstId = stubUrlRstId + result.LockedInfo.StubUrlPath = stubUrlPath + // Override the configured rstIds with the rstId of the stub file rstId. + result.RstCfg.RSTIDs = []uint32{result.LockedInfo.StubUrlRstId} } - currentRSTCfg = entryInfo.Entry.Details.Remote.RemoteStorageTarget - origEntryInfoPtr := entryInfo.GetOrigEntryInfo() - if origEntryInfoPtr != nil { - entryInfoMsg = *origEntryInfoPtr - } else { - entryInfoMsg = msg.EntryInfo{} + stat, err := mountPoint.Lstat(inMountPath) + if err != nil { + return result, err } - ownerNode = entryInfo.Entry.MetaOwnerNode + result.LockedInfo.Size = stat.Size() + result.LockedInfo.Mtime = timestamppb.New(stat.ModTime()) + result.LockedInfo.Mode = uint32(stat.Mode()) - if rstIds == nil { - return lockedInfo, writeLockSet, rstIds, currentRSTCfg, entryInfoMsg, ownerNode, ErrFileHasNoRSTs - } - return + return result, nil } // restorePolicyToDataState maps a RestorePolicy enum value to the corresponding beegfs.DataState. @@ -780,10 +1020,11 @@ func GetOffloadedUrlPartsFromFile(beegfs filesystem.Provider, path string) (uint return urlRstId, urlKey, nil } +var rstUrlRe = regexp.MustCompile(`^rst://([0-9]+):(.+)$`) + func parseRstUrl(url []byte) (uint32, string, error) { urlString := string(url) - re := regexp.MustCompile(`^rst://([0-9]+):(.+)$`) - matches := re.FindStringSubmatch(urlString) + matches := rstUrlRe.FindStringSubmatch(urlString) if len(matches) != 3 { return 0, "", fmt.Errorf("input does not match expected format: rst://:") } @@ -797,26 +1038,6 @@ func parseRstUrl(url []byte) (uint32, string, error) { return uint32(num), s3Key, nil } -func CheckEntry(e entry.Entry, ignoreReaders bool, ignoreWriters bool) error { - if e.Details == nil { - return fmt.Errorf("entry details unavailable (%s)", e.EntryInfoPopulated) - } - var err error - if !ignoreWriters && e.Details.NumSessionsWrite > 0 { - err = ErrFileOpenForWriting - } - if !ignoreReaders && e.Details.NumSessionsRead > 0 { - // Not using errors.Join because it adds a newline when printing each error which looks - // awkward in the CTL output. - if err != nil { - err = ErrFileOpenForReadingAndWriting - } else { - err = ErrFileOpenForReading - } - } - return err -} - func IsValidRstId(rstId uint32) bool { return rstId != 0 } @@ -845,7 +1066,7 @@ func GetDownloadInMountPath(path string, remotePath string, remotePathDir string } if flatten { - relPath = strings.Replace(relPath, "/", "_", -1) + relPath = strings.ReplaceAll(relPath, "/", "_") } if relPath == "." { diff --git a/common/rst/rst_test.go b/common/rst/rst_test.go index 560574de..db0a955d 100644 --- a/common/rst/rst_test.go +++ b/common/rst/rst_test.go @@ -95,6 +95,29 @@ func TestRecreateWorkRequests(t *testing.T) { assert.Nil(t, invalidRequests[0].Type) } +// TestRecreateWorkRequestsPropagatesBulkInfo guards against the WorkRequest.BulkInfo field silently +// staying unset: IsWorkRequestReady and ExecuteWorkRequestPart both gate their bulk-specific defers +// on request.HasBulkInfo() for the *flex.WorkRequest RecreateWorkRequests builds, not the JobRequest. +func TestRecreateWorkRequestsPropagatesBulkInfo(t *testing.T) { + jobBulk := proto.Clone(baseTestJob).(*beeremote.Job) + jobBulk.Request.Type = &beeremote.JobRequest_Sync{ + Sync: &flex.SyncJob{Operation: flex.SyncJob_DOWNLOAD}, + } + jobBulk.Request.BulkInfo = &flex.BulkJobRequestInfo{ + StateMountPath: "state", + Operation: "bulk-retrieve", + JobIndex: 7, + } + + requests := RecreateWorkRequests(jobBulk, getNewTestSegments(baseTestSegments)) + require.Len(t, requests, len(baseTestSegments)) + for _, req := range requests { + require.True(t, req.HasBulkInfo()) + assert.True(t, proto.Equal(jobBulk.Request.BulkInfo, req.GetBulkInfo())) + assert.NotSame(t, jobBulk.Request.BulkInfo, req.GetBulkInfo(), "each WorkRequest must get its own clone, not a shared pointer") + } +} + func TestGenerateSegments(t *testing.T) { type expectation struct { offsetStart int64 diff --git a/common/rst/s3.go b/common/rst/s3.go index ab41e1e5..dd66c1e0 100644 --- a/common/rst/s3.go +++ b/common/rst/s3.go @@ -11,7 +11,6 @@ import ( "io/fs" "net/url" "os" - "path/filepath" "sort" "strings" "sync" @@ -25,7 +24,6 @@ import ( "github.com/aws/smithy-go" doublestar "github.com/bmatcuk/doublestar/v4" "github.com/thinkparq/beegfs-go/common/beegfs" - "github.com/thinkparq/beegfs-go/common/beemsg/msg" "github.com/thinkparq/beegfs-go/common/filesystem" "github.com/thinkparq/beegfs-go/ctl/pkg/ctl/entry" @@ -35,6 +33,87 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) +// s3ApiClient is the low-level s3 transport layer used by S3Client. Provider specific wrappers can +// customize SDK calls here without reimplementing the higher-level RST behavior. +type s3ApiClient interface { + ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) + ListObjectsV2Pages(ctx context.Context, params *s3.ListObjectsV2Input, pageFn func(*s3.ListObjectsV2Output) (bool, error)) error + RestoreObject(ctx context.Context, params *s3.RestoreObjectInput, optFns ...func(*s3.Options)) (*s3.RestoreObjectOutput, error) + HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) + CreateMultipartUpload(ctx context.Context, params *s3.CreateMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error) + AbortMultipartUpload(ctx context.Context, params *s3.AbortMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error) + CompleteMultipartUpload(ctx context.Context, params *s3.CompleteMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error) + PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) + UploadPart(ctx context.Context, params *s3.UploadPartInput, optFns ...func(*s3.Options)) (*s3.UploadPartOutput, error) + GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) + DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) +} + +// defaultS3ApiClient is the default s3ApiClient backed by the AWS SDK's s3 client. +type defaultS3ApiClient struct { + client *s3.Client +} + +var _ s3ApiClient = &defaultS3ApiClient{} + +func (d *defaultS3ApiClient) ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) { + return d.client.ListObjectsV2(ctx, params, optFns...) +} + +func (d *defaultS3ApiClient) ListObjectsV2Pages(ctx context.Context, params *s3.ListObjectsV2Input, pageFn func(*s3.ListObjectsV2Output) (bool, error)) error { + paginator := s3.NewListObjectsV2Paginator(d.client, params) + for paginator.HasMorePages() { + output, err := paginator.NextPage(ctx) + if err != nil { + return err + } + cont, err := pageFn(output) + if err != nil { + return err + } + if !cont { + return nil + } + } + return nil +} + +func (d *defaultS3ApiClient) RestoreObject(ctx context.Context, params *s3.RestoreObjectInput, optFns ...func(*s3.Options)) (*s3.RestoreObjectOutput, error) { + return d.client.RestoreObject(ctx, params, optFns...) +} + +func (d *defaultS3ApiClient) HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + return d.client.HeadObject(ctx, params, optFns...) +} + +func (d *defaultS3ApiClient) CreateMultipartUpload(ctx context.Context, params *s3.CreateMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error) { + return d.client.CreateMultipartUpload(ctx, params, optFns...) +} + +func (d *defaultS3ApiClient) AbortMultipartUpload(ctx context.Context, params *s3.AbortMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error) { + return d.client.AbortMultipartUpload(ctx, params, optFns...) +} + +func (d *defaultS3ApiClient) CompleteMultipartUpload(ctx context.Context, params *s3.CompleteMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error) { + return d.client.CompleteMultipartUpload(ctx, params, optFns...) +} + +func (d *defaultS3ApiClient) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + return d.client.PutObject(ctx, params, optFns...) +} + +func (d *defaultS3ApiClient) UploadPart(ctx context.Context, params *s3.UploadPartInput, optFns ...func(*s3.Options)) (*s3.UploadPartOutput, error) { + return d.client.UploadPart(ctx, params, optFns...) +} + +func (d *defaultS3ApiClient) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + return d.client.GetObject(ctx, params, optFns...) +} + +func (d *defaultS3ApiClient) DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) { + return d.client.DeleteObject(ctx, params, optFns...) +} + type S3StorageClass struct { retrievalTier types.Tier archival bool @@ -44,11 +123,14 @@ type S3StorageClass struct { autoRestore bool // defines whether archived objects should be permitted to be restored. } +// S3Client implements the shared Provider behavior for s3 compatible backends and uses s3ApiClient +// to perform the low-level s3 operations. type S3Client struct { config *flex.RemoteStorageTarget - // TODO: https://github.com/thinkparq/gobee/issues/28 - // Rework client into an `s3Provider` interface type. - client *s3.Client + // s3Config holds provider-specific S3 options used by this client. This allows providers such + // as xtreemstore to reuse the S3 implementation while keeping their own top-level RST type. + s3Config *flex.RemoteStorageTarget_S3 + apiClient s3ApiClient mountPoint filesystem.Provider storageClasses map[types.StorageClass]S3StorageClass isListStartAfterKeySupported *bool @@ -58,15 +140,51 @@ type S3Client struct { var _ Provider = &S3Client{} func newS3(ctx context.Context, rstConfig *flex.RemoteStorageTarget, mountPoint filesystem.Provider) (Provider, error) { - s3Provider := rstConfig.GetS3() + return newS3WithOptions(ctx, rstConfig, rstConfig.GetS3(), mountPoint) +} + +type s3ProviderOption func(*s3ProviderBuildCfg) +type s3ProviderBuildCfg struct { + apiClient func(base s3ApiClient) s3ApiClient + s3Options []func(*s3.Options) +} + +func defaultS3ProviderBuildCfg() s3ProviderBuildCfg { + return s3ProviderBuildCfg{ + apiClient: func(base s3ApiClient) s3ApiClient { return base }, + } +} + +func withS3ApiClient(fn func(s3ApiClient) s3ApiClient) s3ProviderOption { + return func(cfg *s3ProviderBuildCfg) { + if fn != nil { + cfg.apiClient = fn + } + } +} + +// newS3WithOptions constructs an S3Client. withS3ApiClient is applied before the client is +// created, so shared wrapper state can rely on apiClient already being populated. +func newS3WithOptions(ctx context.Context, rstConfig *flex.RemoteStorageTarget, s3Config *flex.RemoteStorageTarget_S3, mountPoint filesystem.Provider, opts ...s3ProviderOption) (Provider, error) { + if s3Config == nil { + return nil, fmt.Errorf("s3 configuration must be specified") + } + + buildCfg := defaultS3ProviderBuildCfg() + for _, opt := range opts { + if opt != nil { + opt(&buildCfg) + } + } + awsCfg, err := awsConfig.LoadDefaultConfig( ctx, - awsConfig.WithBaseEndpoint(s3Provider.GetEndpointUrl()), - awsConfig.WithRegion(s3Provider.GetRegion()), + awsConfig.WithBaseEndpoint(s3Config.GetEndpointUrl()), + awsConfig.WithRegion(s3Config.GetRegion()), awsConfig.WithCredentialsProvider( credentials.NewStaticCredentialsProvider( - s3Provider.GetAccessKey(), - s3Provider.GetSecretKey(), + s3Config.GetAccessKey(), + s3Config.GetSecretKey(), "", // session token ), ), @@ -79,26 +197,34 @@ func newS3(ctx context.Context, rstConfig *flex.RemoteStorageTarget, mountPoint // was deprecated for new regions in 2020. So, check whether the provided endpoint url starts // with the bucket as part of the hostname. Otherwise, use the path-style. // https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html - endpointUrl, err := url.Parse(s3Provider.GetEndpointUrl()) + endpointUrl, err := url.Parse(s3Config.GetEndpointUrl()) if err != nil { return nil, fmt.Errorf("unable to parse s3 end-point: %w", err) } - bucket := s3Provider.GetBucket() + bucket := s3Config.GetBucket() host := endpointUrl.Hostname() usePathStyle := !strings.HasPrefix(host, bucket+".") - client := s3.NewFromConfig(awsCfg, func(o *s3.Options) { + awsClient := s3.NewFromConfig(awsCfg, func(o *s3.Options) { o.UsePathStyle = usePathStyle + for _, optFn := range buildCfg.s3Options { + optFn(o) + } }) + apiClient := buildCfg.apiClient(&defaultS3ApiClient{client: awsClient}) + if apiClient == nil { + return nil, fmt.Errorf("s3 api client wrapper returned nil") + } s3Client := &S3Client{ config: rstConfig, - client: client, + s3Config: s3Config, + apiClient: apiClient, mountPoint: mountPoint, storageClasses: make(map[types.StorageClass]S3StorageClass), isListStartAfterKeySupportedMu: sync.Mutex{}, } - for _, class := range s3Provider.StorageClass { + for _, class := range s3Config.StorageClass { name := types.StorageClass(class.GetName()) if name == "" { return nil, fmt.Errorf("storage class must specify a valid storage class name") @@ -148,14 +274,13 @@ func (s *S3Client) checkStartAfterSupport(ctx context.Context) error { } input := &s3.ListObjectsV2Input{ - Bucket: aws.String(s.config.GetS3().Bucket), + Bucket: aws.String(s.s3Config.Bucket), StartAfter: aws.String("-"), MaxKeys: aws.Int32(0), } - if _, err := s.client.ListObjectsV2(ctx, input); err != nil { - var apiErr smithy.APIError - if errors.As(err, &apiErr) && apiErr.ErrorCode() == "InvalidArgument" && strings.Contains(strings.ToLower(apiErr.ErrorMessage()), "startafter") { + if _, err := s.apiClient.ListObjectsV2(ctx, input); err != nil { + if apiErr, ok := errors.AsType[smithy.APIError](err); ok && apiErr.ErrorCode() == "InvalidArgument" && strings.Contains(strings.ToLower(apiErr.ErrorMessage()), "startafter") { s.isListStartAfterKeySupported = new(bool) *s.isListStartAfterKeySupported = false return nil @@ -240,22 +365,36 @@ func (r *S3Client) GenerateWorkRequests(ctx context.Context, lastJob *beeremote. } } - var writeLockSet bool + undoAppliedPlan := noopUndo + lockAcquired := true defer func() { - if err == nil || errors.Is(err, ErrJobAlreadyOffloaded) { + if err == nil { return } - if writeLockSet { + if !IsErrJobTerminalSentinel(err) { + if undoErr := undoAppliedPlan(); undoErr != nil { + err = fmt.Errorf("%w: failed to undo changes: %w", err, undoErr) + } else { + err = fmt.Errorf("%w: %w", ErrJobFailedPrecondition, err) + } + } + + if lockAcquired && !errors.Is(err, ErrJobAlreadyOffloaded) { if clearWriteLockErr := entry.ClearAccessFlags(ctx, request.Path, beegfs.LockedContentAccessFlags); clearWriteLockErr != nil { err = errors.Join(err, fmt.Errorf("unable to write lock: %w", clearWriteLockErr)) } } }() - if writeLockSet, err = r.prepareJobRequest(ctx, r.getJobRequestCfg(request), sync); err != nil { - return nil, err + if !IsFileLocked(sync.LockedInfo) { + // The file access lock was not previously acquired which means the file state information + // has not been determine and by extension, work request in unprepared. + if undoAppliedPlan, lockAcquired, err = r.prepareJobRequest(ctx, request, sync); err != nil { + return + } } + job.SetExternalId(sync.LockedInfo.ExternalId) switch sync.Operation { @@ -269,9 +408,64 @@ func (r *S3Client) GenerateWorkRequests(ctx context.Context, lastJob *beeremote. return } +// prepareJobRequest acquires the file access lock (if it isn't already held), plans and applies +// any local file state changes needed for the sync operation, updates the file's RST +// configuration if requested, and generates an external ID for the job. It is only called the +// first time GenerateWorkRequests runs for a given job; callers should skip it once +// sync.LockedInfo indicates the lock was already acquired by an earlier call. +func (r *S3Client) prepareJobRequest(ctx context.Context, request *beeremote.JobRequest, sync *flex.SyncJob) (undoAppliedPlan undoFn, lockAcquired bool, err error) { + undoAppliedPlan = noopUndo + cfg := r.getJobRequestCfg(request) + + var pathState *PathState + pathState, err = r.getLockedInfo(ctx, cfg) + lockAcquired = pathState != nil && IsFileLocked(pathState.LockedInfo) && pathState.LockAcquired + if err != nil { + return + } + sync.SetLockedInfo(pathState.LockedInfo) + cfg.SetLockedInfo(pathState.LockedInfo) + + if !FileExists(pathState.LockedInfo) { + err = os.ErrNotExist + return + } + + if !IsFileLocked(pathState.LockedInfo) || (!pathState.LockAcquired && !IsFileOffloaded(pathState.LockedInfo)) { + err = fmt.Errorf("failed to acquire the write lock") + return + } + + var applyPlan applyPlanFn + + if applyPlan, err = PlanFileStateForWorkRequests(ctx, r.mountPoint, cfg); err != nil { + return + } + + undoAppliedPlan, err = applyPlan(pathState) + if err != nil { + return + } + + var externalId string + if externalId, err = r.GenerateExternalId(ctx, cfg); err != nil { + return + } + sync.LockedInfo.SetExternalId(externalId) + return +} + // ExecuteJobBuilderRequest is not implemented and should never be called. -func (r *S3Client) ExecuteJobBuilderRequest(ctx context.Context, workRequest *flex.WorkRequest, jobSubmissionChan chan<- *beeremote.JobRequest) (bool, error) { - return false, ErrUnsupportedOpForRST +func (r *S3Client) ExecuteJobBuilderRequest(ctx context.Context, workRequest *flex.WorkRequest, jobSubmissionCh chan<- *beeremote.JobRequest, workerSaturation []func() float64) *SchedulingResult { + return &SchedulingResult{Err: ErrUnsupportedOpForRST} +} + +func (r *S3Client) IncludeRequestInBulkOperation(ctx context.Context, request *beeremote.JobRequest) (include bool, operation string) { + return false, "" +} + +func (r *S3Client) OpenBulkOperation(ctx context.Context, stateMountPath string, operation string) (clientBulkOperation, error) { + return nil, ErrUnsupportedOpForRST } func (r *S3Client) IsWorkRequestReady(ctx context.Context, request *flex.WorkRequest) (bool, time.Duration, error) { @@ -298,7 +492,7 @@ func (r *S3Client) IsWorkRequestReady(ctx context.Context, request *flex.WorkReq } restoreObjectInput := &s3.RestoreObjectInput{ - Bucket: aws.String(r.config.GetS3().Bucket), + Bucket: aws.String(r.s3Config.Bucket), Key: aws.String(sync.RemotePath), RestoreRequest: restoreRequest, } @@ -306,9 +500,8 @@ func (r *S3Client) IsWorkRequestReady(ctx context.Context, request *flex.WorkReq // Multiple workers may attempt to restore the same object concurrently. In that // case, a RestoreAlreadyInProgress error can occur and should be ignored. If the // restore has already completed, subsequent requests will succeed with HTTP 200 OK. - if _, err := r.client.RestoreObject(ctx, restoreObjectInput); err != nil { - var apiErr smithy.APIError - if !errors.As(err, &apiErr) || apiErr.ErrorCode() != "RestoreAlreadyInProgress" { + if _, err := r.apiClient.RestoreObject(ctx, restoreObjectInput); err != nil { + if apiErr, ok := errors.AsType[smithy.APIError](err); !ok || apiErr.ErrorCode() != "RestoreAlreadyInProgress" { return false, 0, err } } @@ -423,7 +616,7 @@ func (r *S3Client) GetWalk(ctx context.Context, prefix string, chanSize int, res } input := &s3.ListObjectsV2Input{ - Bucket: aws.String(r.config.GetS3().Bucket), + Bucket: aws.String(r.s3Config.Bucket), Prefix: aws.String(unescapedPrefixWithoutPattern), MaxKeys: aws.Int32(int32(maxKeysPerPage)), } @@ -440,18 +633,8 @@ func (r *S3Client) GetWalk(ctx context.Context, prefix string, chanSize int, res var key string var lastKey string - objectPaginator := s3.NewListObjectsV2Paginator(r.client, input) - keysFound = objectPaginator.HasMorePages() - for objectPaginator.HasMorePages() { - output, err := objectPaginator.NextPage(ctx) - if err != nil { - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - send(&filesystem.StreamPathResult{Err: fmt.Errorf("prefix walk was cancelled: %w", err)}) - } else { - send(&filesystem.StreamPathResult{Err: fmt.Errorf("prefix walk failed: %w", err)}) - } - return - } + pageFn := func(output *s3.ListObjectsV2Output) (bool, error) { + keysFound = true // When resuming with s3ResumeToken ContinuationToken and ContinuationStartKey, // search for ContinuationStartKey on the page. If it does not exist and there's a @@ -474,7 +657,7 @@ func (r *S3Client) GetWalk(ctx context.Context, prefix string, chanSize int, res if len(filteredContents) == 0 { if nextGreaterKeyIndex == -1 { // There were no greater keys on the current page. So check the next page. - continue + return true, nil } continuationFindStart = false filteredContents = append(filteredContents, output.Contents[nextGreaterKeyIndex:]...) @@ -499,7 +682,7 @@ func (r *S3Client) GetWalk(ctx context.Context, prefix string, chanSize int, res } else { send(&filesystem.StreamPathResult{ResumeToken: token}) } - return + return false, nil } rt := s3ResumeToken{ContinuationToken: aws.ToString(output.ContinuationToken), ContinuationStartKey: key} @@ -508,11 +691,11 @@ func (r *S3Client) GetWalk(ctx context.Context, prefix string, chanSize int, res } else { send(&filesystem.StreamPathResult{ResumeToken: token}) } - return + return false, nil } if !send(&filesystem.StreamPathResult{Path: key}) { - return + return false, nil } lastKey = key @@ -520,18 +703,26 @@ func (r *S3Client) GetWalk(ctx context.Context, prefix string, chanSize int, res maxKeys-- } } + + return true, nil + } + + err := r.apiClient.ListObjectsV2Pages(ctx, input, pageFn) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + send(&filesystem.StreamPathResult{Err: fmt.Errorf("prefix walk was cancelled: %w", err)}) + } else { + send(&filesystem.StreamPathResult{Err: fmt.Errorf("prefix walk failed: %w", err)}) + } + return } return } if isKey { - _, err := r.client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: aws.String(r.config.GetS3().Bucket), - Key: aws.String(unescapedPrefixWithoutPattern), - }) + _, err := r.headObject(ctx, unescapedPrefixWithoutPattern) if err != nil { - var apiErr smithy.APIError - if errors.As(err, &apiErr) && apiErr.ErrorCode() == "NotFound" { + if apiErr, ok := errors.AsType[smithy.APIError](err); ok && apiErr.ErrorCode() == "NotFound" { // Try walking as a prefix since there was no key. If not a valid prefix // fallback to the original error. if !prefixWalk() { @@ -715,105 +906,114 @@ func (r *S3Client) completeSyncWorkRequests_Download(ctx context.Context, job *b request := job.GetRequest() sync := request.GetSync() + if abort { + if isWorkStarted(workResults) { + // The download was incomplete so replace with a stub file for the requested resource. + if err := CreateOffloadedDataFile(ctx, r.mountPoint, request.Path, sync.RemotePath, request.RemoteStorageTarget, true, restorePolicyToDataState(request.GetRestorePolicy())); err != nil { + return fmt.Errorf("failed to replace incomplete download with stub file: %w", err) + } + return nil + } + + if IsFileOffloaded(sync.LockedInfo) { + if err := CreateOffloadedDataFile(ctx, r.mountPoint, request.Path, sync.LockedInfo.StubUrlPath, sync.LockedInfo.StubUrlRstId, true, restorePolicyToDataState(request.GetRestorePolicy())); err != nil { + return fmt.Errorf("failed to restore original stub file: %w", err) + } + return nil + } + + if !sync.LockedInfo.Exists { + return r.mountPoint.Remove(request.Path) + } + + // File exist and no changes were made so restore the mtime and if needed, restore the file size. + job.SetStopMtime(sync.LockedInfo.Mtime) + mtime := sync.LockedInfo.Mtime.AsTime() + if sync.LockedInfo.Size < sync.LockedInfo.RemoteSize { + // The existing file was enlarged but no changes were made to original contents so we + // can safely restore the contents by reducing the file to its original size. + if err := r.mountPoint.CreateOrResizeFile(request.Path, sync.LockedInfo.Size, true); err != nil { + return fmt.Errorf("failed to restore original file size: %w", err) + } + } + if err := r.mountPoint.Chtimes(request.Path, mtime, mtime); err != nil { + return fmt.Errorf("failed to restore original mtime: %w", err) + } + return nil + } + _, mtime, _, err := r.getObjectMetadata(ctx, sync.RemotePath, false) if err != nil { return fmt.Errorf("unable to verify the remote object has not changed: %w", err) } job.SetStopMtime(timestamppb.New(mtime)) - // Skip checking the file was modified if we were told to abort since the mtime may not have - // been set correctly anyway given the error check is skipped above. - if !abort { - start := job.GetStartMtime().AsTime() - stop := job.GetStopMtime().AsTime() - if !start.Equal(stop) { - return fmt.Errorf("successfully completed all work requests but the remote file or object appears to have been modified (mtime at job start: %s / mtime at job completion: %s)", - start.Format(time.RFC3339), stop.Format(time.RFC3339)) - } - - // Update the downloaded file's access and modification times so they accurately reflect the beegfs-mtime. - absPath := filepath.Join(r.mountPoint.GetMountPath(), request.Path) - if err := os.Chtimes(absPath, mtime, mtime); err != nil { - return fmt.Errorf("failed to update download's mtime: %w", err) - } + start := job.GetStartMtime().AsTime() + stop := job.GetStopMtime().AsTime() + if !start.Equal(stop) { + return fmt.Errorf("successfully completed all work requests but the remote file or object appears to have been modified (mtime at job start: %s / mtime at job completion: %s)", + start.Format(time.RFC3339), stop.Format(time.RFC3339)) + } + if !request.StubLocal { // Clear offloaded data state when contents for a stub file were downloaded successfully. - if !request.StubLocal && IsFileOffloaded(sync.LockedInfo) { + if IsFileOffloaded(sync.LockedInfo) { if err := entry.SetFileDataState(ctx, request.Path, beegfs.DataStateAvailable); err != nil { return fmt.Errorf("unable to clear offloaded data state: %w", err) } } - } - - return nil -} -// prepareJobRequest ensures that sync.LockedInfo is full populated. -func (r *S3Client) prepareJobRequest(ctx context.Context, cfg *flex.JobRequestCfg, sync *flex.SyncJob) (writeLockSet bool, err error) { - lockedInfo := sync.LockedInfo - if IsFileLocked(lockedInfo) && HasRemotePathInfo(lockedInfo) { - return + // Reduce the file size if it's larger than the remote object. This situation means the original + // file size was larger than needed so no additional space was preallocated. This must happen + // before Chtimes below since resizing the file updates its mtime. + if sync.LockedInfo.Size > sync.LockedInfo.RemoteSize { + if err := r.mountPoint.CreateOrResizeFile(request.Path, sync.LockedInfo.RemoteSize, true); err != nil { + return fmt.Errorf("failed to reduce downloaded file to the remote object's size: %w", err) + } + } } - var currentRSTCfg msg.RemoteStorageTarget - var entryInfoMsg msg.EntryInfo - var ownerNode beegfs.Node - var getLockedInfoCalled bool - - if !IsFileLocked(lockedInfo) { - if lockedInfo, writeLockSet, _, currentRSTCfg, entryInfoMsg, ownerNode, err = GetLockedInfo(ctx, r.mountPoint, cfg, cfg.Path, false); err != nil { - err = fmt.Errorf("%w: %w", ErrJobFailedPrecondition, fmt.Errorf("failed to acquire lock: %w", err)) - return - } - getLockedInfoCalled = true - cfg.SetLockedInfo(lockedInfo) - sync.SetLockedInfo(lockedInfo) + // Update the downloaded file's access and modification times so they accurately reflect the beegfs-mtime. + if err := r.mountPoint.Chtimes(request.Path, mtime, mtime); err != nil { + return fmt.Errorf("failed to update download's mtime: %w", err) } - if !HasRemotePathInfo(lockedInfo) { - request := BuildJobRequest(ctx, r, r.mountPoint, cfg) - status := request.GetGenerationStatus() - if status != nil { - err = fmt.Errorf("%w: %s", ErrJobFailedPrecondition, status.Message) - return - } + return nil +} - // Same logic as GetLockedInfo just without getting the lock. - if !getLockedInfoCalled { - var entryInfo *entry.GetEntryCombinedInfo - if entryInfo, err = entry.GetEntry(ctx, nil, entry.GetEntriesCfg{ - Verbose: false, - IncludeOrigMsg: true, - }, cfg.Path); err != nil { - err = fmt.Errorf("failed to get entry info: %w", err) - return +// isWorkStarted returns true whenever a part indicates it was started or, in order to maintain +// backwards compatibility, when the part's optional Started field is nil. +func isWorkStarted(workResults []*flex.Work) bool { + for _, result := range workResults { + for _, part := range result.GetParts() { + if part == nil || part.Started == nil || *part.Started { + return true } - origEntryInfoPtr := entryInfo.GetOrigEntryInfo() - if origEntryInfoPtr != nil { - entryInfoMsg = *origEntryInfoPtr - } else { - entryInfoMsg = msg.EntryInfo{} - } - if entryInfo.Entry.Details == nil { - err = fmt.Errorf("unable to determine remote targets, full entry details unavailable for %s: %s", cfg.Path, entryInfo.Entry.EntryInfoPopulated) - return - } - currentRSTCfg = entryInfo.Entry.Details.Remote.RemoteStorageTarget - ownerNode = entryInfo.Entry.MetaOwnerNode } + } + return false +} - if err = PrepareFileStateForWorkRequests(ctx, r, r.mountPoint, currentRSTCfg, entryInfoMsg, ownerNode, cfg); err != nil { - if !errors.Is(err, ErrJobAlreadyComplete) && !errors.Is(err, ErrJobAlreadyOffloaded) { - err = fmt.Errorf("%w: %s", ErrJobFailedPrecondition, fmt.Sprintf("failed to prepare file state: %s", err.Error())) - } - return - } - sync.SetRemotePath(cfg.RemotePath) - sync.SetFlatten(cfg.Flatten) - sync.SetOverwrite(cfg.Overwrite) +func (r *S3Client) getLockedInfo(ctx context.Context, cfg *flex.JobRequestCfg) (*PathState, error) { + pathState, err := GetPathState(ctx, r.mountPoint, cfg.Path, PathStateWithLock) + if err != nil { + return &pathState, fmt.Errorf("failed to get path state information: %w", err) } - return + remoteSize, remoteMtime, isArchived, isArchiveRestoreAllowed, err := r.GetRemotePathInfo(ctx, cfg) + if err != nil && (cfg.Download || !errors.Is(err, os.ErrNotExist)) { + return &pathState, fmt.Errorf("unable to retrieve remote path information: %w", err) + } + if cfg.Download && isArchived && !isArchiveRestoreAllowed { + return &pathState, fmt.Errorf("remote object is archived and restore is not permitted; rerun with --%s to continue", AllowRestoreFlag) + } + + if !errors.Is(err, os.ErrNotExist) { + pathState.LockedInfo.SetRemoteSize(remoteSize) + pathState.LockedInfo.SetRemoteMtime(timestamppb.New(remoteMtime)) + pathState.LockedInfo.SetIsArchived(isArchived) + } + return &pathState, nil } type s3ArchiveInfo struct { @@ -844,6 +1044,14 @@ func (r *S3Client) archiveStatus(storageClass types.StorageClass, restoreMsg *st return status } +func (r *S3Client) headObject(ctx context.Context, key string) (*s3.HeadObjectOutput, error) { + input := &s3.HeadObjectInput{ + Bucket: aws.String(r.s3Config.Bucket), + Key: aws.String(key), + } + return r.apiClient.HeadObject(ctx, input) +} + // getObjectMetadata returns the object's size in bytes, modification time if it exists. func (r *S3Client) getObjectMetadata(ctx context.Context, key string, keyMustExist bool) (int64, time.Time, *s3ArchiveInfo, error) { if key == "" { @@ -853,15 +1061,9 @@ func (r *S3Client) getObjectMetadata(ctx context.Context, key string, keyMustExi return 0, time.Time{}, nil, nil } - headObjectInput := &s3.HeadObjectInput{ - Bucket: aws.String(r.config.GetS3().Bucket), - Key: aws.String(key), - } - - resp, err := r.client.HeadObject(ctx, headObjectInput) + resp, err := r.headObject(ctx, key) if err != nil { - var apiErr smithy.APIError - if errors.As(err, &apiErr) { + if apiErr, ok := errors.AsType[smithy.APIError](err); ok { if apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "NoSuchKey" { return 0, time.Time{}, nil, os.ErrNotExist } @@ -895,7 +1097,7 @@ func (r *S3Client) createUpload(ctx context.Context, path string, mtime time.Tim } createMultipartUploadInput := &s3.CreateMultipartUploadInput{ - Bucket: aws.String(r.config.GetS3().Bucket), + Bucket: aws.String(r.s3Config.Bucket), Key: aws.String(path), Metadata: metadata, Tagging: tagging, @@ -904,7 +1106,7 @@ func (r *S3Client) createUpload(ctx context.Context, path string, mtime time.Tim createMultipartUploadInput.StorageClass = types.StorageClass(*storageClass) } - result, err := r.client.CreateMultipartUpload(ctx, createMultipartUploadInput) + result, err := r.apiClient.CreateMultipartUpload(ctx, createMultipartUploadInput) if err != nil { return "", err } @@ -914,11 +1116,11 @@ func (r *S3Client) createUpload(ctx context.Context, path string, mtime time.Tim func (r *S3Client) abortUpload(ctx context.Context, uploadID string, remotePath string) error { abortMultipartUploadInput := &s3.AbortMultipartUploadInput{ UploadId: aws.String(uploadID), - Bucket: aws.String(r.config.GetS3().Bucket), + Bucket: aws.String(r.s3Config.Bucket), Key: aws.String(remotePath), } - _, err := r.client.AbortMultipartUpload(ctx, abortMultipartUploadInput) + _, err := r.apiClient.AbortMultipartUpload(ctx, abortMultipartUploadInput) return err } @@ -939,7 +1141,7 @@ func (r *S3Client) finishUpload(ctx context.Context, uploadID string, remotePath }) completeMultipartUploadInput := &s3.CompleteMultipartUploadInput{ - Bucket: aws.String(r.config.GetS3().Bucket), + Bucket: aws.String(r.s3Config.Bucket), Key: aws.String(remotePath), UploadId: aws.String(uploadID), MultipartUpload: &types.CompletedMultipartUpload{ @@ -947,7 +1149,7 @@ func (r *S3Client) finishUpload(ctx context.Context, uploadID string, remotePath }, } - _, err := r.client.CompleteMultipartUpload(ctx, completeMultipartUploadInput) + _, err := r.apiClient.CompleteMultipartUpload(ctx, completeMultipartUploadInput) return err } @@ -1000,7 +1202,7 @@ func (r *S3Client) upload( } input := &s3.PutObjectInput{ - Bucket: aws.String(r.config.GetS3().Bucket), + Bucket: aws.String(r.s3Config.Bucket), Key: aws.String(remotePath), Body: filePart, ChecksumSHA256: aws.String(part.ChecksumSha256), @@ -1013,7 +1215,7 @@ func (r *S3Client) upload( input.StorageClass = types.StorageClass(*storageClass) } - resp, err := r.client.PutObject(ctx, input) + resp, err := r.apiClient.PutObject(ctx, input) if err != nil { return err @@ -1023,7 +1225,7 @@ func (r *S3Client) upload( } uploadPartReq := &s3.UploadPartInput{ - Bucket: aws.String(r.config.GetS3().Bucket), + Bucket: aws.String(r.s3Config.Bucket), Key: aws.String(remotePath), UploadId: aws.String(uploadID), PartNumber: aws.Int32(part.PartNumber), @@ -1031,7 +1233,8 @@ func (r *S3Client) upload( ChecksumSHA256: aws.String(part.ChecksumSha256), } - resp, err := r.client.UploadPart(ctx, uploadPartReq) + part.SetStarted(true) + resp, err := r.apiClient.UploadPart(ctx, uploadPartReq) if err != nil { return err } @@ -1055,17 +1258,19 @@ func (r *S3Client) download(ctx context.Context, path string, remotePath string, defer filePart.Close() getObjectInput := &s3.GetObjectInput{ - Bucket: aws.String(r.config.GetS3().Bucket), + Bucket: aws.String(r.s3Config.Bucket), Key: aws.String(remotePath), Range: aws.String(fmt.Sprintf("bytes=%d-%d", part.OffsetStart, part.OffsetStop)), } - resp, err := r.client.GetObject(ctx, getObjectInput) + resp, err := r.apiClient.GetObject(ctx, getObjectInput) if err != nil { return err } defer resp.Body.Close() + copiedBytes, err := io.Copy(filePart, resp.Body) + part.SetStarted(copiedBytes > 0) if err != nil { return err } diff --git a/common/rst/s3_test.go b/common/rst/s3_test.go index 107b1de2..94935bca 100644 --- a/common/rst/s3_test.go +++ b/common/rst/s3_test.go @@ -13,15 +13,13 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -// A client that can be used for methods that don't actually require interacting with a real S3 -// bucket. Note more complex unit tests would require reworking S3Client.client into an interface so -// it could be mocked. These could be based off the previous sync_tests that existed before the RST -// and Job packages were refactored: -// https://github.com/ThinkParQ/bee-remote/blob/6cdea09384f4dc4446fec61ab968565a7c649a65/internal/job/sync_test.go var testS3Client = &S3Client{ config: &flex.RemoteStorageTarget{ Policies: &flex.RemoteStorageTarget_Policies{}, }, + s3Config: &flex.RemoteStorageTarget_S3{ + Bucket: "test-bucket", + }, } func TestGenerateWorkRequests(t *testing.T) { diff --git a/common/rst/utils.go b/common/rst/utils.go new file mode 100644 index 00000000..88a8d1c0 --- /dev/null +++ b/common/rst/utils.go @@ -0,0 +1,10 @@ +package rst + +import "fmt" + +func appendError(accumulatedErr error, nextErr error) error { + if accumulatedErr == nil { + return nextErr + } + return fmt.Errorf("%w; %w", accumulatedErr, nextErr) +} diff --git a/common/rst/xtreemstore.go b/common/rst/xtreemstore.go new file mode 100644 index 00000000..538560a3 --- /dev/null +++ b/common/rst/xtreemstore.go @@ -0,0 +1,237 @@ +package rst + +import ( + "context" + "fmt" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + smithyhttp "github.com/aws/smithy-go/transport/http" + "github.com/thinkparq/beegfs-go/common/filesystem" + "github.com/thinkparq/protobuf/go/beeremote" + "github.com/thinkparq/protobuf/go/flex" +) + +type xtreemstoreS3Provider struct { + Provider + s3ApiClient + mountPoint filesystem.Provider +} + +var _ Provider = &xtreemstoreS3Provider{} + +type xtreemstoreS3BulkOperation byte + +const ( + xtreemstoreS3BulkOperationUnknown xtreemstoreS3BulkOperation = iota + xtreemstoreS3BulkOperationRetrieve +) + +func (o xtreemstoreS3BulkOperation) String() string { + switch o { + case xtreemstoreS3BulkOperationRetrieve: + return "bulk-retrieve" + default: + return "unknown" + } +} + +func parseBulkOperation(operation string) xtreemstoreS3BulkOperation { + switch operation { + case "bulk-retrieve": + return xtreemstoreS3BulkOperationRetrieve + default: + return xtreemstoreS3BulkOperationUnknown + } +} + +// newXtreemstore initializes an xtreemstore provider by reusing the S3 client implementation. +func newXtreemstore(ctx context.Context, rstConfig *flex.RemoteStorageTarget, mountPoint filesystem.Provider) (Provider, error) { + xtreemstore := rstConfig.GetXtreemstore() + if xtreemstore == nil || xtreemstore.GetS3() == nil { + return nil, fmt.Errorf("xtreemstore configuration must include s3 settings") + } + + wrapper := &xtreemstoreS3Provider{ + mountPoint: mountPoint, + } + s3Client, err := newS3WithOptions(ctx, rstConfig, xtreemstore.GetS3(), mountPoint, + withS3ApiClient(func(base s3ApiClient) s3ApiClient { + wrapper.s3ApiClient = base + return wrapper + }), + ) + if err != nil { + return nil, fmt.Errorf("unable to initialize xtreemstore provider: %w", err) + } + + wrapper.Provider = s3Client + return wrapper, nil +} + +func (x *xtreemstoreS3Provider) HeadObject(ctx context.Context, in *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + // Update xtreemstore head-object api request headers to include storage details. + optFns = append(optFns, func(options *s3.Options) { + options.APIOptions = append(options.APIOptions, smithyhttp.AddHeaderValue("x-amz-meta-xts-request-storage-details", "true")) + options.APIOptions = append(options.APIOptions, smithyhttp.AddHeaderValue("x-amz-optional-object-attributes", "RestoreStatus")) + }) + + return x.s3ApiClient.HeadObject(ctx, in, optFns...) +} + +func (x *xtreemstoreS3Provider) GenerateWorkRequests(ctx context.Context, lastJob *beeremote.Job, job *beeremote.Job, availableWorkers int) (requests []*flex.WorkRequest, err error) { + request := job.GetRequest() + defer func() { + if request.HasBulkInfo() { + bulkInfo := request.GetBulkInfo() + operation := parseBulkOperation(bulkInfo.Operation) + + switch operation { + case xtreemstoreS3BulkOperationRetrieve: + if err != nil { + if bulkErr := xtreemstoreS3BulkRetrieveMarkComplete(bulkInfo, x.GetConfig().GetId(), x.mountPoint.GetMountPath()); bulkErr != nil { + err = fmt.Errorf("%w: failed to mark bulk request complete: %w", err, bulkErr) + } + } else if bulkErr := xtreemstoreS3BulkRetrieveMarkReceived(bulkInfo, x.GetConfig().GetId(), x.mountPoint.GetMountPath()); bulkErr != nil { + err = fmt.Errorf("%w: failed to mark bulk request received: %w", err, bulkErr) + } + default: + err = fmt.Errorf("%w: unknown xtreemstore bulk operation %q, unable to mark request complete: %w", err, bulkInfo.Operation, ErrUnsupportedOpForRST) + } + } + }() + + requests, err = x.Provider.GenerateWorkRequests(ctx, lastJob, job, availableWorkers) + return +} + +func (x *xtreemstoreS3Provider) ExecuteWorkRequestPart(ctx context.Context, request *flex.WorkRequest, part *flex.Work_Part) (err error) { + defer func() { + if request.HasBulkInfo() { + bulkInfo := request.GetBulkInfo() + operation := parseBulkOperation(bulkInfo.Operation) + + switch operation { + case xtreemstoreS3BulkOperationRetrieve: + if err != nil { + // It's safe to mark the same request as complete for multiple work requests. + if bulkErr := xtreemstoreS3BulkRetrieveMarkComplete(bulkInfo, x.GetConfig().GetId(), x.mountPoint.GetMountPath()); bulkErr != nil { + err = fmt.Errorf("%w: failed to mark bulk request complete: %w", err, bulkErr) + } + } + default: + err = fmt.Errorf("%w: unknown xtreemstore bulk operation %q, unable to mark request complete: %w", err, bulkInfo.Operation, ErrUnsupportedOpForRST) + } + } + }() + + err = x.Provider.ExecuteWorkRequestPart(ctx, request, part) + return +} + +func (x *xtreemstoreS3Provider) IsWorkRequestReady(ctx context.Context, request *flex.WorkRequest) (ready bool, delay time.Duration, err error) { + if request.HasBulkInfo() { + bulkInfo := request.GetBulkInfo() + operation := parseBulkOperation(bulkInfo.Operation) + defer func() { + switch operation { + case xtreemstoreS3BulkOperationRetrieve: + if err != nil { + // It's safe to mark the same request as complete for multiple work requests. + if bulkErr := xtreemstoreS3BulkRetrieveMarkComplete(bulkInfo, x.GetConfig().GetId(), x.mountPoint.GetMountPath()); bulkErr != nil { + err = fmt.Errorf("%w: failed to mark bulk request complete: %w", err, bulkErr) + } + } + default: + // err already explains the operation is unsupported; there's nothing to mark complete. + } + }() + + switch operation { + case xtreemstoreS3BulkOperationRetrieve: + if bulkErr := xtreemstoreS3BulkRetrieveError(bulkInfo, x.GetConfig().GetId(), x.mountPoint.GetMountPath()); bulkErr != nil { + // Bulk operation requests must be ready before they are sent, so either an error occurred + // or the bulk request was aborted. For a bulk retrieve operation, the resource was + // retrieved but removed from the tape buffer before the download. + err = fmt.Errorf("bulk %s operation failed: %w", bulkInfo.Operation, bulkErr) + } else { + ready = true + } + default: + err = fmt.Errorf("failed to determine bulk request readiness for operation %q: %w", bulkInfo.Operation, ErrUnsupportedOpForRST) + } + } else { + ready, delay, err = x.Provider.IsWorkRequestReady(ctx, request) + } + + return +} + +func (x *xtreemstoreS3Provider) CompleteWorkRequests(ctx context.Context, job *beeremote.Job, workResults []*flex.Work, abort bool) (err error) { + request := job.GetRequest() + defer func() { + if request.HasBulkInfo() { + bulkInfo := request.GetBulkInfo() + operation := parseBulkOperation(bulkInfo.Operation) + + switch operation { + case xtreemstoreS3BulkOperationRetrieve: + if bulkErr := xtreemstoreS3BulkRetrieveMarkComplete(bulkInfo, x.GetConfig().GetId(), x.mountPoint.GetMountPath()); bulkErr != nil { + if err != nil { + err = fmt.Errorf("%w; failed to mark bulk request complete: %w", err, bulkErr) + } + } + default: + // err already explains the operation is unsupported; there's nothing to mark complete. + } + } + }() + + err = x.Provider.CompleteWorkRequests(ctx, job, workResults, abort) + return +} + +func (x *xtreemstoreS3Provider) IncludeRequestInBulkOperation(ctx context.Context, request *beeremote.JobRequest) (include bool, operation string) { + if !request.HasSync() { + return + } + + sync := request.GetSync() + lockedInfo := sync.GetLockedInfo() + if lockedInfo == nil { + return + } + + if lockedInfo.IsArchived { + include = true + operation = xtreemstoreS3BulkOperationRetrieve.String() + } + return +} + +func (x *xtreemstoreS3Provider) OpenBulkOperation(ctx context.Context, stateMountPath string, operation string) (clientBulkOperation, error) { + switch parseBulkOperation(operation) { + case xtreemstoreS3BulkOperationRetrieve: + manager := x.newXtreemstoreS3BulkRetrieveManager(stateMountPath, operation) + if err := manager.openState(); err != nil { + manager.closeState() + return nil, fmt.Errorf("failed to open bulk operation: %w", err) + } + + return manager, nil + default: + return nil, ErrUnsupportedOpForRST + } +} + +func (x *xtreemstoreS3Provider) newXtreemstoreS3BulkRetrieveManager(stateMountPath string, operation string) *xtreemstoreS3BulkRetrieveManager { + return &xtreemstoreS3BulkRetrieveManager{ + s3ApiClient: x, + rstId: x.GetConfig().GetId(), + bucket: x.GetConfig().GetXtreemstore().S3.Bucket, + mountPath: x.mountPoint.GetMountPath(), + stateMountPath: stateMountPath, + operation: operation, + state: &xtreemstoreS3BulkRetrieveManagerState{}, + } +} diff --git a/common/rst/xtreemstore_test.go b/common/rst/xtreemstore_test.go new file mode 100644 index 00000000..9857b4b5 --- /dev/null +++ b/common/rst/xtreemstore_test.go @@ -0,0 +1,195 @@ +package rst + +import ( + "context" + "os" + "path" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/thinkparq/beegfs-go/common/filesystem" + "github.com/thinkparq/protobuf/go/beeremote" + "github.com/thinkparq/protobuf/go/flex" +) + +// stubMountPoint fakes a filesystem.Provider that points at a real on-disk directory, since +// xtreemstoreS3BulkRetrieveManager reads/writes its state files with the os package directly +// rather than through the Provider interface. +type stubMountPoint struct { + filesystem.Provider + mountPath string +} + +func (s stubMountPoint) GetMountPath() string { + return s.mountPath +} + +func TestIncludeRequestInBulkOperation(t *testing.T) { + x := &xtreemstoreS3Provider{} + + tests := []struct { + name string + request *beeremote.JobRequest + wantInclude bool + wantOperation string + }{ + { + name: "non-sync request is never included", + request: &beeremote.JobRequest{Type: &beeremote.JobRequest_Mock{Mock: &flex.MockJob{}}}, + wantInclude: false, + }, + { + name: "sync request without locked info is never included", + request: &beeremote.JobRequest{Type: &beeremote.JobRequest_Sync{Sync: &flex.SyncJob{}}}, + wantInclude: false, + }, + { + name: "sync request that is not archived is not included", + request: &beeremote.JobRequest{Type: &beeremote.JobRequest_Sync{Sync: &flex.SyncJob{LockedInfo: &flex.JobLockedInfo{IsArchived: false}}}}, + wantInclude: false, + }, + { + name: "archived sync request is included in the bulk-retrieve operation", + request: &beeremote.JobRequest{Type: &beeremote.JobRequest_Sync{Sync: &flex.SyncJob{LockedInfo: &flex.JobLockedInfo{IsArchived: true}}}}, + wantInclude: true, + wantOperation: "bulk-retrieve", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + include, operation := x.IncludeRequestInBulkOperation(context.Background(), tt.request) + assert.Equal(t, tt.wantInclude, include) + assert.Equal(t, tt.wantOperation, operation) + }) + } +} + +func TestXtreemstoreProviderIsWorkRequestReady(t *testing.T) { + t.Run("non-sync request is rejected", func(t *testing.T) { + x := &xtreemstoreS3Provider{} + ready, _, err := x.IsWorkRequestReady(context.Background(), &flex.WorkRequest{Type: &flex.WorkRequest_Mock{Mock: &flex.MockJob{}}}) + assert.False(t, ready) + assert.ErrorIs(t, err, ErrReqAndRSTTypeMismatch) + }) + + t.Run("bulk request with no recorded error is ready without consulting the Provider", func(t *testing.T) { + mountPath := t.TempDir() + mockProvider := &MockClient{} + x := &xtreemstoreS3Provider{ + Provider: mockProvider, + mountPoint: stubMountPoint{mountPath: mountPath}, + } + mockProvider.On("GetConfig").Return(&flex.RemoteStorageTarget{Id: 1}) + + request := &flex.WorkRequest{ + Type: &flex.WorkRequest_Sync{Sync: &flex.SyncJob{}}, + BulkInfo: &flex.BulkJobRequestInfo{StateMountPath: "state", Operation: "bulk-retrieve"}, + } + ready, _, err := x.IsWorkRequestReady(context.Background(), request) + require.NoError(t, err) + assert.True(t, ready) + mockProvider.AssertNotCalled(t, "IsWorkRequestReady", mock.Anything) + }) + + t.Run("bulk request with a recorded error is not ready and surfaces the error", func(t *testing.T) { + mountPath := t.TempDir() + mockProvider := &MockClient{} + x := &xtreemstoreS3Provider{ + Provider: mockProvider, + mountPoint: stubMountPoint{mountPath: mountPath}, + } + mockProvider.On("GetConfig").Return(&flex.RemoteStorageTarget{Id: 1}) + + bulkInfo := &flex.BulkJobRequestInfo{StateMountPath: "state", Operation: "bulk-retrieve"} + errDir := path.Join(mountPath, bulkInfo.StateMountPath, bulkInfo.Operation) + require.NoError(t, os.MkdirAll(errDir, 0o700)) + require.NoError(t, os.WriteFile(path.Join(errDir, "errors"), []byte("object no longer exists"), 0o600)) + + request := &flex.WorkRequest{ + Type: &flex.WorkRequest_Sync{Sync: &flex.SyncJob{}}, + BulkInfo: bulkInfo, + } + ready, _, err := x.IsWorkRequestReady(context.Background(), request) + assert.False(t, ready) + assert.ErrorContains(t, err, "object no longer exists") + }) + + t.Run("non-bulk request delegates entirely to the embedded Provider", func(t *testing.T) { + mockProvider := &MockClient{} + x := &xtreemstoreS3Provider{Provider: mockProvider} + + request := &flex.WorkRequest{Type: &flex.WorkRequest_Sync{Sync: &flex.SyncJob{}}} + mockProvider.On("IsWorkRequestReady", request).Return(true, 5*time.Second, nil) + + ready, delay, err := x.IsWorkRequestReady(context.Background(), request) + require.NoError(t, err) + assert.True(t, ready) + assert.Equal(t, 5*time.Second, delay) + mockProvider.AssertExpectations(t) + }) +} + +func TestXtreemstoreProviderCompleteWorkRequests(t *testing.T) { + t.Run("non-sync request is rejected", func(t *testing.T) { + x := &xtreemstoreS3Provider{} + job := &beeremote.Job{Request: &beeremote.JobRequest{Type: &beeremote.JobRequest_Mock{Mock: &flex.MockJob{}}}} + err := x.CompleteWorkRequests(context.Background(), job, nil, false) + assert.ErrorIs(t, err, ErrReqAndRSTTypeMismatch) + }) + + t.Run("bulk request marks the job complete and still delegates to the Provider", func(t *testing.T) { + mountPath := t.TempDir() + mockProvider := &MockClient{} + x := &xtreemstoreS3Provider{ + Provider: mockProvider, + mountPoint: stubMountPoint{mountPath: mountPath}, + } + mockProvider.On("GetConfig").Return(&flex.RemoteStorageTarget{Id: 1}) + + bulkInfo := &flex.BulkJobRequestInfo{StateMountPath: "state", Operation: "bulk-retrieve", JobIndex: 0} + statusDir := path.Join(mountPath, bulkInfo.StateMountPath, bulkInfo.Operation) + require.NoError(t, os.MkdirAll(statusDir, 0o700)) + require.NoError(t, os.WriteFile(path.Join(statusDir, "status"), xtreemstoreS3BulkRequestAdded.Bytes(), 0o600)) + + job := &beeremote.Job{Request: &beeremote.JobRequest{ + Type: &beeremote.JobRequest_Sync{Sync: &flex.SyncJob{}}, + BulkInfo: bulkInfo, + }} + mockProvider.On("CompleteWorkRequests", job, mock.Anything, false).Return(nil) + + err := x.CompleteWorkRequests(context.Background(), job, nil, false) + require.NoError(t, err) + mockProvider.AssertExpectations(t) + + status, err := os.ReadFile(path.Join(statusDir, "status")) + require.NoError(t, err) + assert.Equal(t, xtreemstoreS3BulkRequestComplete.Bytes(), status) + }) + + t.Run("a failure marking the bulk job complete is joined with the Provider's own error", func(t *testing.T) { + mountPath := t.TempDir() + mockProvider := &MockClient{} + x := &xtreemstoreS3Provider{ + Provider: mockProvider, + mountPoint: stubMountPoint{mountPath: mountPath}, + } + mockProvider.On("GetConfig").Return(&flex.RemoteStorageTarget{Id: 1}) + + // No status file was ever created for this bulk operation, so marking it complete fails. + bulkInfo := &flex.BulkJobRequestInfo{StateMountPath: "state", Operation: "bulk-retrieve", JobIndex: 0} + job := &beeremote.Job{Request: &beeremote.JobRequest{ + Type: &beeremote.JobRequest_Sync{Sync: &flex.SyncJob{}}, + BulkInfo: bulkInfo, + }} + mockProvider.On("CompleteWorkRequests", job, mock.Anything, false).Return(assert.AnError) + + err := x.CompleteWorkRequests(context.Background(), job, nil, false) + require.Error(t, err) + assert.ErrorContains(t, err, "failed to mark bulk request complete") + assert.ErrorIs(t, err, assert.AnError) + }) +} diff --git a/common/rst/xtreemstorebulkretrieve.go b/common/rst/xtreemstorebulkretrieve.go new file mode 100644 index 00000000..527de223 --- /dev/null +++ b/common/rst/xtreemstorebulkretrieve.go @@ -0,0 +1,896 @@ +package rst + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "path" + "strings" + "sync" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + smithyhttp "github.com/aws/smithy-go/transport/http" + "github.com/thinkparq/protobuf/go/beeremote" + "github.com/thinkparq/protobuf/go/flex" + "golang.org/x/sync/errgroup" +) + +const ( + XTS_SYSTEM = ".xts-system" + XTS_SYSTEM_ERRORS = XTS_SYSTEM + "/errors" + XTS_SYSTEM_RETRIEVE_SESSION = XTS_SYSTEM + "/retrieve-session.json" + XTS_SYSTEM_RETRIEVE_BATCH_LIST = XTS_SYSTEM + "/retrieve-batch-list.json" + XTS_SYSTEM_RETRIEVE_BATCH_FMT = XTS_SYSTEM + "/retrieve-batch-%d.json" + + reschedulePoolingDelay = 1 * time.Minute // This should be configurable + rescheduleMaxDelay = 5 * time.Minute // This should be configurable +) + +var ( + ErrActiveRetrieveSessionAlreadyExists = errors.New("active retrieve-session already exists") +) + +type xtreemstoreS3BulkRetrieveManager struct { + s3ApiClient + rstId uint32 + bucket string + operation string + mountPath string + stateMountPath string + state *xtreemstoreS3BulkRetrieveManagerState + includedJobs int64 + statusHandle *os.File + recordHandle *os.File +} + +var _ clientBulkOperation = &xtreemstoreS3BulkRetrieveManager{} + +type xtreemstoreS3BulkRetrieveManagerState struct { + SessionRetrieveId string `json:"active-retrieve-id"` + SessionJobStart int64 `json:"active-job-start"` + SessionJobEnd int64 `json:"active-job-end"` +} + +type xtreemstoreS3BulkRetrieveSessionInfo struct { + Active bool `json:"active"` + RetrieveId string `json:"retrieve-id"` + Started time.Time `json:"started"` +} + +type xtreemstoreS3BulkRetrieveBatchInfo struct { + Number int64 `json:"number"` + Objects int64 `json:"objects"` + Size int64 `json:"size"` +} + +type xtreemstoreS3BulkRetrieveRequest struct { + Ids []string `json:"ids,omitempty"` + BucketRetrieve bool `json:"bucket-retrieve,omitempty"` +} + +type xtreemstoreS3BulkRequestStatus byte + +const ( + // Request has been added to bulk operation. + xtreemstoreS3BulkRequestAdded xtreemstoreS3BulkRequestStatus = iota + // Request has been sent from the bulk operation and is waiting for GenerateWorkRequests to + // acknowledge by marking it xtreemstoreS3BulkRequestReceived. + xtreemstoreS3BulkRequestSent + // Request has been received by GenerateWorkRequests. + xtreemstoreS3BulkRequestReceived + // Request has been completed from the perspective of the bulk operation but has not been + // acknowledged by the bulk operation yet. + xtreemstoreS3BulkRequestComplete + // Request has been completed and bulk operation has acknowledge the completion. + xtreemstoreS3BulkRequestCompleteAck +) + +func (s xtreemstoreS3BulkRequestStatus) Bytes() []byte { + return []byte{byte(s)} +} + +type xtreemstoreS3BulkStatuses struct { + jobStatuses []byte + jobCount int64 + offset int64 // this will correspond the xtreemstoreS3BulkRetrieveManager.state.ActiveJobStart at the time retrieved +} + +func (s *xtreemstoreS3BulkStatuses) Get(jobIndex int64) (status xtreemstoreS3BulkRequestStatus, err error) { + if jobIndex < s.offset { + err = fmt.Errorf("invalid index for active session") + return + } + + statusesJobIndex := jobIndex - s.offset + if statusesJobIndex >= int64(len(s.jobStatuses)) { + err = fmt.Errorf("invalid index for active session") + return + } + return xtreemstoreS3BulkRequestStatus(s.jobStatuses[statusesJobIndex]), nil +} + +func (s *xtreemstoreS3BulkStatuses) All() []xtreemstoreS3BulkRequestStatus { + statuses := make([]xtreemstoreS3BulkRequestStatus, s.jobCount) + for jobIndex := range s.jobCount { + // Ignore status error since the jobIndex is valid. + status, _ := s.Get(jobIndex) + statuses[jobIndex] = status + } + + return statuses +} + +// xtreemstoreS3BulkRetrieveMarkReceived marks a request sent by a bulk operation as complete. +func xtreemstoreS3BulkRetrieveMarkReceived(bulkInfo *flex.BulkJobRequestInfo, rstId uint32, mountPath string) error { + manager := &xtreemstoreS3BulkRetrieveManager{ + rstId: rstId, + mountPath: mountPath, + stateMountPath: bulkInfo.StateMountPath, + operation: bulkInfo.Operation, + } + return manager.MarkReceived(bulkInfo.JobIndex) +} + +// xtreemstoreS3BulkRetrieveMarkComplete marks a request sent by a bulk operation as complete. +func xtreemstoreS3BulkRetrieveMarkComplete(bulkInfo *flex.BulkJobRequestInfo, rstId uint32, mountPath string) error { + manager := &xtreemstoreS3BulkRetrieveManager{ + rstId: rstId, + mountPath: mountPath, + stateMountPath: bulkInfo.StateMountPath, + operation: bulkInfo.Operation, + } + return manager.MarkComplete(bulkInfo.JobIndex) +} + +// xtreemstoreS3BulkRetrieveError retrieves any bulk operation errors. If no errors were found then +// nil will be returned. +func xtreemstoreS3BulkRetrieveError(bulkInfo *flex.BulkJobRequestInfo, rstId uint32, mountPath string) error { + m := &xtreemstoreS3BulkRetrieveManager{ + rstId: rstId, + mountPath: mountPath, + stateMountPath: bulkInfo.StateMountPath, + operation: bulkInfo.Operation, + } + message, err := os.ReadFile(m.getErrorsPath()) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("unable to retrieve bulk operation error message for %q: %w", bulkInfo.Operation, err) + } + return fmt.Errorf("%s", message) +} + +func (m *xtreemstoreS3BulkRetrieveManager) AddRequest(ctx context.Context, request *beeremote.JobRequest) (err error) { + if !request.HasSync() { + return ErrReqAndRSTTypeMismatch + } + if !request.HasBulkInfo() { + return fmt.Errorf("missing request bulkInfo") + } + request.GetBulkInfo().SetJobIndex(m.includedJobs) + + if _, err = m.statusHandle.Write(xtreemstoreS3BulkRequestAdded.Bytes()); err != nil { + return + } + + remotePath := request.GetSync().GetRemotePath() + if m.includedJobs == 0 { + _, err = m.recordHandle.WriteString(remotePath) + } else { + _, err = m.recordHandle.WriteString("\n" + remotePath) + } + + m.includedJobs++ + return +} + +func (m *xtreemstoreS3BulkRetrieveManager) Execute(ctx context.Context) (walkCh <-chan *BulkStreamPathResult, getResults BulkExecuteResultFn, err error) { + var reschedule bool + var delay time.Duration + var executeErr error + + executeWalkCh := make(chan *BulkStreamPathResult, 128) + + wg := sync.WaitGroup{} + wg.Go(func() { + defer close(executeWalkCh) + reschedule, delay, executeErr = m.execute(ctx, executeWalkCh) + }) + + getResults = func() *SchedulingResult { + wg.Wait() + return &SchedulingResult{ + Reschedule: reschedule, + Delay: delay, + Err: executeErr, + } + } + return executeWalkCh, getResults, nil +} + +// Cancel releases any xtreemstore-side resources reserved for this bulk operation (the active +// retrieve-session and its batches, if any) and records reason to the errors file so any request +// that already passed IsWorkRequestReady can see why the operation was cancelled. It deliberately +// does not resolve individual jobIndexes or delete local state: +// - Requests still Added were never handed off anywhere else, so nothing downstream needs to hear +// about them; they're simply dropped. +// - Requests already Sent/Received depend on their own Job's normal lifecycle +// (IsWorkRequestReady, ExecuteWorkRequestPart, or CompleteWorkRequests) to resolve, and that Job +// may still be in flight. Deleting local state out from under it here would make its own +// mark-complete calls fail (or worse, collide with a future reuse of these files). Local state is +// only removed via Destroy, once the owning builder job itself is torn down for good. +// - Requests already Complete/CompleteAck are already terminal and are left as-is. +// +// reason is always returned (joined with any error releasing xtreemstore resources) so callers treat +// a cancelled bulk operation as a failure rather than a clean success. +func (m *xtreemstoreS3BulkRetrieveManager) Cancel(ctx context.Context, reason error) (walkCh <-chan *BulkStreamPathResult, wait BulkCancelResultFn, err error) { + cancelWalkCh := make(chan *BulkStreamPathResult) + + g, ctx := errgroup.WithContext(ctx) + g.Go(func() error { + defer close(cancelWalkCh) + + if err := m.recordError(reason); err != nil { + return appendError(reason, fmt.Errorf("failed to record cancellation reason: %w", err)) + } + + sessionInfo, err := m.getSessionInfo(ctx) + if err != nil { + return appendError(reason, fmt.Errorf("unable to determine whether retrieve-session is active: %w", err)) + } + + if sessionInfo.Active && sessionInfo.RetrieveId == m.state.SessionRetrieveId { + batchInfos, err := m.getSessionBatchInfo(ctx) + if err != nil { + return appendError(reason, fmt.Errorf("failed to get retrieve-session batch info: %w", err)) + } + + for _, batchInfo := range batchInfos { + if err := m.deleteSessionBatch(ctx, batchInfo); err != nil { + return appendError(reason, fmt.Errorf("failed to delete retrieve-session batch: %w", err)) + } + } + + if err := m.destroyRetrieveSession(ctx); err != nil { + return appendError(reason, fmt.Errorf("failed to deactivate retrieve-session: %w", err)) + } + } + + return reason + }) + + return cancelWalkCh, g.Wait, nil +} + +// recordError writes reason to the operation's shared errors file, overwriting any previous content. +func (m *xtreemstoreS3BulkRetrieveManager) recordError(reason error) error { + return os.WriteFile(m.getErrorsPath(), []byte(reason.Error()), 0o600) +} + +func (m *xtreemstoreS3BulkRetrieveManager) Close(ctx context.Context) error { + return m.closeState() +} + +func (m *xtreemstoreS3BulkRetrieveManager) Destroy(ctx context.Context) error { + return m.deleteState() +} + +func (m *xtreemstoreS3BulkRetrieveManager) deleteState() error { + var errs []error + errs = append(errs, removeIfExists(m.getStatusPath())) + errs = append(errs, removeIfExists(m.getRecordPath())) + errs = append(errs, removeIfExists(m.getErrorsPath())) + errs = append(errs, removeIfExists(m.getManagerPath())) + return errors.Join(errs...) +} + +// removeIfExists removes the file at path, returning nil if it does not exist since the state +// files aren't guaranteed to have been created yet when deleteState() is called. +func removeIfExists(path string) error { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil +} + +func (m *xtreemstoreS3BulkRetrieveManager) execute(ctx context.Context, walkCh chan<- *BulkStreamPathResult) (reschedule bool, delay time.Duration, err error) { + for { + if ready, err := m.ensureSessionActive(ctx); err != nil { + return false, 0, err + } else if !ready { + return true, rescheduleMaxDelay, nil + } + + if batchesComplete, err := m.processSessionBatches(ctx, walkCh); err != nil { + return false, 0, err + } else if !batchesComplete { + return true, reschedulePoolingDelay, nil + } + + if err = m.destroyRetrieveSession(ctx); err != nil { + return false, 0, fmt.Errorf("retrieve-session completed successfully, but the active session could not be destroyed and manual intervention is required: %w", err) + } + + if m.includedJobs == m.state.SessionJobEnd { + // no more requests were add + break + } + } + + return false, 0, nil +} + +func (m *xtreemstoreS3BulkRetrieveManager) ensureSessionActive(ctx context.Context) (ready bool, err error) { + sessionInfo, sessionInfoErr := m.getSessionInfo(ctx) + if sessionInfoErr != nil { + err = fmt.Errorf("unable to determine whether retrieve-session is active: %w", sessionInfoErr) + return + } + + if sessionInfo.Active { + ready = sessionInfo.RetrieveId == m.state.SessionRetrieveId + } else if startSessionErr := m.startSession(ctx); startSessionErr != nil { + if !errors.Is(startSessionErr, ErrActiveRetrieveSessionAlreadyExists) { + err = fmt.Errorf("failed to start retrieve-session: %w", startSessionErr) + } + } else { + ready = true + } + return +} + +func (m *xtreemstoreS3BulkRetrieveManager) processSessionBatches(ctx context.Context, walkCh chan<- *BulkStreamPathResult) (success bool, err error) { + batchInfos, err := m.getSessionBatchInfo(ctx) + if err != nil { + return false, fmt.Errorf("failed to load retrieve-session batch info: %w", err) + } + + defer func() { + m.saveManagerState() + }() + + for _, batchInfo := range batchInfos { + if batchComplete, err := m.processSessionBatch(ctx, walkCh, batchInfo); err != nil || !batchComplete { + return false, err + } + } + return true, nil +} + +// processSessionBatch processes the retrieve-session batch and returns whether it completed. +func (m *xtreemstoreS3BulkRetrieveManager) processSessionBatch( + ctx context.Context, + walkCh chan<- *BulkStreamPathResult, + batchInfo xtreemstoreS3BulkRetrieveBatchInfo, +) (allComplete bool, err error) { + activeRecordMap, err := m.getActiveRecordsMap() + if err != nil { + return false, fmt.Errorf("failed to get record mappings for the active retrieve-session: %w", err) + } + + var activeStatuses *xtreemstoreS3BulkStatuses + if activeStatuses, err = m.getActiveSessionStatuses(); err != nil { + return false, fmt.Errorf("failed to get request statuses for active retrieve-session: %w", err) + } + + var keys []string + if keys, err = m.getSessionBatchKeys(ctx, batchInfo); err != nil { + return false, fmt.Errorf("failed to retrieve batch keys: %w", err) + } + + allComplete = true + for _, key := range keys { + jobIndex, ok := activeRecordMap[key] + if !ok { + return false, fmt.Errorf("unable to determine status for key: %s", key) + } + status, statusErr := activeStatuses.Get(jobIndex) + if statusErr != nil { + return false, fmt.Errorf("unable to determine status: %w", statusErr) + } + + if done, err := m.processSessionBatchKey(ctx, walkCh, key, jobIndex, status); err != nil { + return false, err + } else if !done { + allComplete = false + } + } + + if allComplete { + if err = m.deleteSessionBatch(ctx, batchInfo); err != nil { + err = fmt.Errorf("failed to delete retrieve-session batch: %w", err) + } + } + return +} + +// processSessionBatchKey advances one key's bulk-retrieve state machine a step and reports whether +// it has reached a terminal (Complete/CompleteAck) state. Reporting per-key rather than mutating a +// shared flag from inside the switch means a batch can never be marked complete just because +// whichever key happened to be checked last was already done. +func (m *xtreemstoreS3BulkRetrieveManager) processSessionBatchKey( + ctx context.Context, + walkCh chan<- *BulkStreamPathResult, + key string, + jobIndex int64, + status xtreemstoreS3BulkRequestStatus, +) (done bool, err error) { + result := &BulkStreamPathResult{ + Path: key, + RstId: m.rstId, + BulkInfo: &flex.BulkJobRequestInfo{StateMountPath: m.stateMountPath, Operation: m.operation, JobIndex: jobIndex}, + } + + switch status { + // A request that is xtreemstoreS3BulkRequestSent means that remote never received the the job request as the result of a sync worker crash; otherwise, + // the status would already be xtreemstoreS3BulkRequestReceived. + case xtreemstoreS3BulkRequestAdded, xtreemstoreS3BulkRequestSent: + ready, err := m.isObjectReadyForDownload(ctx, key) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return false, fmt.Errorf("failed to determine restore state. Record: %s, Status: %v: %w", key, status, err) + } + result.Err = &RequestCancelError{Reason: fmt.Errorf("object no longer exists")} + walkCh <- result + if err := m.MarkCompleteAck(jobIndex); err != nil { + return false, fmt.Errorf("remote object no longer exists but failed to mark bulk job request as complete. Record: %s, Status: %v", key, status) + } + return true, nil + } + if !ready { + return false, nil + } + walkCh <- result + if err := m.MarkSent(jobIndex); err != nil { + return false, fmt.Errorf("failed to mark bulk job request as complete. Record: %s, Status: %v: %w", key, status, err) + } + return false, nil + case xtreemstoreS3BulkRequestReceived: + return false, nil + case xtreemstoreS3BulkRequestComplete: + if err := m.MarkCompleteAck(jobIndex); err != nil { + return false, fmt.Errorf("failed to mark bulk job request as complete and acknowledged. Record: %s, Status: %v", key, status) + } + return true, nil + case xtreemstoreS3BulkRequestCompleteAck: + return true, nil + default: + result.Err = fmt.Errorf("unexpected record status. Record: %s, Status: %v", key, status) + walkCh <- result + if err := m.MarkCompleteAck(jobIndex); err != nil { + return false, fmt.Errorf("failed to mark bulk job request as complete. Record: %s, Status: %v: %w", key, status, err) + } + return true, nil + } +} + +func (m *xtreemstoreS3BulkRetrieveManager) isObjectReadyForDownload(ctx context.Context, key string) (bool, error) { + input := &s3.HeadObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(key), + } + resp, err := m.s3ApiClient.HeadObject(ctx, input) + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) && (apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "NoSuchKey") { + return false, os.ErrNotExist + } + return false, fmt.Errorf("head object for key %q: %w", key, err) + } + + switch resp.StorageClass { + case types.StorageClassStandard: + return true, nil + case types.StorageClassGlacier: + return resp.Restore != nil && strings.Contains(*resp.Restore, `ongoing-request="false"`), nil + default: + return false, fmt.Errorf("unexpected storage class, %s", resp.StorageClass) + } + +} + +func (m *xtreemstoreS3BulkRetrieveManager) loadManagerState() error { + f, err := os.OpenFile(m.getManagerPath(), os.O_RDONLY, os.FileMode(0600)) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + *m.state = xtreemstoreS3BulkRetrieveManagerState{} + } else { + defer f.Close() + if err := json.NewDecoder(f).Decode(m.state); err != nil { + return err + } + } + + // includedJobs is reconstructed from the status file rather than persisted in manager.json, so + // it stays correct even when manager.json doesn't exist (or predates the status file). It is the + // sole source of truth for the next JobIndex to assign, so this must run on every load. + if statusInfo, err := os.Stat(m.getStatusPath()); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + m.includedJobs = 0 + } else { + m.includedJobs = statusInfo.Size() + } + + return nil +} + +func (m *xtreemstoreS3BulkRetrieveManager) saveManagerState() (err error) { + var f *os.File + if f, err = os.OpenFile(m.getManagerPath(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(0600)); err != nil { + return + } + + defer func() { + if closeErr := f.Close(); closeErr != nil { + err = errors.Join(err, closeErr) + } + }() + + err = json.NewEncoder(f).Encode(m.state) + return +} + +func (m *xtreemstoreS3BulkRetrieveManager) openState() (err error) { + if err = m.loadManagerState(); err != nil { + err = fmt.Errorf("failed to load manager state: %w", err) + return + } + + if err = os.MkdirAll(m.getStateMountPath(), 0o700); err != nil { + return + } + + if m.statusHandle, err = os.OpenFile(m.getStatusPath(), os.O_WRONLY|os.O_CREATE|os.O_APPEND, os.FileMode(0600)); err != nil { + return + } + // statusHandle is already assigned, so a failure here will be cleaned up by the caller via closeState(). + if m.recordHandle, err = os.OpenFile(m.getRecordPath(), os.O_WRONLY|os.O_CREATE|os.O_APPEND, os.FileMode(0600)); err != nil { + return + } + return +} + +func (m *xtreemstoreS3BulkRetrieveManager) closeState() (err error) { + if m.statusHandle != nil { + err = errors.Join(err, m.statusHandle.Close()) + } + if m.recordHandle != nil { + err = errors.Join(err, m.recordHandle.Close()) + } + return +} + +func (m *xtreemstoreS3BulkRetrieveManager) MarkSent(jobIndex int64) error { + return m.markJobStatus(xtreemstoreS3BulkRequestSent, jobIndex) +} + +func (m *xtreemstoreS3BulkRetrieveManager) MarkReceived(jobIndex int64) error { + return m.markJobStatus(xtreemstoreS3BulkRequestReceived, jobIndex) +} + +func (m *xtreemstoreS3BulkRetrieveManager) MarkComplete(jobIndex int64) error { + return m.markJobStatus(xtreemstoreS3BulkRequestComplete, jobIndex) +} + +func (m *xtreemstoreS3BulkRetrieveManager) MarkCompleteAck(jobIndex int64) error { + return m.markJobStatus(xtreemstoreS3BulkRequestCompleteAck, jobIndex) +} + +func (m *xtreemstoreS3BulkRetrieveManager) markJobStatus(status xtreemstoreS3BulkRequestStatus, jobIndex int64) error { + f, err := os.OpenFile(m.getStatusPath(), os.O_WRONLY, os.FileMode(0600)) + if err != nil { + return err + } + defer func() { + if closeErr := f.Close(); closeErr != nil { + err = errors.Join(err, closeErr) + } + }() + + _, err = f.WriteAt(status.Bytes(), jobIndex) + return err +} + +func (m *xtreemstoreS3BulkRetrieveManager) getStateMountPath() string { + return path.Join(m.mountPath, m.stateMountPath, m.operation) +} + +func (m *xtreemstoreS3BulkRetrieveManager) getStatusPath() string { + return path.Join(m.getStateMountPath(), "status") +} + +func (m *xtreemstoreS3BulkRetrieveManager) getRecordPath() string { + return path.Join(m.getStateMountPath(), "record") +} + +func (m *xtreemstoreS3BulkRetrieveManager) getErrorsPath() string { + return path.Join(m.getStateMountPath(), "errors") +} + +func (m *xtreemstoreS3BulkRetrieveManager) getManagerPath() string { + return path.Join(m.getStateMountPath(), "manager.json") +} + +func (m *xtreemstoreS3BulkRetrieveManager) getSessionInfo(ctx context.Context) (*xtreemstoreS3BulkRetrieveSessionInfo, error) { + getObjectInput := &s3.GetObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(XTS_SYSTEM_RETRIEVE_SESSION), + } + + resp, err := m.s3ApiClient.GetObject(ctx, getObjectInput) + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) && (apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "NoSuchKey") { + return &xtreemstoreS3BulkRetrieveSessionInfo{}, nil + } + return nil, err + } + defer resp.Body.Close() + + info := &xtreemstoreS3BulkRetrieveSessionInfo{} + if err := json.NewDecoder(resp.Body).Decode(info); err != nil { + return nil, err + } + return info, nil +} + +func (m *xtreemstoreS3BulkRetrieveManager) getSessionBatchInfo(ctx context.Context) ([]xtreemstoreS3BulkRetrieveBatchInfo, error) { + getObjectInput := &s3.GetObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(XTS_SYSTEM_RETRIEVE_BATCH_LIST), + } + + resp, err := m.s3ApiClient.GetObject(ctx, getObjectInput) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var info []xtreemstoreS3BulkRetrieveBatchInfo + if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + return nil, fmt.Errorf("decode retrieve batch info: %w", err) + } + + return info, nil +} + +func (m *xtreemstoreS3BulkRetrieveManager) getSessionBatchKeys(ctx context.Context, batchInfo xtreemstoreS3BulkRetrieveBatchInfo) ([]string, error) { + getObjectInput := &s3.GetObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(fmt.Sprintf(XTS_SYSTEM_RETRIEVE_BATCH_FMT, batchInfo.Number)), + } + + resp, err := m.s3ApiClient.GetObject(ctx, getObjectInput) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var keys []string + if err := json.NewDecoder(resp.Body).Decode(&keys); err != nil { + return nil, fmt.Errorf("decode retrieve batch keys for batch %d: %w", batchInfo.Number, err) + } + + return keys, nil +} + +func (m *xtreemstoreS3BulkRetrieveManager) deleteSessionBatch(ctx context.Context, batchInfo xtreemstoreS3BulkRetrieveBatchInfo) error { + _, err := m.s3ApiClient.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(fmt.Sprintf(XTS_SYSTEM_RETRIEVE_BATCH_FMT, batchInfo.Number)), + }) + if err != nil { + return fmt.Errorf("delete retrieve batch %d: %w", batchInfo.Number, err) + } + + return nil +} + +func (m *xtreemstoreS3BulkRetrieveManager) startSession(ctx context.Context) (err error) { + if m.includedJobs == 0 { + return fmt.Errorf("retrieve-session requires at least one key") + } + + previousState := *m.state + cleanupCreatedSession := func(reason error) error { + *m.state = previousState + if cleanupErr := m.destroyRetrieveSession(ctx); cleanupErr != nil { + return fmt.Errorf("retrieve-session was created but local ownership state could not be persisted, cleanup also failed, and manual intervention is required: %w", errors.Join(reason, cleanupErr)) + } + return reason + } + + activeJobStart := m.state.SessionJobEnd + activeJobEnd := m.includedJobs + keys, err := m.getRecords(activeJobStart, activeJobEnd) + if err != nil { + return err + } + + retrieveSessionJson, err := json.Marshal(&xtreemstoreS3BulkRetrieveRequest{Ids: keys}) + if err != nil { + return fmt.Errorf("failed to marshal retrieve-session request: %w", err) + } + + _, err = m.s3ApiClient.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(XTS_SYSTEM_RETRIEVE_SESSION), + Body: bytes.NewReader(retrieveSessionJson), + ContentType: aws.String("application/json"), + }) + if err != nil { + var responseErr *smithyhttp.ResponseError + if errors.As(err, &responseErr) && responseErr.HTTPStatusCode() == http.StatusConflict { + *m.state = previousState + return ErrActiveRetrieveSessionAlreadyExists + } + + return fmt.Errorf("failed to start retrieve-session: %w", err) + } + + sessionInfo, err := m.getSessionInfo(ctx) + if err != nil { + return cleanupCreatedSession(fmt.Errorf("failed to recover retrieve-session ownership state after creation: %w", err)) + } + + m.state.SessionJobStart = activeJobStart + m.state.SessionJobEnd = activeJobEnd + m.state.SessionRetrieveId = sessionInfo.RetrieveId + + if err = m.saveManagerState(); err != nil { + return cleanupCreatedSession(fmt.Errorf("failed to store retrieve-session ownership state after creation: %w", err)) + } + + return nil +} + +// destroyRetrieveSession deletes the active retrieve session. This must only be called after an +// active session has been confirmed to belong to the manager. +func (m *xtreemstoreS3BulkRetrieveManager) destroyRetrieveSession(ctx context.Context) error { + _, err := m.s3ApiClient.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(XTS_SYSTEM_RETRIEVE_SESSION), + }) + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) && (apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "NoSuchKey") { + return nil + } + return fmt.Errorf("destroy retrieve session: %w", err) + } + + return nil +} + +func (m *xtreemstoreS3BulkRetrieveManager) getActiveRecordsMap() (map[string]int64, error) { + return m.getRecordsMap(m.state.SessionJobStart, m.state.SessionJobEnd) +} + +// func (m *xtreemstoreS3BulkRetrieveManager) getRecordsMapFromActiveStart() (map[string]int64, error) { +// return m.getRecordsMap(m.state.ActiveJobStart, -1) +// } + +func (m *xtreemstoreS3BulkRetrieveManager) getActiveSessionStatuses() (*xtreemstoreS3BulkStatuses, error) { + return m.getStatuses(m.state.SessionJobStart, m.state.SessionJobEnd) +} + +// getRecordsMap returns a mapping of record paths to job indexes for the specified range. Set end +// to -1 to get all mappings beginning from the start index. +func (m *xtreemstoreS3BulkRetrieveManager) getRecordsMap(start int64, end int64) (map[string]int64, error) { + if end == -1 { + end = m.includedJobs + } + if start < 0 || end < start { + return nil, fmt.Errorf("invalid active record range: start=%d end=%d", start, end) + } + + keyMap := map[string]int64{} + if start == end { + return keyMap, nil + } + + f, err := os.OpenFile(m.getRecordPath(), os.O_RDONLY, os.FileMode(0600)) + if err != nil { + return nil, err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for index := int64(0); scanner.Scan(); index++ { + if index < start { + continue + } + if index >= end { + break + } + keyMap[scanner.Text()] = index + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("failed to read bulk entry keys file: %w", err) + } + + return keyMap, nil +} + +// getRecords returns a range of record paths. Set end to -1 to get all records beginning with start. +func (m *xtreemstoreS3BulkRetrieveManager) getRecords(start int64, end int64) ([]string, error) { + if end == -1 { + end = m.includedJobs + } + if start < 0 || end < start { + return nil, fmt.Errorf("invalid active record range: start=%d end=%d", start, end) + } + if start == end { + return []string{}, nil + } + + f, err := os.OpenFile(m.getRecordPath(), os.O_RDONLY, os.FileMode(0600)) + if err != nil { + return nil, err + } + defer f.Close() + + keys := make([]string, 0, max(0, int(end-start))) + scanner := bufio.NewScanner(f) + for index := int64(0); scanner.Scan(); index++ { + if index < start { + continue + } + if index >= end { + break + } + keys = append(keys, scanner.Text()) + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("failed to read queued records: %w", err) + } + + return keys, nil +} + +// getStatuses returns statuses. Set end to -1 to get all statues beginning with start. +func (m *xtreemstoreS3BulkRetrieveManager) getStatuses(start int64, end int64) (*xtreemstoreS3BulkStatuses, error) { + if end == -1 { + end = m.includedJobs + } + if start < 0 || end < start { + return nil, fmt.Errorf("invalid active record range: start=%d end=%d", start, end) + } + + statuses := &xtreemstoreS3BulkStatuses{offset: start} + if start == end { + return statuses, nil + } + + f, err := os.OpenFile(m.getStatusPath(), os.O_RDONLY, os.FileMode(0600)) + if err != nil { + return nil, err + } + defer f.Close() + + statuses.jobCount = end - start + statuses.jobStatuses = make([]byte, statuses.jobCount) + if n, err := f.ReadAt(statuses.jobStatuses, int64(statuses.offset)); err != nil { + return nil, fmt.Errorf("failed to read bulk entry state file: %w", err) + } else if n != len(statuses.jobStatuses) { + return nil, fmt.Errorf("invalid bulk entry state") + } + + return statuses, nil +} diff --git a/common/rst/xtreemstorebulkretrieve_test.go b/common/rst/xtreemstorebulkretrieve_test.go new file mode 100644 index 00000000..50535349 --- /dev/null +++ b/common/rst/xtreemstorebulkretrieve_test.go @@ -0,0 +1,343 @@ +package rst + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "sync" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + smithyhttp "github.com/aws/smithy-go/transport/http" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/thinkparq/protobuf/go/beeremote" + "github.com/thinkparq/protobuf/go/flex" +) + +// fakeS3ApiClient is a minimal, hand-rolled s3ApiClient scoped to exactly what +// TestBulkRetrieveExecuteStopsReschedulingOnceAllComplete needs to drive the real (non-executeTest) +// execute() path: a single retrieve-session containing one batch with every id from the PUT body, +// and objects that always report ready-for-download -- this test exercises session/batch/reschedule +// bookkeeping, not tape-restore timing. A single instance must be shared across every +// xtreemstoreS3BulkRetrieveManager the test constructs (mirroring a real S3 bucket's state +// persisting across manager reloads), rather than one fake per manager. +type fakeS3ApiClient struct { + mu sync.Mutex + active bool + id string + ids []string +} + +var _ s3ApiClient = &fakeS3ApiClient{} + +func fakeNotFoundErr() error { + return &smithy.GenericAPIError{Code: "NoSuchKey", Message: "key not found"} +} + +func fakeConflictErr() error { + return &smithyhttp.ResponseError{ + Response: &smithyhttp.Response{Response: &http.Response{StatusCode: http.StatusConflict}}, + Err: &smithy.GenericAPIError{Code: "ActiveRetrieveSessionAlreadyExists"}, + } +} + +func fakeJSONBody(v any) io.ReadCloser { + b, err := json.Marshal(v) + if err != nil { + panic(err) + } + return io.NopCloser(bytes.NewReader(b)) +} + +func (f *fakeS3ApiClient) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + f.mu.Lock() + defer f.mu.Unlock() + + switch aws.ToString(params.Key) { + case XTS_SYSTEM_RETRIEVE_SESSION: + if !f.active { + return nil, fakeNotFoundErr() + } + return &s3.GetObjectOutput{Body: fakeJSONBody(xtreemstoreS3BulkRetrieveSessionInfo{ + Active: true, + RetrieveId: f.id, + Started: time.Now(), + })}, nil + case XTS_SYSTEM_RETRIEVE_BATCH_LIST: + if !f.active { + return nil, fakeNotFoundErr() + } + return &s3.GetObjectOutput{Body: fakeJSONBody([]xtreemstoreS3BulkRetrieveBatchInfo{ + {Number: 0, Objects: int64(len(f.ids)), Size: 0}, + })}, nil + case fmt.Sprintf(XTS_SYSTEM_RETRIEVE_BATCH_FMT, 0): + if !f.active { + return nil, fakeNotFoundErr() + } + return &s3.GetObjectOutput{Body: fakeJSONBody(f.ids)}, nil + default: + return nil, fakeNotFoundErr() + } +} + +func (f *fakeS3ApiClient) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + if aws.ToString(params.Key) != XTS_SYSTEM_RETRIEVE_SESSION { + return nil, fmt.Errorf("fakeS3ApiClient: unexpected PutObject key %q", aws.ToString(params.Key)) + } + + f.mu.Lock() + defer f.mu.Unlock() + if f.active { + return nil, fakeConflictErr() + } + + body, err := io.ReadAll(params.Body) + if err != nil { + return nil, err + } + var req xtreemstoreS3BulkRetrieveRequest + if err := json.Unmarshal(body, &req); err != nil { + return nil, err + } + + f.active = true + f.id = "test-retrieve-id" + f.ids = req.Ids + return &s3.PutObjectOutput{}, nil +} + +func (f *fakeS3ApiClient) DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) { + f.mu.Lock() + defer f.mu.Unlock() + + if aws.ToString(params.Key) == XTS_SYSTEM_RETRIEVE_SESSION { + f.active = false + f.id = "" + f.ids = nil + } + return &s3.DeleteObjectOutput{}, nil +} + +func (f *fakeS3ApiClient) HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + // Always ready: this test exercises reschedule/completion bookkeeping, not restore timing. + return &s3.HeadObjectOutput{StorageClass: types.StorageClassStandard}, nil +} + +func (f *fakeS3ApiClient) ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) { + return nil, fmt.Errorf("fakeS3ApiClient: ListObjectsV2 not used by this test") +} + +func (f *fakeS3ApiClient) ListObjectsV2Pages(ctx context.Context, params *s3.ListObjectsV2Input, pageFn func(*s3.ListObjectsV2Output) (bool, error)) error { + return fmt.Errorf("fakeS3ApiClient: ListObjectsV2Pages not used by this test") +} + +func (f *fakeS3ApiClient) RestoreObject(ctx context.Context, params *s3.RestoreObjectInput, optFns ...func(*s3.Options)) (*s3.RestoreObjectOutput, error) { + return nil, fmt.Errorf("fakeS3ApiClient: RestoreObject not used by this test") +} + +func (f *fakeS3ApiClient) CreateMultipartUpload(ctx context.Context, params *s3.CreateMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error) { + return nil, fmt.Errorf("fakeS3ApiClient: CreateMultipartUpload not used by this test") +} + +func (f *fakeS3ApiClient) AbortMultipartUpload(ctx context.Context, params *s3.AbortMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error) { + return nil, fmt.Errorf("fakeS3ApiClient: AbortMultipartUpload not used by this test") +} + +func (f *fakeS3ApiClient) CompleteMultipartUpload(ctx context.Context, params *s3.CompleteMultipartUploadInput, optFns ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error) { + return nil, fmt.Errorf("fakeS3ApiClient: CompleteMultipartUpload not used by this test") +} + +func (f *fakeS3ApiClient) UploadPart(ctx context.Context, params *s3.UploadPartInput, optFns ...func(*s3.Options)) (*s3.UploadPartOutput, error) { + return nil, fmt.Errorf("fakeS3ApiClient: UploadPart not used by this test") +} + +// TestBulkRetrieveExecuteStopsReschedulingOnceAllComplete reproduces the reported issue in +// isolation: after every dispatched bulk-retrieve record is marked complete (exactly as +// CompleteWorkRequests -> xtreemstoreS3BulkMarkRequestComplete does for each individual sub-job), +// a fresh Execute() pass (as happens on every builder-job reschedule) should stop asking to +// reschedule. +func TestBulkRetrieveExecuteStopsReschedulingOnceAllComplete(t *testing.T) { + tmpDir := t.TempDir() + const stateMountPath = "state" + const operation = "bulk-retrieve" + + // Shared across every manager instance the test constructs, mirroring how a real S3 bucket's + // retrieve-session state persists across manager reloads. + fake := &fakeS3ApiClient{} + + newManager := func(t *testing.T) *xtreemstoreS3BulkRetrieveManager { + m := &xtreemstoreS3BulkRetrieveManager{ + s3ApiClient: fake, + rstId: 1, + mountPath: tmpDir, + stateMountPath: stateMountPath, + operation: operation, + state: &xtreemstoreS3BulkRetrieveManagerState{}, + } + require.NoError(t, m.openState()) + return m + } + + m := newManager(t) + for _, p := range []string{"/a", "/b", "/c"} { + req := &beeremote.JobRequest{ + Type: &beeremote.JobRequest_Sync{Sync: &flex.SyncJob{RemotePath: p}}, + BulkInfo: &flex.BulkJobRequestInfo{}, + } + require.NoError(t, m.AddRequest(context.Background(), req)) + } + require.NoError(t, m.closeState()) + + // First Execute pass: everything is Initialized, so all 3 get dispatched and marked Sent. + m = newManager(t) + walkCh, getResults, err := m.Execute(context.Background()) + require.NoError(t, err) + + var dispatched []*flex.BulkJobRequestInfo + for r := range walkCh { + require.NoError(t, r.Err) + dispatched = append(dispatched, r.BulkInfo) + } + result := getResults() + require.NoError(t, result.Err) + assert.True(t, result.Reschedule, "should reschedule while records are still Sent") + require.Len(t, dispatched, 3) + require.NoError(t, m.closeState()) + + // Simulate every dispatched sub-job completing successfully, exactly like + // xtreemstoreS3BulkMarkRequestComplete does when CompleteWorkRequests is called. + for _, bulkInfo := range dispatched { + completeManager := &xtreemstoreS3BulkRetrieveManager{ + mountPath: tmpDir, + stateMountPath: bulkInfo.StateMountPath, + operation: bulkInfo.Operation, + } + err = completeManager.MarkComplete(bulkInfo.JobIndex) + require.NoError(t, err) + } + + // Second Execute pass (fresh manager instance, exactly as happens on a real builder-job + // reschedule): every record is now Complete, so this should NOT ask to reschedule again. + m = newManager(t) + walkCh2, getResults2, err := m.Execute(context.Background()) + require.NoError(t, err) + for r := range walkCh2 { + t.Fatalf("expected no further dispatch once everything is complete, got: %+v", r) + } + result2 := getResults2() + require.NoError(t, result2.Err) + assert.False(t, result2.Reschedule, "bulk operation should stop rescheduling once every record is marked complete") +} + +// TestProcessSessionBatchKeyReceivedStillWaits ensures a record that has reached +// xtreemstoreS3BulkRequestReceived (GenerateWorkRequests successfully created a Job for it) is +// still treated as in-progress, not as an unexpected status. Falling through to the default case +// here would force-MarkCompleteAck a record whose real download may still be running in +// ExecuteWorkRequestPart, and report a bogus "unexpected record status" error for it. +func TestProcessSessionBatchKeyReceivedStillWaits(t *testing.T) { + m := &xtreemstoreS3BulkRetrieveManager{rstId: 1} + walkCh := make(chan *BulkStreamPathResult, 1) + + done, err := m.processSessionBatchKey(context.Background(), walkCh, "/a", 0, xtreemstoreS3BulkRequestReceived) + require.NoError(t, err) + assert.False(t, done, "a Received record is still in flight and must not be reported done") + + select { + case r := <-walkCh: + t.Fatalf("expected no result to be sent for a Received record, got: %+v", r) + default: + } +} + +// stubHeadObjectClient fakes only HeadObject, the sole s3ApiClient method isObjectReadyForDownload +// calls. +type stubHeadObjectClient struct { + s3ApiClient + output *s3.HeadObjectOutput + err error +} + +func (s *stubHeadObjectClient) HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + return s.output, s.err +} + +// TestIsObjectReadyForDownload covers every storage-class/error branch of isObjectReadyForDownload, +// including the "unexpected storage class" default case that TestBulkRetrieveExecuteStops... does +// not exercise (that test only ever sees StorageClassStandard). +func TestIsObjectReadyForDownload(t *testing.T) { + tests := []struct { + name string + output *s3.HeadObjectOutput + err error + wantReady bool + wantErrIs error + wantErrContains string + }{ + { + name: "standard storage class is ready", + output: &s3.HeadObjectOutput{StorageClass: types.StorageClassStandard}, + wantReady: true, + }, + { + name: "glacier object with completed restore is ready", + output: &s3.HeadObjectOutput{StorageClass: types.StorageClassGlacier, Restore: aws.String(`ongoing-request="false", expiry-date="Fri, 01 Jan 2027 00:00:00 GMT"`)}, + wantReady: true, + }, + { + name: "glacier object with restore in progress is not ready", + output: &s3.HeadObjectOutput{StorageClass: types.StorageClassGlacier, Restore: aws.String(`ongoing-request="true"`)}, + wantReady: false, + }, + { + name: "glacier object with no restore requested yet is not ready", + output: &s3.HeadObjectOutput{StorageClass: types.StorageClassGlacier}, + wantReady: false, + }, + { + name: "missing object maps to os.ErrNotExist", + err: fakeNotFoundErr(), + wantReady: false, + wantErrIs: os.ErrNotExist, + }, + { + name: "unrecognized storage class is an error", + output: &s3.HeadObjectOutput{StorageClass: types.StorageClass("UNKNOWN")}, + wantReady: false, + wantErrContains: "unexpected storage class", + }, + { + name: "generic API error is wrapped, not treated as not-exist", + err: fmt.Errorf("connection reset"), + wantReady: false, + wantErrContains: "head object for key", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := &xtreemstoreS3BulkRetrieveManager{ + s3ApiClient: &stubHeadObjectClient{output: tt.output, err: tt.err}, + bucket: "test-bucket", + } + ready, err := m.isObjectReadyForDownload(context.Background(), "some/key") + assert.Equal(t, tt.wantReady, ready) + switch { + case tt.wantErrIs != nil: + assert.ErrorIs(t, err, tt.wantErrIs) + case tt.wantErrContains != "": + assert.ErrorContains(t, err, tt.wantErrContains) + default: + assert.NoError(t, err) + } + }) + } +} diff --git a/common/scheduler/scheduler.go b/common/scheduler/scheduler.go index 1966f2ca..cd95b83a 100644 --- a/common/scheduler/scheduler.go +++ b/common/scheduler/scheduler.go @@ -318,6 +318,8 @@ func (s *Scheduler) SetNextRescheduledTime(ExecuteAfter time.Time, priority int) } } +type AddRescheduleWorkTokenFn func(submissionId string, ExecuteAfter time.Time) + // AddRescheduleWorkToken adds a rescheduleWorkToken and sets the next // check time if needed. func (s *Scheduler) AddRescheduleWorkToken(submissionId string, ExecuteAfter time.Time) { diff --git a/ctl/internal/cmd/rst/list.go b/ctl/internal/cmd/rst/list.go index 7bb6d674..e24b70eb 100644 --- a/ctl/internal/cmd/rst/list.go +++ b/ctl/internal/cmd/rst/list.go @@ -52,18 +52,16 @@ func runListCmd(cmd *cobra.Command, cfg rst.GetRSTCfg) error { switch rst.WhichType() { case flex.RemoteStorageTarget_S3_case: - stringBuilder := strings.Builder{} rstType = "s3" - rst.GetS3().ProtoReflect().Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { - if string(fd.Name()) == "secret_key" && !cfg.ShowSecrets { - stringBuilder.WriteString(fmt.Sprintf("%s: *****, ", fd.Name())) - } else { - stringBuilder.WriteString(fmt.Sprintf("%s: %s, ", fd.Name(), v)) - } - return true - }) - // Get rid of the last comma+space in the printed configuration. - rstConfiguration = stringBuilder.String()[:stringBuilder.Len()-2] + rstConfiguration = formatS3RSTConfiguration(rst.GetS3(), cfg.ShowSecrets) + case flex.RemoteStorageTarget_Xtreemstore_case: + rstType = "xtreemstore" + xtreemstoreConfig := rst.GetXtreemstore() + if xtreemstoreConfig == nil { + rstConfiguration = "xtreemstore configuration is not set" + break + } + rstConfiguration = formatS3RSTConfiguration(xtreemstoreConfig.GetS3(), cfg.ShowSecrets) default: if !cfg.ShowSecrets { rstType = "unknown" @@ -85,3 +83,25 @@ func runListCmd(cmd *cobra.Command, cfg rst.GetRSTCfg) error { return nil } + +func formatS3RSTConfiguration(s3Config *flex.RemoteStorageTarget_S3, showSecrets bool) string { + if s3Config == nil { + return "s3 configuration is not set" + } + + stringBuilder := strings.Builder{} + s3Config.ProtoReflect().Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { + if string(fd.Name()) == "secret_key" && !showSecrets { + fmt.Fprintf(&stringBuilder, "%s: *****, ", fd.Name()) + } else { + fmt.Fprintf(&stringBuilder, "%s: %s, ", fd.Name(), v) + } + return true + }) + + if stringBuilder.Len() == 0 { + return "" + } + // Get rid of the last comma+space in the printed configuration. + return stringBuilder.String()[:stringBuilder.Len()-2] +} diff --git a/ctl/pkg/ctl/entry/entry.go b/ctl/pkg/ctl/entry/entry.go index b2b0cafe..242ebf35 100644 --- a/ctl/pkg/ctl/entry/entry.go +++ b/ctl/pkg/ctl/entry/entry.go @@ -704,7 +704,20 @@ func getPrimaryMetaNode(ctx context.Context, mappings *util.Mappings, entryInfo } func GetFileDataState(ctx context.Context, path string) (beegfs.DataState, error) { - state, err := getFileState(ctx, path) + entryInfoMsg, _, ownerNode, err := GetEntryAndOwnerFromPath(ctx, nil, path) + if err != nil { + return beegfs.DataStateMask.GetDataState(), fmt.Errorf("failed to get file data state: %w", err) + } + + state, err := getFileState(ctx, ownerNode, entryInfoMsg) + if err != nil { + return beegfs.DataStateMask.GetDataState(), fmt.Errorf("failed to get file data state: %w", err) + } + return state.GetDataState(), nil +} + +func GetFileDataStateWithEntryInfo(ctx context.Context, path string, entryInfoMsg msg.EntryInfo, ownerNode beegfs.Node) (beegfs.DataState, error) { + state, err := getFileState(ctx, ownerNode, entryInfoMsg) if err != nil { return beegfs.DataStateMask.GetDataState(), fmt.Errorf("failed to get file data state: %w", err) } @@ -712,17 +725,21 @@ func GetFileDataState(ctx context.Context, path string) (beegfs.DataState, error } func SetFileDataState(ctx context.Context, path string, state beegfs.DataState) error { - entry, _, ownerNode, err := GetEntryAndOwnerFromPath(ctx, nil, path) + entryInfoMsg, _, ownerNode, err := GetEntryAndOwnerFromPath(ctx, nil, path) if err != nil { return fmt.Errorf("unable to retrieve entry info: %w", err) } + return SetFileDataStateWithEntryInfo(ctx, path, state, entryInfoMsg, ownerNode) +} + +func SetFileDataStateWithEntryInfo(ctx context.Context, path string, state beegfs.DataState, entryInfoMsg msg.EntryInfo, ownerNode beegfs.Node) error { store, err := config.NodeStore(ctx) if err != nil { return err } info := &msg.GetEntryInfoResponse{} - err = store.RequestTCP(ctx, ownerNode.Uid, &msg.GetEntryInfoRequest{EntryInfo: entry}, info) + err = store.RequestTCP(ctx, ownerNode.Uid, &msg.GetEntryInfoRequest{EntryInfo: entryInfoMsg}, info) if err != nil { return fmt.Errorf("unable to get data state for path, %s: %w", path, err) } @@ -733,7 +750,7 @@ func SetFileDataState(ctx context.Context, path string, state beegfs.DataState) } response := &msg.SetFileStateResponse{} - err = store.RequestTCP(ctx, ownerNode.Uid, &msg.SetFileStateRequest{EntryInfo: entry, FileState: fs}, response) + err = store.RequestTCP(ctx, ownerNode.Uid, &msg.SetFileStateRequest{EntryInfo: entryInfoMsg, FileState: fs}, response) if err != nil { return err } @@ -742,7 +759,7 @@ func SetFileDataState(ctx context.Context, path string, state beegfs.DataState) // concurrently. In these cases, OpsErr_INODELOCKED is returned and will be retried once // more. if response.Result == beegfs.OpsErr_INODELOCKED { - err = store.RequestTCP(ctx, ownerNode.Uid, &msg.GetEntryInfoRequest{EntryInfo: entry}, info) + err = store.RequestTCP(ctx, ownerNode.Uid, &msg.GetEntryInfoRequest{EntryInfo: entryInfoMsg}, info) if err != nil { return fmt.Errorf("unable to get data state for path, %s: %w", path, err) } @@ -752,7 +769,7 @@ func SetFileDataState(ctx context.Context, path string, state beegfs.DataState) return nil } - err = store.RequestTCP(ctx, ownerNode.Uid, &msg.SetFileStateRequest{EntryInfo: entry, FileState: fs}, response) + err = store.RequestTCP(ctx, ownerNode.Uid, &msg.SetFileStateRequest{EntryInfo: entryInfoMsg, FileState: fs}, response) if err != nil { return err } @@ -768,64 +785,89 @@ func SetFileDataState(ctx context.Context, path string, state beegfs.DataState) } func GetFileAccessFlags(ctx context.Context, path string) (beegfs.AccessFlags, error) { - state, err := getFileState(ctx, path) + entryInfoMsg, _, ownerNode, err := GetEntryAndOwnerFromPath(ctx, nil, path) + if err != nil { + return beegfs.AccessFlagMask.GetAccessFlags(), fmt.Errorf("failed to get file access flags: %w", err) + } + + state, err := getFileState(ctx, ownerNode, entryInfoMsg) if err != nil { return beegfs.AccessFlagMask.GetAccessFlags(), fmt.Errorf("failed to get file access flags: %w", err) } return state.GetAccessFlags(), nil } +func GetFileAccessFlagsWithEntryInfo(ctx context.Context, path string, entryInfoMsg msg.EntryInfo, ownerNode beegfs.Node) (beegfs.AccessFlags, error) { + state, err := getFileState(ctx, ownerNode, entryInfoMsg) + if err != nil { + return beegfs.AccessFlagMask.GetAccessFlags(), fmt.Errorf("failed to get file access flags: %w", err) + } + return state.GetAccessFlags(), nil +} func SetAccessFlags(ctx context.Context, path string, flags beegfs.AccessFlags) error { - if err := setAccessFlags(ctx, path, flags, false); err != nil { + entryInfoMsg, _, ownerNode, err := GetEntryAndOwnerFromPath(ctx, nil, path) + if err != nil { + return fmt.Errorf("failed to set file access flags: %w", err) + } + + if err := setAccessFlags(ctx, path, flags, false, entryInfoMsg, ownerNode); err != nil { + return fmt.Errorf("failed to set file access flags: %w", err) + } + return nil +} + +func SetAccessFlagsWithEntryInfo(ctx context.Context, path string, flags beegfs.AccessFlags, entryInfoMsg msg.EntryInfo, ownerNode beegfs.Node) error { + if err := setAccessFlags(ctx, path, flags, false, entryInfoMsg, ownerNode); err != nil { return fmt.Errorf("failed to set file access flags: %w", err) } return nil } func ClearAccessFlags(ctx context.Context, path string, flags beegfs.AccessFlags) error { - if err := setAccessFlags(ctx, path, flags, true); err != nil { + entryInfoMsg, _, ownerNode, err := GetEntryAndOwnerFromPath(ctx, nil, path) + if err != nil { + return fmt.Errorf("failed to clear file access flags: %w", err) + } + + if err := setAccessFlags(ctx, path, flags, true, entryInfoMsg, ownerNode); err != nil { return fmt.Errorf("failed to clear file access flags: %w", err) } return nil } -func getFileState(ctx context.Context, path string) (beegfs.FileState, error) { - entry, _, ownerNode, err := GetEntryAndOwnerFromPath(ctx, nil, path) - if err != nil { - mask := beegfs.NewFileState(beegfs.AccessFlagMask.GetAccessFlags(), beegfs.AccessFlagMask.GetDataState()) - return mask, err +func ClearAccessFlagsWithEntryInfo(ctx context.Context, path string, flags beegfs.AccessFlags, entryInfoMsg msg.EntryInfo, ownerNode beegfs.Node) error { + if err := setAccessFlags(ctx, path, flags, true, entryInfoMsg, ownerNode); err != nil { + return fmt.Errorf("failed to clear file access flags: %w", err) } + return nil +} +func getFileState(ctx context.Context, ownerNode beegfs.Node, entryInfoMsg msg.EntryInfo) (beegfs.FileState, error) { store, err := config.NodeStore(ctx) if err != nil { - mask := beegfs.NewFileState(beegfs.AccessFlagMask.GetAccessFlags(), beegfs.AccessFlagMask.GetDataState()) + mask := beegfs.NewFileState(beegfs.AccessFlagMask.GetAccessFlags(), beegfs.DataStateMask.GetDataState()) return mask, err } - request := &msg.GetEntryInfoRequest{EntryInfo: entry} + request := &msg.GetEntryInfoRequest{EntryInfo: entryInfoMsg} resp := &msg.GetEntryInfoResponse{} err = store.RequestTCP(ctx, ownerNode.Uid, request, resp) if err != nil { - mask := beegfs.NewFileState(beegfs.AccessFlagMask.GetAccessFlags(), beegfs.AccessFlagMask.GetDataState()) + mask := beegfs.NewFileState(beegfs.AccessFlagMask.GetAccessFlags(), beegfs.DataStateMask.GetDataState()) return mask, err } return resp.FileState, nil } -func setAccessFlags(ctx context.Context, path string, flags beegfs.AccessFlags, clearFlags bool) error { - entry, _, ownerNode, err := GetEntryAndOwnerFromPath(ctx, nil, path) - if err != nil { - return err - } - +func setAccessFlags(ctx context.Context, path string, flags beegfs.AccessFlags, clearFlags bool, entryInfoMsg msg.EntryInfo, ownerNode beegfs.Node) error { store, err := config.NodeStore(ctx) if err != nil { return err } info := &msg.GetEntryInfoResponse{} - err = store.RequestTCP(ctx, ownerNode.Uid, &msg.GetEntryInfoRequest{EntryInfo: entry}, info) + err = store.RequestTCP(ctx, ownerNode.Uid, &msg.GetEntryInfoRequest{EntryInfo: entryInfoMsg}, info) if err != nil { return err } @@ -842,7 +884,7 @@ func setAccessFlags(ctx context.Context, path string, flags beegfs.AccessFlags, } response := &msg.SetFileStateResponse{} - err = store.RequestTCP(ctx, ownerNode.Uid, &msg.SetFileStateRequest{EntryInfo: entry, FileState: fs}, response) + err = store.RequestTCP(ctx, ownerNode.Uid, &msg.SetFileStateRequest{EntryInfo: entryInfoMsg, FileState: fs}, response) if err != nil { return err } @@ -852,7 +894,7 @@ func setAccessFlags(ctx context.Context, path string, flags beegfs.AccessFlags, // more. if response.Result == beegfs.OpsErr_INODELOCKED { info := &msg.GetEntryInfoResponse{} - err = store.RequestTCP(ctx, ownerNode.Uid, &msg.GetEntryInfoRequest{EntryInfo: entry}, info) + err = store.RequestTCP(ctx, ownerNode.Uid, &msg.GetEntryInfoRequest{EntryInfo: entryInfoMsg}, info) if err != nil { return err } @@ -868,7 +910,7 @@ func setAccessFlags(ctx context.Context, path string, flags beegfs.AccessFlags, return nil } - err = store.RequestTCP(ctx, ownerNode.Uid, &msg.SetFileStateRequest{EntryInfo: entry, FileState: fs}, response) + err = store.RequestTCP(ctx, ownerNode.Uid, &msg.SetFileStateRequest{EntryInfo: entryInfoMsg, FileState: fs}, response) if err != nil { return err } @@ -908,16 +950,25 @@ func SetFileRstPattern(ctx context.Context, path string, rstIds []uint32, cooldo } ownerNode = entryInfo.Entry.MetaOwnerNode } - store, err := config.NodeStore(ctx) - if err != nil { - return err - } + + updateRequired := false newRSTCfg := currentRSTCfg - if rstIds != nil { + if !rstIdsMatch(currentRSTCfg.RSTIDs, rstIds) { newRSTCfg.RSTIDs = rstIds + updateRequired = true } - if cooldownSecs != nil { + if cooldownSecs != nil && currentRSTCfg.CoolDownPeriod != *cooldownSecs { newRSTCfg.CoolDownPeriod = *cooldownSecs + updateRequired = true + } + + if !updateRequired { + return nil + } + + store, err := config.NodeStore(ctx) + if err != nil { + return err } req := &msg.SetFilePatternRequest{EntryInfo: entryInfoMsg, RST: newRSTCfg} resp := &msg.SetFilePatternResponse{} @@ -930,6 +981,26 @@ func SetFileRstPattern(ctx context.Context, path string, rstIds []uint32, cooldo return nil } +// rstIdsMatch returns whether the old match the new rstIds regardless of the order. +func rstIdsMatch(old []uint32, new []uint32) bool { + if len(old) != len(new) { + return false + } + + counts := make(map[uint32]int, len(old)) + for _, rstId := range old { + counts[rstId]++ + } + + for _, rstId := range new { + counts[rstId]-- + if counts[rstId] < 0 { + return false + } + } + return true +} + // SetDirRstPattern fetches the directory entry once and applies the specified RST fields using a // single SetDirPatternRequest, preserving the existing stripe pattern and all other RST fields. // Pass nil for rstIds to leave the current IDs unchanged; pass nil for cooldownSecs to leave the diff --git a/ctl/pkg/ctl/rst/status.go b/ctl/pkg/ctl/rst/status.go index 8b5e0049..b675f14f 100644 --- a/ctl/pkg/ctl/rst/status.go +++ b/ctl/pkg/ctl/rst/status.go @@ -413,10 +413,13 @@ func getPathStatusFromTarget( ) (*GetStatusResult, error) { // Default to any specified targets specified in cfg otherwise attempt to use rstIds returned // from GetLockedInfo. - lockedInfo, _, rstIds, _, _, _, err := rst.GetLockedInfo(ctx, mountPoint, &flex.JobRequestCfg{}, fsPath, true) + lockedInfoResult, err := rst.GetPathState(ctx, mountPoint, fsPath, rst.PathStateNoLock) + lockedInfo := lockedInfoResult.LockedInfo + rstIds := lockedInfoResult.RstCfg.RSTIDs if len(cfg.RemoteTargets) != 0 { rstIds = cfg.RemoteTargets } + // GetLockedInfo returns any known information along with the defaults. So, if lockedInfo.Mode // is non-zero then it is valid to check the file type and is safe to do so before checking the // GetLockedInfo error. diff --git a/go.mod b/go.mod index 1c375eed..d0b3b635 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.11.1 - github.com/thinkparq/protobuf v0.8.4-0.20260616194611-ae240cc861a3 + github.com/thinkparq/protobuf v0.8.5-0.20260806181427-5e8a7380fff5 go.opentelemetry.io/contrib/bridges/otelzap v0.18.0 go.opentelemetry.io/otel v1.44.0 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.20.0 diff --git a/go.sum b/go.sum index a3bb4e19..b01334ca 100644 --- a/go.sum +++ b/go.sum @@ -187,8 +187,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/thinkparq/protobuf v0.8.4-0.20260616194611-ae240cc861a3 h1:OgKcIFT99ji0EV3B9rO/L6LYvslDY2R7PeURKj5I5HE= -github.com/thinkparq/protobuf v0.8.4-0.20260616194611-ae240cc861a3/go.mod h1:iqZlCWzy4WL/0k5I/AU6muxmUif4XcSwjC3mvwkkhHs= +github.com/thinkparq/protobuf v0.8.5-0.20260806181427-5e8a7380fff5 h1:SfepKbvo6wb8j5bMBFlMwWbjzdFYYr1b0z8j9BveE6A= +github.com/thinkparq/protobuf v0.8.5-0.20260806181427-5e8a7380fff5/go.mod h1:iqZlCWzy4WL/0k5I/AU6muxmUif4XcSwjC3mvwkkhHs= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= diff --git a/rst/remote/build/beegfs-remote.toml b/rst/remote/build/beegfs-remote.toml index ce75abea..c1845810 100644 --- a/rst/remote/build/beegfs-remote.toml +++ b/rst/remote/build/beegfs-remote.toml @@ -122,6 +122,18 @@ path-db = "/var/lib/beegfs/remote/path.badger" # access-key = "" # secret-key = "" +# [[remote-storage-target]] +# id = "4" +# name = "xtreemstore" +# policies = { FastStartMaxSize = 104857600 } +# +# [remote-storage-target.xtreemstore] +# endpoint-url = "https://:" +# region = "" +# bucket = "" +# access-key = "" +# secret-key = "" + # # --- Section 1.4: [Telemetry Settings] --- # @@ -273,9 +285,8 @@ rate-limit-event-types = ["*"] # tls-disable: Set to disable TLS encryption and send all gRPC messages in clear text over the # network. Discouraged for production as it exposes data to potential interception and tampering. - # -# --- Section 3.4: [Job Settings] --- +# --- Section 3.5: [Job Settings] --- # # Job configuration must be specified under [job]. These settings affect how jobs are tracked and diff --git a/rst/remote/internal/job/manager.go b/rst/remote/internal/job/manager.go index 9b82eb8c..1a04c9de 100644 --- a/rst/remote/internal/job/manager.go +++ b/rst/remote/internal/job/manager.go @@ -788,16 +788,20 @@ func (m *Manager) SubmitJobRequest(jr *beeremote.JobRequest) (*beeremote.JobResu } - rstClient, ok := m.workerManager.RemoteStorageTargets[job.Request.GetRemoteStorageTarget()] - if !ok { - return nil, fmt.Errorf("rejecting job because the requested RST does not exist: %d", job.Request.GetRemoteStorageTarget()) - } - var jobSubmission workermgr.JobSubmission - if jr.GenerationStatus != nil { - status := jr.GenerationStatus - if status != nil { - switch status.State { + if jr.HasGenerationStatus() { + status := jr.GetGenerationStatus() + if _, ok := m.workerManager.RemoteStorageTargets[job.Request.GetRemoteStorageTarget()]; !ok { + // A FAILED_PRECONDITION with an unknown rstId means the builder encountered a file + // whose RST config references an rstId that no longer exists (or never did). Treat it + // as ErrJobFailedPrecondition so the job gets the error rather than rejected. + if status.GetState() == beeremote.JobRequest_GenerationStatus_FAILED_PRECONDITION { + err = fmt.Errorf("%w: %s", rst.ErrJobFailedPrecondition, status.Message) + } else { + return nil, fmt.Errorf("rejecting job because the requested RST does not exist: %d", job.Request.GetRemoteStorageTarget()) + } + } else { + switch status.GetState() { case beeremote.JobRequest_GenerationStatus_ALREADY_COMPLETE: // ParseDataTime will return the parsed mtime or a zero-mtime. Either way we should // mark the job as complete so ignore the err. @@ -814,6 +818,10 @@ func (m *Manager) SubmitJobRequest(jr *beeremote.JobRequest) (*beeremote.JobResu } } } else { + rstClient, ok := m.workerManager.RemoteStorageTargets[job.Request.GetRemoteStorageTarget()] + if !ok { + return nil, fmt.Errorf("rejecting job because the requested RST does not exist: %d", job.Request.GetRemoteStorageTarget()) + } jobSubmission, err = job.GenerateSubmission(m.ctx, lastJob, rstClient) } @@ -1067,12 +1075,7 @@ func (m *Manager) UpdateJobs(jobUpdate *beeremote.UpdateJobsRequest) (*beeremote err := m.releaseUnusedFileLockFunc(jobUpdate.GetPath(), pathEntry.Value) if err != nil { response.SetOk(false) - message := "unable to clear lock: " + err.Error() - if response.Message != "" { - response.SetMessage(fmt.Sprintf("%s; %s", response.Message, message)) - } else { - response.SetMessage(message) - } + response.SetMessage(appendMessage(response.Message, "unable to clear lock: "+err.Error())) } }() @@ -1086,7 +1089,7 @@ func (m *Manager) UpdateJobs(jobUpdate *beeremote.UpdateJobsRequest) (*beeremote // from some other job, don't overwrite it: response.SetOk(success && response.GetOk()) if newMessage != "" { - response.SetMessage(response.GetMessage() + "; " + newMessage) + response.SetMessage(appendMessage(response.GetMessage(), newMessage)) } // Only if the user requested a deletion and the job is safe to delete mark it for deletion: if jobUpdate.GetNewState() == beeremote.UpdateJobsRequest_DELETED && safeToDelete { @@ -1275,11 +1278,11 @@ func (m *Manager) updateJobState(job *Job, newState beeremote.UpdateJobsRequest_ if !ok { if forceUpdate { status.SetState(beeremote.Job_CANCELLED) - status.SetMessage(status.GetMessage() + (status.GetMessage() + "; unable to request the RST abort this job because the specified RST no longer exists (ignoring because this is a forced update)")) + status.SetMessage(appendMessage(status.GetMessage(), "unable to request the RST abort this job because the specified RST no longer exists (ignoring because this is a forced update)")) return true, true, "" } status.SetState(beeremote.Job_FAILED) - status.SetMessage(status.GetMessage() + (status.GetMessage() + "; unable to request the RST abort this job because the specified RST no longer exists (add it back or force the update to cancel the job anyway)")) + status.SetMessage(appendMessage(status.GetMessage(), "unable to request the RST abort this job because the specified RST no longer exists (add it back or force the update to cancel the job anyway)")) return false, false, "" } @@ -1287,16 +1290,16 @@ func (m *Manager) updateJobState(job *Job, newState beeremote.UpdateJobsRequest_ if err != nil { if forceUpdate { status.SetState(beeremote.Job_CANCELLED) - status.SetMessage(status.GetMessage() + (status.GetMessage() + "; error requesting the RST abort this job (ignoring because this is a forced update): " + err.Error())) + status.SetMessage(appendMessage(status.GetMessage(), "error requesting the RST abort this job (ignoring because this is a forced update): "+err.Error())) return true, true, "" } status.SetState(beeremote.Job_FAILED) - status.SetMessage(status.GetMessage() + (status.GetMessage() + "; error requesting the RST abort this job (try again or force the update to cancel the job anyway): " + err.Error())) + status.SetMessage(appendMessage(status.GetMessage(), "error requesting the RST abort this job (try again or force the update to cancel the job anyway): "+err.Error())) return false, false, "" } status.SetState(beeremote.Job_CANCELLED) - status.SetMessage(status.GetMessage() + "; successfully requested the RST abort this job") + status.SetMessage(appendMessage(status.GetMessage(), "successfully requested the RST abort this job")) m.log.Debug("successfully updated job", zap.Any("job", job)) return true, true, "" } @@ -1361,7 +1364,29 @@ func (m *Manager) UpdateWork(workResult *flex.Work) error { allSameState := true for _, workResult := range job.WorkResults { if !workResult.InTerminalState() && !workResult.RequiresUserIntervention() { - // Don't do anything else if all work requests haven't reached a terminal state or aren't failed. + + if entryToUpdate.Status().GetState() == flex.Work_RUNNING { + status := job.GetStatus() + status.SetState(beeremote.Job_RUNNING) + status.SetUpdated(timestamppb.Now()) + + switch job.Request.WhichType() { + case beeremote.JobRequest_Builder_case: + // Builder jobs only ever have one work request, so workResult == entryToUpdate + // here. If that changes, this is unstable: map iteration order isn't fixed, and + // we return on the first running result found, so the message shown may not be + // from the work result that was just updated. + builderStatus := workResult.WorkResult.GetStatus() + status.SetMessage(builderStatus.Message) + default: + // Don't do anything else if all work requests haven't reached a terminal state + // or aren't failed. Reflect active execution once any worker reports progress, + // but don't finalize job state until all work requests have finished or need + // intervention. + status.SetMessage("one or more work requests are in progress") + } + } + return nil } // Verify all work requests have reached the same terminal state. @@ -1398,12 +1423,7 @@ func (m *Manager) UpdateWork(workResult *flex.Work) error { if !job.InActiveState() { if err := m.releaseUnusedFileLockFunc(workResult.GetPath(), pathEntry.Value); err != nil { status.SetState(beeremote.Job_FAILED) - message := "unable to clear lock: " + err.Error() - if status.Message != "" { - status.SetMessage(fmt.Sprintf("%s; %s", status.Message, message)) - } else { - status.SetMessage(message) - } + status.SetMessage(appendMessage(status.Message, "unable to clear lock: "+err.Error())) } } }() diff --git a/rst/remote/internal/job/manager_test.go b/rst/remote/internal/job/manager_test.go index 2983101e..e3c511cd 100644 --- a/rst/remote/internal/job/manager_test.go +++ b/rst/remote/internal/job/manager_test.go @@ -1020,6 +1020,158 @@ func TestUpdateWorkIgnoresTerminalStateJob(t *testing.T) { } } +func TestUpdateJobsCancelsFailedBuilderJobUsingStoredJobBuilderInfo(t *testing.T) { + tmpPathDBPath, cleanupPathDBPath, err := tempPathForTesting(testDBBasePath) + require.NoError(t, err, "error setting up for test") + defer cleanupPathDBPath(t) + + log, err := logger.New(logger.Config{Type: "stdout", Level: 5}, nil) + require.NoError(t, err) + workerMgrConfig := workermgr.Config{} + workerConfigs := []worker.Config{ + { + ID: "0", + Name: "test-node-0", + Type: worker.Mock, + MaxReconnectBackOff: 5, + MockConfig: worker.MockConfig{ + Expectations: []worker.MockExpectation{ + { + MethodName: "connect", + ReturnArgs: []any{false, nil}, + }, + { + MethodName: "SubmitWork", + Args: []any{mock.Anything}, + ReturnArgs: []any{ + flex.Work_Status_builder{ + State: flex.Work_SCHEDULED, + Message: "test expects a scheduled request", + }.Build(), + nil, + }, + }, + { + MethodName: "disconnect", + ReturnArgs: []any{nil}, + }, + }, + }, + }, + } + + mountPoint := filesystem.NewMockFS() + remoteStorageTargets := []*flex.RemoteStorageTarget{flex.RemoteStorageTarget_builder{Id: 1, Mock: new("test")}.Build()} + workerManager, err := workermgr.NewManager(context.Background(), log, workerMgrConfig, workerConfigs, remoteStorageTargets, &flex.BeeRemoteNode{}, mountPoint, map[string]*flex.Feature{}) + require.NoError(t, err) + require.NoError(t, workerManager.Start()) + + mockRST, ok := workerManager.RemoteStorageTargets[1].(*rst.MockClient) + require.True(t, ok) + + jobMgrConfig := Config{ + PathDBPath: tmpPathDBPath, + } + + jobManager := NewManager(log, jobMgrConfig, workerManager, withIgnoreReleaseUnusedFileLockFunc()) + require.NoError(t, jobManager.Start()) + + builderRequest := beeremote.JobRequest_builder{ + Path: "/test/builder", + Name: "builder job", + Priority: 3, + RemoteStorageTarget: 1, + Builder: flex.BuilderJob_builder{ + Cfg: flex.JobRequestCfg_builder{ + Path: "/test/builder", + RemoteStorageTarget: 1, + Download: true, + RemotePath: "remote/test/builder", + }.Build(), + }.Build(), + }.Build() + + mockRST.On("GenerateWorkRequests", mock.MatchedBy(func(job *beeremote.Job) bool { + return job.GetRequest().HasBuilder() && + job.GetRequest().GetPath() == builderRequest.GetPath() && + job.GetRequest().GetRemoteStorageTarget() == builderRequest.GetRemoteStorageTarget() + }), 0).Return([]*flex.WorkRequest{ + flex.WorkRequest_builder{ + JobId: "ignored-by-worker-manager", + RequestId: "0", + Path: builderRequest.GetPath(), + RemoteStorageTarget: 1, + Mock: flex.MockJob_builder{}.Build(), + }.Build(), + }, nil, nil).Once() + + jobResponse, err := jobManager.SubmitJobRequest(builderRequest) + require.NoError(t, err) + require.NotNil(t, jobResponse) + + jobID := jobResponse.GetJob().GetId() + bulkStateMountPath := ".beegfs-rst/job/" + jobID + "/1" + expectedJobBuilderInfo := flex.Work_JobBuilderInfo_builder{ + BulkOperations: []*flex.BulkOperation{ + flex.BulkOperation_builder{ + StateMountPath: bulkStateMountPath, + RstId: 1, + Operation: "retrieve", + }.Build(), + }, + }.Build() + mockRST.On("CompleteWorkRequests", mock.MatchedBy(func(job *beeremote.Job) bool { + return job.GetId() == jobID && job.GetRequest().HasBuilder() + }), mock.MatchedBy(func(workResults []*flex.Work) bool { + return len(workResults) == 1 && + workResults[0].GetRequestId() == "0" && + workResults[0].HasJobBuilderInfo() && + proto.Equal(workResults[0].GetJobBuilderInfo(), expectedJobBuilderInfo) + }), true).Return(nil).Once() + + workResult := flex.Work_builder{ + Path: builderRequest.GetPath(), + JobId: jobID, + RequestId: "0", + Status: flex.Work_Status_builder{ + State: flex.Work_FAILED, + Message: "job builder failed to complete bulk operation(s): bulk restore session failed", + }.Build(), + Parts: []*flex.Work_Part{}, + JobBuilderInfo: expectedJobBuilderInfo, + }.Build() + + err = jobManager.UpdateWork(workResult) + require.NoError(t, err) + + getJobsRequest := beeremote.GetJobsRequest_builder{ + ByJobIdAndPath: beeremote.GetJobsRequest_QueryIdAndPath_builder{ + JobId: jobID, + Path: builderRequest.GetPath(), + }.Build(), + IncludeWorkRequests: false, + IncludeWorkResults: true, + }.Build() + responses := make(chan *beeremote.GetJobsResponse, 1) + err = jobManager.GetJobs(context.Background(), getJobsRequest, responses) + require.NoError(t, err) + getJobsResponse := <-responses + require.Equal(t, beeremote.Job_FAILED, getJobsResponse.GetResults()[0].GetJob().GetStatus().GetState()) + require.True(t, getJobsResponse.GetResults()[0].GetWorkResults()[0].GetWork().HasJobBuilderInfo()) + + updateJobRequest := beeremote.UpdateJobsRequest_builder{ + JobId: new(jobID), + Path: builderRequest.GetPath(), + NewState: beeremote.UpdateJobsRequest_CANCELLED, + }.Build() + updateJobResponse, err := jobManager.UpdateJobs(updateJobRequest) + require.NoError(t, err) + require.True(t, updateJobResponse.GetOk()) + require.Equal(t, beeremote.Job_CANCELLED, updateJobResponse.GetResults()[0].GetJob().GetStatus().GetState()) + + mockRST.AssertExpectations(t) +} + func TestSubmitJobRequestSentinelErrorHandling(t *testing.T) { tmpPathDBPath, cleanupPathDBPath, err := tempPathForTesting(testDBBasePath) require.NoError(t, err, "error setting up for test") diff --git a/rst/remote/internal/job/utils.go b/rst/remote/internal/job/utils.go index b947ee71..1638d51f 100644 --- a/rst/remote/internal/job/utils.go +++ b/rst/remote/internal/job/utils.go @@ -18,3 +18,10 @@ func getProtoWorkResults(workResults map[string]worker.WorkResult) []*beeremote. } return workResultsForResponse } + +func appendMessage(original string, addition string) string { + if original == "" { + return addition + } + return original + "; " + addition +} diff --git a/rst/sync/internal/beeremote/client.go b/rst/sync/internal/beeremote/client.go index 68f05884..965667e5 100644 --- a/rst/sync/internal/beeremote/client.go +++ b/rst/sync/internal/beeremote/client.go @@ -147,13 +147,7 @@ func (c *Client) SubmitJobRequest(ctx context.Context, jobRequest *beeremote.Job return fmt.Errorf("BeeRemote client is not ready") } - if err := c.submitJob(ctx, jobRequest); err != nil { - if _, ok := status.FromError(err); !ok { - return fmt.Errorf("received an unknown error (no gRPC status), most likely this is a bug: %w", err) - } - return err - } - return nil + return c.submitJob(ctx, jobRequest) } func (c *Client) Disconnect() error { diff --git a/rst/sync/internal/beeremote/errors.go b/rst/sync/internal/beeremote/errors.go index e1c7bcfb..696e25bc 100644 --- a/rst/sync/internal/beeremote/errors.go +++ b/rst/sync/internal/beeremote/errors.go @@ -4,6 +4,7 @@ import "errors" var ( ErrNilConfiguration = errors.New("cannot apply nil configuration") - ErrInvalidAddress = errors.New("address provided for BeeRemote is invalid") - ErrUnableToConnect = errors.New("unable to setup connection to BeeRemote") + ErrInvalidAddress = errors.New("address provided for BeeGFS Remote is invalid") + ErrUnableToConnect = errors.New("unable to setup connection to BeeGFS Remote") + ErrUnavailable = errors.New("BeeGFS Remote is unavailable") ) diff --git a/rst/sync/internal/beeremote/grpc.go b/rst/sync/internal/beeremote/grpc.go index 97d37198..2853c63f 100644 --- a/rst/sync/internal/beeremote/grpc.go +++ b/rst/sync/internal/beeremote/grpc.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/thinkparq/beegfs-go/common/beegfs/beegrpc" + "github.com/thinkparq/beegfs-go/common/rst" "github.com/thinkparq/beegfs-go/ctl/pkg/config" "github.com/thinkparq/protobuf/go/beeremote" "github.com/thinkparq/protobuf/go/flex" @@ -105,17 +106,30 @@ func (c *grpcProvider) updateWork(ctx context.Context, workResult *flex.Work) er } func (c *grpcProvider) submitJob(ctx context.Context, jobRequest *beeremote.JobRequest) error { - _, err := c.client.SubmitJob(ctx, beeremote.SubmitJobRequest_builder{Request: jobRequest}.Build()) + resp, err := c.client.SubmitJob(ctx, beeremote.SubmitJobRequest_builder{Request: jobRequest}.Build()) if err != nil { if st, ok := status.FromError(err); ok { // TLS misconfiguration can cause a confusing error message so we handle it explicitly. // Note this is just a hint to the user, other error conditions may have the same // message so we don't adjust behavior (i.e., treat it as fatal). if strings.Contains(st.Message(), "error reading server preface: EOF") { - return fmt.Errorf("%w (hint: check TLS is configured correctly on the client and server)", err) + err = fmt.Errorf("%w (hint: check TLS is configured correctly on the client and server)", err) } } - return err + return fmt.Errorf("%w: %w", ErrUnavailable, err) + } + + switch resp.GetStatus() { + case beeremote.SubmitJobResponse_ALREADY_COMPLETE: + return rst.ErrJobAlreadyComplete + case beeremote.SubmitJobResponse_ALREADY_OFFLOADED: + return rst.ErrJobAlreadyOffloaded + case beeremote.SubmitJobResponse_EXISTING: + return rst.ErrJobAlreadyExists + case beeremote.SubmitJobResponse_NOT_ALLOWED: + return rst.ErrJobNotAllowed + case beeremote.SubmitJobResponse_FAILED_PRECONDITION: + return rst.ErrJobFailedPrecondition } return nil diff --git a/rst/sync/internal/workmgr/manager.go b/rst/sync/internal/workmgr/manager.go index a854a557..53250416 100644 --- a/rst/sync/internal/workmgr/manager.go +++ b/rst/sync/internal/workmgr/manager.go @@ -4,10 +4,12 @@ import ( "context" "errors" "fmt" + "math" "path" "reflect" "strconv" "sync" + "sync/atomic" "time" "github.com/dgraph-io/badger/v4" @@ -110,7 +112,7 @@ type Manager struct { // entry while its being processed because it would be better to block another goroutine than // risk two goroutines acting on the same entry concurrently. This should only ever happen if // there is a bug. - workJournal *kvstore.MapStore[workEntry] + workJournal *kvstore.MapStore[*workEntry] // The jobStore keeps a mapping of job IDs to submission IDs in the journal for each of their // work requests. The inner map is a map of work request IDs to their submission ID in the // workJournal. This allows the worker node to handle multiple work request for a single job. @@ -194,7 +196,7 @@ func NewAndStart(log *logger.Logger, config Config, beeRemoteClient *beeremote.C // Setup work journal: workJournalOpts := badger.DefaultOptions(m.config.WorkJournalPath) workJournalOpts = workJournalOpts.WithLogger(logger.NewBadgerLoggerBridge("workJournal", m.log.Logger)) - workJournal, closeWorkJournal, err := kvstore.NewMapStore[workEntry](workJournalOpts) + workJournal, closeWorkJournal, err := kvstore.NewMapStore[*workEntry](workJournalOpts) if err != nil { return nil, fmt.Errorf("unable to setup work journal: %w", err) } @@ -289,6 +291,8 @@ func (m *Manager) manage(deferredFuncs []func() error) { m.mgrWG.Done() }() + workerSaturation := m.startUpdateWorkerSaturation(time.Second, 60*time.Second) + // completedWork is how workers signal when they are no longer working on a request. It may have // been completed successfully or cancelled, but either way it should be removed from the active // work map and new request(s) can be pulled to the active work queue and map. @@ -305,6 +309,7 @@ func (m *Manager) manage(deferredFuncs []func() error) { jobStore: m.jobStore, beeRemoteClient: m.beeRemoteClient, rescheduleWork: m.scheduler.AddRescheduleWorkToken, + workerSaturation: workerSaturation, metrics: m.metrics, } m.workerWG.Add(1) @@ -383,6 +388,75 @@ func (m *Manager) manage(deferredFuncs []func() error) { } } +// startUpdateWorkerSaturation starts a go routine that every second recomputes decayed worker +// saturation averages for the given windows (e.g. 1s, 60s) from the activeWork map's occupancy +// relative to NumWorkers, so workers can read current saturation through the returned pointers +// without needing direct access to the manager. Saturation is expressed as a percentage of +// NumWorkers: 100 means as many work items are in flight as there are workers (roughly "every +// worker has work"), and it is intentionally allowed to exceed 100 when a backlog builds up beyond +// worker capacity. Note activeWorkQueue's occupancy is not used here: workers pull off it almost +// immediately, so its length stays near zero regardless of how busy the workers actually are, +// against a capacity (ActiveWorkQueueSize) sized in the tens of thousands. activeWork instead +// counts every work item that is queued or actively being processed, which is a meaningful +// fraction of NumWorkers. Only one goroutine may ever drive a given set of returned pointers this +// way. + +// startUpdateWorkerSaturation returns a list of getter functions for worker saturation over each +// window duration. The worker saturation is the moving average of active-work per workers expressed +// as a percentage. where samples are updated every second. 100% means there is a job for every +// worker. +func (m *Manager) startUpdateWorkerSaturation(windows ...time.Duration) []func() float64 { + tick := time.Second + tickSeconds := tick.Seconds() + decay := make([]float64, len(windows)) + for i, window := range windows { + decay[i] = math.Exp(-tickSeconds / window.Seconds()) + } + + saturation := make([]atomic.Uint64, len(windows)) + statFns := make([]func() float64, len(windows)) + for i := range saturation { + statFns[i] = func() float64 { + return math.Float64frombits(saturation[i].Load()) + } + } + + m.mgrWG.Go(func() { + ticker := time.NewTicker(tick) + defer ticker.Stop() + + for { + select { + case <-m.mgrCtx.Done(): + return + case <-ticker.C: + numWorkers := m.config.NumWorkers + if numWorkers <= 0 { + for i := range windows { + saturation[i].Store(0) + } + continue + } + + current := float64(m.activeWorkLen()) / float64(numWorkers) * 100 + for i := range windows { + average := math.Float64frombits(saturation[i].Load())*decay[i] + current*(1-decay[i]) + saturation[i].Store(math.Float64bits(average)) + } + } + } + }) + + return statFns +} + +// activeWorkLen returns the current number of in-flight work items (queued or being processed). +func (m *Manager) activeWorkLen() int { + m.activeWorkMu.RLock() + defer m.activeWorkMu.RUnlock() + return len(m.activeWork) +} + // pullInWork moves ready work from the priority range to the activeWork map. func (m *Manager) pullInWork(start string, stop string, availableTokens *int) (nextSubmissionId string, err error) { nextSubmissionId = start @@ -564,8 +638,10 @@ func (m *Manager) initScheduler(priority int, start string, stop string) (entrie return } - var rescheduledCount int var scheduledCount int + var rescheduledCount int + var replayCount int + var unrecoverableCount int var submissionId string isNextSubmissionIdSet := false workRequestPriority := priorityIdMap[int32(priority+1)] @@ -573,7 +649,16 @@ func (m *Manager) initScheduler(priority int, start string, stop string) (entrie submissionId = submission.Key entry := submission.Entry.Value - if entry.ExecuteAfter.IsZero() { + var status *flex.Work_Status + var state flex.Work_State + if entry.WorkResult != nil { + if status = entry.WorkResult.GetStatus(); status != nil { + state = status.GetState() + } + } + + switch state { + case flex.Work_SCHEDULED: m.scheduler.AddWorkToken(submissionId) if !isNextSubmissionIdSet { m.scheduler.SetNextSubmissionId(submissionId, priority) @@ -586,7 +671,7 @@ func (m *Manager) initScheduler(priority int, start string, stop string) (entrie attrPriority.Int(priority+1), ), ) - } else { + case flex.Work_RESCHEDULED: m.scheduler.AddRescheduleWorkToken(submissionId, entry.ExecuteAfter) rescheduledCount++ m.metrics.workRequests.Add(context.Background(), 1, @@ -595,9 +680,27 @@ func (m *Manager) initScheduler(priority int, start string, stop string) (entrie attrPriority.Int(priority+1), ), ) + + case flex.Work_RUNNING, flex.Work_COMPLETED: + // Submission has already been scheduled and executed so treat it as rescheduled work + // since this preserves the next submissionId priority scheduler boundary. + m.scheduler.AddRescheduleWorkToken(submissionId, time.Time{}) + replayCount++ + default: + m.log.Warn("skipping unrecoverable work journal entry during scheduler init", + zap.String("submissionId", submissionId), + zap.String("jobId", entry.WorkRequest.GetJobId()), + zap.String("requestId", entry.WorkRequest.GetRequestId()), + zap.String("state", state.String()), + zap.Time("executeAfter", entry.ExecuteAfter), + zap.Bool("hasWorkResult", entry.WorkResult != nil), + zap.Bool("hasStatus", status != nil), + ) + m.scheduler.AddRescheduleWorkToken(submissionId, time.Time{}) + unrecoverableCount++ } - entriesFound++ + entriesFound++ submission, err = nextItem() if err != nil { err = fmt.Errorf("unable to get work journal entry: %w", err) @@ -605,7 +708,8 @@ func (m *Manager) initScheduler(priority int, start string, stop string) (entrie } } - if scheduledCount == 0 && rescheduledCount > 0 { + allRescheduledCount := rescheduledCount + replayCount + if scheduledCount == 0 && allRescheduledCount > 0 { // All recovered were rescheduled work request so increment the last known rescheduled // submissionId to get the nextExpectedSubmissionId. nextExpectedSubmissionId, _, err := scheduler.IncrementSubmissionId(submissionId) @@ -616,8 +720,14 @@ func (m *Manager) initScheduler(priority int, start string, stop string) (entrie } } - if scheduledCount > 0 || rescheduledCount > 0 { - m.log.Info(" recovered work requests", zap.String("priority", workRequestPriority), zap.Int("scheduled", scheduledCount), zap.Int("rescheduled", rescheduledCount)) + if scheduledCount > 0 || rescheduledCount > 0 || replayCount > 0 || unrecoverableCount > 0 { + m.log.Info(" recovered work requests", + zap.String("priority", workRequestPriority), + zap.Int("scheduled", scheduledCount), + zap.Int("rescheduled", rescheduledCount), + zap.Int("replayed", replayCount), + zap.Int("unrecoverable", unrecoverableCount), + ) } return } @@ -663,7 +773,7 @@ func (m *Manager) SubmitWorkRequest(wr *flex.WorkRequest) (*flex.Work, error) { } submissionId, priority := scheduler.CreateSubmissionId(key, wr.GetPriority()) - _, workEntry, commitAndReleaseWork, err := m.workJournal.CreateAndLockEntry(submissionId) + _, workEntry, commitAndReleaseWork, err := m.workJournal.CreateAndLockEntry(submissionId, kvstore.WithValue(&workEntry{})) if err != nil { return nil, fmt.Errorf("unable to create work journal entry for job ID %s work request ID %s: %w", jobId, workRequestId, err) } diff --git a/rst/sync/internal/workmgr/manager_test.go b/rst/sync/internal/workmgr/manager_test.go index e08d2ee8..93e84758 100644 --- a/rst/sync/internal/workmgr/manager_test.go +++ b/rst/sync/internal/workmgr/manager_test.go @@ -16,9 +16,12 @@ import ( "github.com/thinkparq/beegfs-go/common/logger" "github.com/thinkparq/beegfs-go/common/rst" "github.com/thinkparq/beegfs-go/rst/sync/internal/beeremote" + pbr "github.com/thinkparq/protobuf/go/beeremote" "github.com/thinkparq/protobuf/go/flex" "go.uber.org/zap/zapcore" "go.uber.org/zap/zaptest" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" ) @@ -71,6 +74,41 @@ func matchRespIDsAndStatus(expectedJobID string, expectedRequestID string, expec }) } +func matchRespIDsStatusAndBuilderBulkOperations(expectedJobID string, expectedRequestID string, expectedState flex.Work_State, expectedBulkOperations []*flex.BulkOperation) any { + return mock.MatchedBy(func(actual *flex.Work) bool { + if actual.GetJobId() != expectedJobID || actual.GetRequestId() != expectedRequestID || actual.GetStatus().GetState() != expectedState { + return false + } + if !actual.HasJobBuilderInfo() { + return false + } + return proto.Equal(actual.GetJobBuilderInfo(), flex.Work_JobBuilderInfo_builder{BulkOperations: expectedBulkOperations}.Build()) + }) +} + +func matchRespIDsStatusAndBuilderInfo(expectedJobID string, expectedRequestID string, expectedState flex.Work_State, expectedBulkOperations []*flex.BulkOperation) any { + return mock.MatchedBy(func(actual *flex.Work) bool { + if actual.GetJobId() != expectedJobID || actual.GetRequestId() != expectedRequestID || actual.GetStatus().GetState() != expectedState { + return false + } + if !actual.HasJobBuilderInfo() { + return false + } + return proto.Equal(actual.GetJobBuilderInfo(), flex.Work_JobBuilderInfo_builder{BulkOperations: expectedBulkOperations}.Build()) + }) +} + +func matchSubmittedJobRequest(expectedPath string, expectedOperation string, expectedJobIndex int64) any { + return mock.MatchedBy(func(actual *pbr.JobRequest) bool { + return actual.GetPath() == expectedPath && + actual.HasSync() && + actual.GetSync().GetOperation() == flex.SyncJob_DOWNLOAD && + actual.HasBulkInfo() && + actual.GetBulkInfo().GetOperation() == expectedOperation && + actual.GetBulkInfo().GetJobIndex() == expectedJobIndex + }) +} + type testConfig struct { Config logLevel int8 @@ -222,6 +260,7 @@ func TestSubmitWorkRequest(t *testing.T) { // First simulate a successful request: mockRST.On("ExecuteWorkRequestPart", mock.Anything, matchJobAndRequestID("0", "0"), mock.Anything).Return(nil).Times(2) mockRST.On("IsWorkRequestReady", matchJobAndRequestID("0", "0")).Return(true, time.Duration(0), nil).Times(1) + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("0", "0", flex.Work_RUNNING)).Return(nil).Times(1) mockBeeRemote.On("updateWork", matchRespIDsAndStatus("0", "0", flex.Work_COMPLETED)).Return(nil).Times(1) testRequest1 := proto.Clone(baseTestRequest).(*flex.WorkRequest) resp, err := mgr.SubmitWorkRequest(testRequest1) @@ -234,6 +273,7 @@ func TestSubmitWorkRequest(t *testing.T) { // Then simulate the RST returning an error (note if an error happens the state is always failed): mockRST.On("ExecuteWorkRequestPart", mock.Anything, matchJobAndRequestID("1", "0"), mock.Anything).Return(fmt.Errorf("test wants an error")).Times(1) mockRST.On("IsWorkRequestReady", matchJobAndRequestID("1", "0")).Return(true, time.Duration(0), nil).Times(1) + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("1", "0", flex.Work_RUNNING)).Return(nil).Times(1) mockBeeRemote.On("updateWork", matchRespIDsAndStatus("1", "0", flex.Work_FAILED)).Return(nil).Times(1) testRequest2 := proto.Clone(baseTestRequest).(*flex.WorkRequest) testRequest2.SetJobId("1") @@ -252,6 +292,7 @@ func TestSubmitWorkRequest(t *testing.T) { // Then simulate the request completing, but it was not able to be sent to BeeRemote. // Also for some "reason" a job ID was skipped, but it should still get picked up. mockRST.On("ExecuteWorkRequestPart", mock.Anything, matchJobAndRequestID("3", "1"), mock.Anything).Return(nil).Times(2) + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("3", "1", flex.Work_RUNNING)).Return(nil).Times(1) mockBeeRemote.On("updateWork", matchRespIDsAndStatus("3", "1", flex.Work_COMPLETED)).Return(fmt.Errorf("test requests a failed response from BeeRemote")) mockRST.On("IsWorkRequestReady", matchJobAndRequestID("3", "1")).Return(true, time.Duration(0), nil).Times(1) testRequest3 := proto.Clone(baseTestRequest).(*flex.WorkRequest) @@ -277,6 +318,563 @@ func TestSubmitWorkRequest(t *testing.T) { // out more than likely something went wrong around how contexts are setup. } +// Verifies a builder work request completes in a single execution, the worker forwards the +// bulk-generated child job request to BeeRemote with its BulkInfo intact, then marks the builder +// work request completed and cleans up its local state. +func TestSubmitBuilderWorkRequestWithBulkOperation_CompletesInSingleExecution(t *testing.T) { + mgr, deferredFuncs, err := getTestManager(t) + defer func() { + for i := len(deferredFuncs) - 1; i >= 0; i-- { + deferredFuncs[i](t) + } + }() + require.NoError(t, err) + defer mgr.Stop() + + mockRST := &rst.MockClient{} + mgr.remoteStorageTargets.SetMockClientForTesting(0, mockRST) + mockBeeRemote, _ := mgr.beeRemoteClient.Provider.(*beeremote.MockProvider) + + // Submit a builder work request that should emit a single bulk-tagged child job. + builderRequest := flex.WorkRequest_builder{ + JobId: "bulk-builder-job", + RequestId: "0", + ExternalId: "builder-extid", + Path: "/bulk/source", + RemoteStorageTarget: 0, + Builder: flex.BuilderJob_builder{ + Cfg: flex.JobRequestCfg_builder{ + Path: "/bulk/source", + RemoteStorageTarget: 0, + Download: true, + RemotePath: "remote/bulk-source", + }.Build(), + }.Build(), + }.Build() + + mockRST.On("IsWorkRequestReady", matchJobAndRequestID("bulk-builder-job", "0")).Return(true, time.Duration(0), nil).Times(1) + mockRST.On("ExecuteJobBuilderRequest", mock.Anything, matchJobAndRequestID("bulk-builder-job", "0"), mock.Anything). + Run(func(args mock.Arguments) { + jobSubmissionChan := args.Get(2).(chan<- *pbr.JobRequest) + jobSubmissionChan <- &pbr.JobRequest{ + Path: "/bulk/source", + RemoteStorageTarget: 0, + Type: &pbr.JobRequest_Sync{ + Sync: &flex.SyncJob{ + Operation: flex.SyncJob_DOWNLOAD, + RemotePath: "remote/bulk-source", + }, + }, + BulkInfo: &flex.BulkJobRequestInfo{ + StateMountPath: ".beegfs-rst/job/bulk-builder-job/0", + Operation: "retrieve", + JobIndex: 0, + }, + } + }). + Return(false, time.Duration(0), nil, nil).Times(1) + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("bulk-builder-job", "0", flex.Work_RUNNING)).Return(nil).Times(1) + mockBeeRemote.On("submitJob", matchSubmittedJobRequest("/bulk/source", "retrieve", 0)).Return(nil).Times(1) + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("bulk-builder-job", "0", flex.Work_COMPLETED)).Return(nil).Times(1) + + // Queue the builder work and let the worker drive it through completion. + resp, err := mgr.SubmitWorkRequest(builderRequest) + require.NoError(t, err) + require.NotNil(t, resp) + + time.Sleep(defaultSleepTime * time.Second) + + // A successful builder flow should leave no persisted or active work behind. + require.NoError(t, assertDBEntriesLenForTesting(mgr, 0)) + require.Len(t, mgr.activeWork, 0) + mockRST.AssertExpectations(t) + mockBeeRemote.AssertExpectations(t) +} + +// Verifies the resume path for builder bulk operations. The first execution generates a bulk child +// job and reschedules the builder work, the second execution represents the bulk operation still +// pending and reschedules again without submitting a new child job, and the third execution +// completes by submitting the final bulk child job. The test checks that the builder work remains +// persisted while the bulk operation is incomplete and is only cleaned up after the bulk flow +// finishes. +func TestSubmitBuilderWorkRequestWithBulkOperation_ReschedulesThenCompletes(t *testing.T) { + mgr, deferredFuncs, err := getTestManager(t) + defer func() { + for i := len(deferredFuncs) - 1; i >= 0; i-- { + deferredFuncs[i](t) + } + }() + require.NoError(t, err) + defer mgr.Stop() + + mockRST := &rst.MockClient{} + mgr.remoteStorageTargets.SetMockClientForTesting(0, mockRST) + mockBeeRemote, _ := mgr.beeRemoteClient.Provider.(*beeremote.MockProvider) + + // Submit a builder work request whose initial walk, later bulk wait, and final completion + // happen across three separate executions. + builderRequest := flex.WorkRequest_builder{ + JobId: "bulk-builder-reschedule-job", + RequestId: "0", + ExternalId: "builder-extid", + Path: "/bulk/source", + RemoteStorageTarget: 0, + Builder: flex.BuilderJob_builder{ + Cfg: flex.JobRequestCfg_builder{ + Path: "/bulk/source", + RemoteStorageTarget: 0, + Download: true, + RemotePath: "remote/bulk-source", + }.Build(), + }.Build(), + }.Build() + + firstRescheduleSent := make(chan struct{}) + secondRescheduleSent := make(chan struct{}) + completedSent := make(chan struct{}) + + getWorkState := func() (flex.Work_State, bool) { + jobEntry, getErr := mgr.jobStore.GetEntry("bulk-builder-reschedule-job") + if getErr != nil { + return flex.Work_CREATED, false + } + submissionID, ok := jobEntry.Value["0"] + if !ok { + return flex.Work_CREATED, false + } + + workEntry, getErr := mgr.workJournal.GetEntry(submissionID) + if getErr != nil { + return flex.Work_CREATED, false + } + return workEntry.Value.WorkResult.GetStatus().GetState(), true + } + + mockRST.On("IsWorkRequestReady", matchJobAndRequestID("bulk-builder-reschedule-job", "0")).Return(true, time.Duration(0), nil).Times(3) + mockRST.On("ExecuteJobBuilderRequest", mock.Anything, matchJobAndRequestID("bulk-builder-reschedule-job", "0"), mock.Anything). + Run(func(args mock.Arguments) { + jobSubmissionChan := args.Get(2).(chan<- *pbr.JobRequest) + jobSubmissionChan <- &pbr.JobRequest{ + Path: "/bulk/source/first", + RemoteStorageTarget: 0, + Type: &pbr.JobRequest_Sync{ + Sync: &flex.SyncJob{ + Operation: flex.SyncJob_DOWNLOAD, + RemotePath: "remote/bulk-source/first", + }, + }, + BulkInfo: &flex.BulkJobRequestInfo{ + StateMountPath: ".beegfs-rst/job/bulk-builder-reschedule-job/0", + Operation: "retrieve", + JobIndex: 0, + }, + } + }). + Return(true, 500*time.Millisecond, nil, nil).Once() + mockRST.On("ExecuteJobBuilderRequest", mock.Anything, matchJobAndRequestID("bulk-builder-reschedule-job", "0"), mock.Anything). + Return(true, 500*time.Millisecond, nil, nil).Once() + mockRST.On("ExecuteJobBuilderRequest", mock.Anything, matchJobAndRequestID("bulk-builder-reschedule-job", "0"), mock.Anything). + Run(func(args mock.Arguments) { + jobSubmissionChan := args.Get(2).(chan<- *pbr.JobRequest) + jobSubmissionChan <- &pbr.JobRequest{ + Path: "/bulk/source/final", + RemoteStorageTarget: 0, + Type: &pbr.JobRequest_Sync{ + Sync: &flex.SyncJob{ + Operation: flex.SyncJob_DOWNLOAD, + RemotePath: "remote/bulk-source/final", + }, + }, + BulkInfo: &flex.BulkJobRequestInfo{ + StateMountPath: ".beegfs-rst/job/bulk-builder-reschedule-job/0", + Operation: "retrieve", + JobIndex: 1, + }, + } + }). + Return(false, time.Duration(0), nil, nil).Once() + + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("bulk-builder-reschedule-job", "0", flex.Work_RUNNING)).Return(nil).Times(3) + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("bulk-builder-reschedule-job", "0", flex.Work_RESCHEDULED)). + Run(func(args mock.Arguments) { close(firstRescheduleSent) }). + Return(nil).Once() + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("bulk-builder-reschedule-job", "0", flex.Work_RESCHEDULED)). + Run(func(args mock.Arguments) { close(secondRescheduleSent) }). + Return(nil).Once() + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("bulk-builder-reschedule-job", "0", flex.Work_COMPLETED)). + Run(func(args mock.Arguments) { close(completedSent) }). + Return(nil).Once() + mockBeeRemote.On("submitJob", matchSubmittedJobRequest("/bulk/source/first", "retrieve", 0)).Return(nil).Times(1) + mockBeeRemote.On("submitJob", matchSubmittedJobRequest("/bulk/source/final", "retrieve", 1)).Return(nil).Times(1) + + // Start the first execution. + resp, err := mgr.SubmitWorkRequest(builderRequest) + require.NoError(t, err) + require.NotNil(t, resp) + + // After the initial builder pass, the request should be rescheduled with persisted state. + select { + case <-firstRescheduleSent: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for first builder reschedule") + } + require.Eventually(t, func() bool { + state, ok := getWorkState() + return ok && state == flex.Work_RESCHEDULED + }, 2*time.Second, 25*time.Millisecond) + require.NoError(t, assertDBEntriesLenForTesting(mgr, 1)) + + // The next execution represents the outstanding bulk operation still waiting to finish. + select { + case <-secondRescheduleSent: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for second builder reschedule") + } + require.Eventually(t, func() bool { + state, ok := getWorkState() + return ok && state == flex.Work_RESCHEDULED + }, 2*time.Second, 25*time.Millisecond) + require.NoError(t, assertDBEntriesLenForTesting(mgr, 1)) + + // The final execution should submit the last child job and clean everything up. + select { + case <-completedSent: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for builder completion") + } + require.Eventually(t, func() bool { + mgr.activeWorkMu.RLock() + activeLen := len(mgr.activeWork) + mgr.activeWorkMu.RUnlock() + return assertDBEntriesLenForTesting(mgr, 0) == nil && activeLen == 0 + }, 2*time.Second, 25*time.Millisecond) + + mockRST.AssertExpectations(t) + mockBeeRemote.AssertExpectations(t) +} + +// Verifies that if a builder execution submits one or more bulk child jobs and then reports a +// ErrBuilderFailed, the worker still forwards the already-emitted child jobs, marks the builder work as +// failed because the bulk operation did not finish, and cleans up the local work state. +func TestSubmitBuilderWorkRequestWithBulkOperation_FailsAfterPartialSubmission(t *testing.T) { + mgr, deferredFuncs, err := getTestManager(t) + defer func() { + for i := len(deferredFuncs) - 1; i >= 0; i-- { + deferredFuncs[i](t) + } + }() + require.NoError(t, err) + defer mgr.Stop() + + mockRST := &rst.MockClient{} + mgr.remoteStorageTargets.SetMockClientForTesting(0, mockRST) + mockBeeRemote, _ := mgr.beeRemoteClient.Provider.(*beeremote.MockProvider) + + builderRequest := flex.WorkRequest_builder{ + JobId: "bulk-builder-bulkerr-job", + RequestId: "0", + ExternalId: "builder-extid", + Path: "/bulk/source", + RemoteStorageTarget: 0, + Builder: flex.BuilderJob_builder{ + Cfg: flex.JobRequestCfg_builder{ + Path: "/bulk/source", + RemoteStorageTarget: 0, + Download: true, + RemotePath: "remote/bulk-source", + }.Build(), + }.Build(), + }.Build() + + failedSent := make(chan struct{}) + + mockRST.On("IsWorkRequestReady", matchJobAndRequestID("bulk-builder-bulkerr-job", "0")).Return(true, time.Duration(0), nil).Times(1) + mockRST.On("ExecuteJobBuilderRequest", mock.Anything, matchJobAndRequestID("bulk-builder-bulkerr-job", "0"), mock.Anything). + Run(func(args mock.Arguments) { + workRequest := args.Get(1).(*flex.WorkRequest) + workRequest.GetBuilder().BulkOperations = []*flex.BulkOperation{ + flex.BulkOperation_builder{ + StateMountPath: ".beegfs-rst/job/bulk-builder-bulkerr-job/1", + RstId: 1, + Operation: "retrieve", + }.Build(), + } + jobSubmissionChan := args.Get(2).(chan<- *pbr.JobRequest) + jobSubmissionChan <- &pbr.JobRequest{ + Path: "/bulk/source/first", + RemoteStorageTarget: 0, + Type: &pbr.JobRequest_Sync{ + Sync: &flex.SyncJob{ + Operation: flex.SyncJob_DOWNLOAD, + RemotePath: "remote/bulk-source/first", + }, + }, + BulkInfo: &flex.BulkJobRequestInfo{ + StateMountPath: ".beegfs-rst/job/bulk-builder-bulkerr-job/0", + Operation: "retrieve", + JobIndex: 0, + }, + } + jobSubmissionChan <- &pbr.JobRequest{ + Path: "/bulk/source/second", + RemoteStorageTarget: 0, + Type: &pbr.JobRequest_Sync{ + Sync: &flex.SyncJob{ + Operation: flex.SyncJob_DOWNLOAD, + RemotePath: "remote/bulk-source/second", + }, + }, + BulkInfo: &flex.BulkJobRequestInfo{ + StateMountPath: ".beegfs-rst/job/bulk-builder-bulkerr-job/0", + Operation: "retrieve", + JobIndex: 1, + }, + } + }). + Return(false, time.Duration(0), rst.MarkBuilderFailed(fmt.Errorf("bulk restore session failed"))).Once() + + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("bulk-builder-bulkerr-job", "0", flex.Work_RUNNING)).Return(nil).Times(1) + mockBeeRemote.On("submitJob", matchSubmittedJobRequest("/bulk/source/first", "retrieve", 0)).Return(nil).Times(1) + mockBeeRemote.On("submitJob", matchSubmittedJobRequest("/bulk/source/second", "retrieve", 1)).Return(nil).Times(1) + mockBeeRemote.On("updateWork", matchRespIDsStatusAndBuilderBulkOperations("bulk-builder-bulkerr-job", "0", flex.Work_FAILED, []*flex.BulkOperation{ + flex.BulkOperation_builder{ + StateMountPath: ".beegfs-rst/job/bulk-builder-bulkerr-job/1", + RstId: 1, + Operation: "retrieve", + }.Build(), + })). + Run(func(args mock.Arguments) { close(failedSent) }). + Return(nil).Once() + + resp, err := mgr.SubmitWorkRequest(builderRequest) + require.NoError(t, err) + require.NotNil(t, resp) + + select { + case <-failedSent: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for builder failure after ErrBuilderFailed") + } + + require.Eventually(t, func() bool { + mgr.activeWorkMu.RLock() + activeLen := len(mgr.activeWork) + mgr.activeWorkMu.RUnlock() + return assertDBEntriesLenForTesting(mgr, 0) == nil && activeLen == 0 + }, 2*time.Second, 25*time.Millisecond) + + mockRST.AssertExpectations(t) + mockBeeRemote.AssertExpectations(t) +} + +// Verifies that if a builder reports ErrBuilderFailed before any bulk operations were actually created, +// the worker still reports the builder as failed and includes an empty JobBuilderInfo so the +// provider can decide whether any cleanup is needed. +func TestSubmitBuilderWorkRequestWithBulkErrAndNoBulkOperations_FailsWithEmptyBuilderInfo(t *testing.T) { + mgr, deferredFuncs, err := getTestManager(t) + defer func() { + for i := len(deferredFuncs) - 1; i >= 0; i-- { + deferredFuncs[i](t) + } + }() + require.NoError(t, err) + defer mgr.Stop() + + mockRST := &rst.MockClient{} + mgr.remoteStorageTargets.SetMockClientForTesting(0, mockRST) + mockBeeRemote, _ := mgr.beeRemoteClient.Provider.(*beeremote.MockProvider) + + builderRequest := flex.WorkRequest_builder{ + JobId: "bulk-builder-no-bulkops-job", + RequestId: "0", + ExternalId: "builder-extid", + Path: "/bulk/source", + RemoteStorageTarget: 0, + Builder: flex.BuilderJob_builder{ + Cfg: flex.JobRequestCfg_builder{ + Path: "/bulk/source", + RemoteStorageTarget: 0, + Download: true, + RemotePath: "remote/bulk-source", + }.Build(), + }.Build(), + }.Build() + + failedSent := make(chan struct{}) + + mockRST.On("IsWorkRequestReady", matchJobAndRequestID("bulk-builder-no-bulkops-job", "0")).Return(true, time.Duration(0), nil).Times(1) + mockRST.On("ExecuteJobBuilderRequest", mock.Anything, matchJobAndRequestID("bulk-builder-no-bulkops-job", "0"), mock.Anything). + Return(false, time.Duration(0), rst.MarkBuilderFailed(fmt.Errorf("bulk restore session failed before session creation"))).Once() + + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("bulk-builder-no-bulkops-job", "0", flex.Work_RUNNING)).Return(nil).Times(1) + mockBeeRemote.On("updateWork", matchRespIDsStatusAndBuilderInfo("bulk-builder-no-bulkops-job", "0", flex.Work_FAILED, []*flex.BulkOperation{})). + Run(func(args mock.Arguments) { + actual := args.Get(0).(*flex.Work) + require.Contains(t, actual.GetStatus().GetMessage(), "bulk restore session failed before session creation") + close(failedSent) + }). + Return(nil).Once() + + resp, err := mgr.SubmitWorkRequest(builderRequest) + require.NoError(t, err) + require.NotNil(t, resp) + + select { + case <-failedSent: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for builder failure after ErrBuilderFailed without bulk operations") + } + + require.Eventually(t, func() bool { + mgr.activeWorkMu.RLock() + activeLen := len(mgr.activeWork) + mgr.activeWorkMu.RUnlock() + return assertDBEntriesLenForTesting(mgr, 0) == nil && activeLen == 0 + }, 2*time.Second, 25*time.Millisecond) + + mockRST.AssertExpectations(t) + mockBeeRemote.AssertExpectations(t) +} + +// Verifies that the worker does not perform any duplicate-child suppression itself across builder +// reschedule/resume. If the builder emits the same bulk child job again on a later execution, the +// worker forwards it again to BeeRemote, which then rejects it as AlreadyExists and causes the +// builder work to end in a cancelled/error-counted terminal state. +func TestSubmitBuilderWorkRequestWithBulkOperation_DuplicateChildSubmissionAcrossResume(t *testing.T) { + mgr, deferredFuncs, err := getTestManager(t) + defer func() { + for i := len(deferredFuncs) - 1; i >= 0; i-- { + deferredFuncs[i](t) + } + }() + require.NoError(t, err) + defer mgr.Stop() + + mockRST := &rst.MockClient{} + mgr.remoteStorageTargets.SetMockClientForTesting(0, mockRST) + mockBeeRemote, _ := mgr.beeRemoteClient.Provider.(*beeremote.MockProvider) + + builderRequest := flex.WorkRequest_builder{ + JobId: "bulk-builder-duplicate-job", + RequestId: "0", + ExternalId: "builder-extid", + Path: "/bulk/source", + RemoteStorageTarget: 0, + Builder: flex.BuilderJob_builder{ + Cfg: flex.JobRequestCfg_builder{ + Path: "/bulk/source", + RemoteStorageTarget: 0, + Download: true, + RemotePath: "remote/bulk-source", + }.Build(), + }.Build(), + }.Build() + + firstRescheduleSent := make(chan struct{}) + cancelledSent := make(chan struct{}) + + getWorkState := func() (flex.Work_State, bool) { + jobEntry, getErr := mgr.jobStore.GetEntry("bulk-builder-duplicate-job") + if getErr != nil { + return flex.Work_CREATED, false + } + submissionID, ok := jobEntry.Value["0"] + if !ok { + return flex.Work_CREATED, false + } + + workEntry, getErr := mgr.workJournal.GetEntry(submissionID) + if getErr != nil { + return flex.Work_CREATED, false + } + return workEntry.Value.WorkResult.GetStatus().GetState(), true + } + + mockRST.On("IsWorkRequestReady", matchJobAndRequestID("bulk-builder-duplicate-job", "0")).Return(true, time.Duration(0), nil).Times(2) + mockRST.On("ExecuteJobBuilderRequest", mock.Anything, matchJobAndRequestID("bulk-builder-duplicate-job", "0"), mock.Anything). + Run(func(args mock.Arguments) { + jobSubmissionChan := args.Get(2).(chan<- *pbr.JobRequest) + jobSubmissionChan <- &pbr.JobRequest{ + Path: "/bulk/source/duplicate", + RemoteStorageTarget: 0, + Type: &pbr.JobRequest_Sync{ + Sync: &flex.SyncJob{ + Operation: flex.SyncJob_DOWNLOAD, + RemotePath: "remote/bulk-source/duplicate", + }, + }, + BulkInfo: &flex.BulkJobRequestInfo{ + StateMountPath: ".beegfs-rst/job/bulk-builder-duplicate-job/0", + Operation: "retrieve", + JobIndex: 0, + }, + } + }). + Return(true, 500*time.Millisecond, nil, nil).Once() + mockRST.On("ExecuteJobBuilderRequest", mock.Anything, matchJobAndRequestID("bulk-builder-duplicate-job", "0"), mock.Anything). + Run(func(args mock.Arguments) { + jobSubmissionChan := args.Get(2).(chan<- *pbr.JobRequest) + jobSubmissionChan <- &pbr.JobRequest{ + Path: "/bulk/source/duplicate", + RemoteStorageTarget: 0, + Type: &pbr.JobRequest_Sync{ + Sync: &flex.SyncJob{ + Operation: flex.SyncJob_DOWNLOAD, + RemotePath: "remote/bulk-source/duplicate", + }, + }, + BulkInfo: &flex.BulkJobRequestInfo{ + StateMountPath: ".beegfs-rst/job/bulk-builder-duplicate-job/0", + Operation: "retrieve", + JobIndex: 0, + }, + } + }). + Return(false, time.Duration(0), nil, nil).Once() + + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("bulk-builder-duplicate-job", "0", flex.Work_RUNNING)).Return(nil).Times(2) + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("bulk-builder-duplicate-job", "0", flex.Work_RESCHEDULED)). + Run(func(args mock.Arguments) { close(firstRescheduleSent) }). + Return(nil).Once() + mockBeeRemote.On("submitJob", matchSubmittedJobRequest("/bulk/source/duplicate", "retrieve", 0)).Return(nil).Once() + mockBeeRemote.On("submitJob", matchSubmittedJobRequest("/bulk/source/duplicate", "retrieve", 0)). + Return(status.Error(codes.AlreadyExists, "duplicate child job")).Once() + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("bulk-builder-duplicate-job", "0", flex.Work_CANCELLED)). + Run(func(args mock.Arguments) { close(cancelledSent) }). + Return(nil).Once() + + resp, err := mgr.SubmitWorkRequest(builderRequest) + require.NoError(t, err) + require.NotNil(t, resp) + + select { + case <-firstRescheduleSent: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for first builder reschedule") + } + + require.Eventually(t, func() bool { + state, ok := getWorkState() + return ok && state == flex.Work_RESCHEDULED + }, 2*time.Second, 25*time.Millisecond) + + select { + case <-cancelledSent: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for duplicate child submission result") + } + + require.Eventually(t, func() bool { + mgr.activeWorkMu.RLock() + activeLen := len(mgr.activeWork) + mgr.activeWorkMu.RUnlock() + return assertDBEntriesLenForTesting(mgr, 0) == nil && activeLen == 0 + }, 2*time.Second, 25*time.Millisecond) + + mockRST.AssertExpectations(t) + mockBeeRemote.AssertExpectations(t) +} + // This test intentionally reuses the same job ID (potentially with different requests for that // job), to also verify handling when the node has multiple requests for the same job. func TestUpdateRequests(t *testing.T) { @@ -300,9 +898,13 @@ func TestUpdateRequests(t *testing.T) { // Simulate a request that isn't completed due to an error from the RST (note if an error // happens the state is always failed). Force the the request to stay active because it can't // send a response to BeeRemote. + failedSent := make(chan struct{}) mockRST.On("ExecuteWorkRequestPart", mock.Anything, matchJobAndRequestID("1", "2"), mock.Anything).Return(fmt.Errorf("test wants an error")).Times(1) mockRST.On("IsWorkRequestReady", matchJobAndRequestID("1", "2")).Return(true, time.Duration(0), nil).Times(1) - mockBeeRemote.On("updateWork", matchRespIDsAndStatus("1", "2", flex.Work_FAILED)).Return(fmt.Errorf("test requests a failed response from BeeRemote")) + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("1", "2", flex.Work_RUNNING)).Return(nil).Times(1) + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("1", "2", flex.Work_FAILED)). + Run(func(args mock.Arguments) { close(failedSent) }). + Return(fmt.Errorf("test requests a failed response from BeeRemote")) testRequest2 := proto.Clone(baseTestRequest).(*flex.WorkRequest) testRequest2.SetJobId("1") testRequest2.SetRequestId("2") @@ -310,8 +912,11 @@ func TestUpdateRequests(t *testing.T) { require.NoError(t, err) require.NotNil(t, resp) - // Sleep to allow the request enough time to get to an error state: - time.Sleep(defaultSleepTime * time.Second) + select { + case <-failedSent: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for failed work result") + } // Now try to cancel the request: updateRequest := flex.UpdateWorkRequest_builder{ @@ -328,9 +933,13 @@ func TestUpdateRequests(t *testing.T) { // Resubmit the same job ID and request. This time there is no error on the RST. // Force the the request to stay active because it can't send a response to BeeRemote. + failedSent = make(chan struct{}) mockRST.On("ExecuteWorkRequestPart", mock.Anything, matchJobAndRequestID("1", "2"), mock.Anything).Return(nil).Times(2) mockRST.On("IsWorkRequestReady", matchJobAndRequestID("1", "2")).Return(true, time.Duration(0), nil).Times(1) - mockBeeRemote.On("updateWork", matchRespIDsAndStatus("1", "2", flex.Work_COMPLETED)).Return(fmt.Errorf("test requests a failed response from BeeRemote")) + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("1", "2", flex.Work_RUNNING)).Return(nil).Times(1) + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("1", "2", flex.Work_COMPLETED)). + Run(func(args mock.Arguments) { close(failedSent) }). + Return(fmt.Errorf("test requests a failed response from BeeRemote")) testRequest2_2 := proto.Clone(baseTestRequest).(*flex.WorkRequest) testRequest2_2.SetJobId("1") testRequest2_2.SetRequestId("2") @@ -338,8 +947,11 @@ func TestUpdateRequests(t *testing.T) { require.NoError(t, err) require.NotNil(t, resp) - // Sleep to allow the request enough time to get to an error state: - time.Sleep(defaultSleepTime * time.Second) + select { + case <-failedSent: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for completed work result") + } // We should not be able to cancel completed requests: resp, err = mgr.UpdateWork(updateRequest) @@ -353,9 +965,13 @@ func TestUpdateRequests(t *testing.T) { // worker should no longer be trying to send the request to BeeRemote making it available for // another request. Force the the request to stay active (tying up the worker) because it can't // send a response to BeeRemote. + failedSent = make(chan struct{}) mockRST.On("ExecuteWorkRequestPart", mock.Anything, matchJobAndRequestID("1", "3"), mock.Anything).Return(nil).Times(2) mockRST.On("IsWorkRequestReady", matchJobAndRequestID("1", "3")).Return(true, time.Duration(0), nil).Times(1) - mockBeeRemote.On("updateWork", matchRespIDsAndStatus("1", "3", flex.Work_COMPLETED)).Return(fmt.Errorf("test requests a failed response from BeeRemote")) + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("1", "3", flex.Work_RUNNING)).Return(nil).Times(1) + mockBeeRemote.On("updateWork", matchRespIDsAndStatus("1", "3", flex.Work_COMPLETED)). + Run(func(args mock.Arguments) { close(failedSent) }). + Return(fmt.Errorf("test requests a failed response from BeeRemote")) testRequest3 := proto.Clone(baseTestRequest).(*flex.WorkRequest) testRequest3.SetJobId("1") testRequest3.SetRequestId("3") @@ -363,8 +979,11 @@ func TestUpdateRequests(t *testing.T) { require.NoError(t, err) require.NotNil(t, resp) - // Sleep to allow enough time for the request to get picked up and become active - time.Sleep(defaultSleepTime * time.Second) + select { + case <-failedSent: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for third completed work result") + } // Now submit another request for the same job: testRequest4 := proto.Clone(baseTestRequest).(*flex.WorkRequest) diff --git a/rst/sync/internal/workmgr/utils.go b/rst/sync/internal/workmgr/utils.go index 00e0c0dd..270a16fd 100644 --- a/rst/sync/internal/workmgr/utils.go +++ b/rst/sync/internal/workmgr/utils.go @@ -21,9 +21,10 @@ func tempPathForTesting(path string) (string, func(tb testing.TB), error) { } cleanup := func(tb testing.TB) { - // If we cleanup to quickly the DB may not have shutdown. - time.Sleep(1 * time.Second) - require.NoError(tb, os.RemoveAll(tempDBPath), "error cleaning up after test") + // Poll for cleanup instead of sleeping a full second per temp path. + require.Eventually(tb, func() bool { + return os.RemoveAll(tempDBPath) == nil + }, 2*time.Second, 25*time.Millisecond, "error cleaning up after test") } return tempDBPath, cleanup, nil @@ -179,3 +180,10 @@ func generatePartsFromSegment(segment *flex.WorkRequest_Segment) func() (int32, return partNumber, offsetStart, offsetStop } } + +func appendMessage(original string, addition string) string { + if original == "" { + return addition + } + return original + "; " + addition +} diff --git a/rst/sync/internal/workmgr/work.go b/rst/sync/internal/workmgr/work.go index a5085282..5aa7e9ec 100644 --- a/rst/sync/internal/workmgr/work.go +++ b/rst/sync/internal/workmgr/work.go @@ -5,16 +5,20 @@ import ( "encoding/gob" "errors" "fmt" + "runtime" + "strings" "sync" "time" "github.com/thinkparq/beegfs-go/common/kvstore" "github.com/thinkparq/beegfs-go/common/rst" + "github.com/thinkparq/beegfs-go/common/scheduler" "github.com/thinkparq/beegfs-go/rst/sync/internal/beeremote" pbr "github.com/thinkparq/protobuf/go/beeremote" "github.com/thinkparq/protobuf/go/flex" "go.opentelemetry.io/otel/metric" "go.uber.org/zap" + "golang.org/x/sync/errgroup" "golang.org/x/sys/unix" "google.golang.org/protobuf/proto" ) @@ -128,10 +132,11 @@ type worker struct { workQueue <-chan workAssignment completedWork chan<- workIdentifier remoteStorageTargets *rst.ClientStore - workJournal *kvstore.MapStore[workEntry] + workJournal *kvstore.MapStore[*workEntry] jobStore *kvstore.MapStore[map[string]string] beeRemoteClient *beeremote.Client - rescheduleWork func(submissionId string, ExecuteAfter time.Time) + rescheduleWork scheduler.AddRescheduleWorkTokenFn + workerSaturation []func() float64 metrics managerMetrics } @@ -319,10 +324,22 @@ func (w *worker) process(work workAssignment) { return } - // Update the entry in BadgerDB so other goroutines can get read only access to the result. + // Update the entry in BadgerDB so other goroutines can get read only access to the result, then + // make a best-effort, non-blocking attempt to notify BeeRemote that the work request is running. + if state == flex.Work_SCHEDULED { + // Rescheduled work status messages will carry information aggregative status information. + // So, just set the running state information when the status state is scheduled. + status.SetMessage("attempting to carry out the work request") + } status.SetState(flex.Work_RUNNING) - status.SetMessage("attempting to carry out the work request") - commitJournalEntry(kvstore.WithUpdateOnly(true)) + + if err := commitJournalEntry(kvstore.WithUpdateOnly(true)); err != nil { + log.Warn("error updating journal work entry to running", zap.Error(err)) + } + if _, err := w.beeRemoteClient.UpdateWorkRequest(work.ctx, result.Work); err != nil { + log.Warn("unable to update remote job status to running; continuing work request without retrying", zap.Error(err)) + } + if request.HasBuilder() { cleanupEntries = w.processBuilder(work, client, entry) } else { @@ -330,7 +347,7 @@ func (w *worker) process(work workAssignment) { } } -func (w *worker) processWork(work workAssignment, client rst.Provider, entry workEntry, commitWorkPart func(), log *zap.Logger) (cleanupEntries bool) { +func (w *worker) processWork(work workAssignment, client rst.Provider, entry *workEntry, commitWorkPart func(), log *zap.Logger) (cleanupEntries bool) { request := entry.WorkRequest result := entry.WorkResult status := result.GetStatus() @@ -403,77 +420,84 @@ func (w *worker) processWork(work workAssignment, client rst.Provider, entry wor return } -func (w *worker) processBuilder(work workAssignment, client rst.Provider, entry workEntry) (cleanupEntries bool) { - request := entry.WorkRequest - result := entry.WorkResult - status := result.GetStatus() +// builderJobSubmissionWorkerMultiplier scales GOMAXPROCS to size the pool of goroutines that drain +// jobSubmissionCh concurrently. Submission is dominated by the SubmitJobRequest RPC round trip +// (network/BeeRemote-side work, not CPU), so a single consumer goroutine becomes a serialization +// bottleneck long before the job builder itself runs out of work to produce, especially for bulk +// operations that can ready thousands of requests at once. BeeRemote locks per-path (not globally) +// when handling SubmitJobRequest, so concurrent submissions for different paths shouldn't contend. +const builderJobSubmissionWorkerMultiplier = 4 - var reschedule bool - var err error - jobSubmissionChan := make(chan *pbr.JobRequest, 2048) - go func() { - defer close(jobSubmissionChan) - reschedule, err = client.ExecuteJobBuilderRequest(work.ctx, request.WorkRequest, jobSubmissionChan) - }() +func (w *worker) processBuilder(work workAssignment, client rst.Provider, entry *workEntry) (cleanupEntries bool) { + workRequest := entry.WorkRequest.WorkRequest + workResult := entry.WorkResult + builder := workRequest.GetBuilder() - total := 0 - totalErrors := 0 -processJobs: for { - select { - case <-work.ctx.Done(): - status.SetState(flex.Work_CANCELLED) - status.SetMessage("work context was cancelled before job requests could be created") - if w.sendWorkResult(work, result.Work) { - cleanupEntries = true - } - for range jobSubmissionChan { - } + jobSubmissionCh := make(chan *pbr.JobRequest, 2048) + g, gCtx := errgroup.WithContext(work.ctx) + + var schedulingResult *rst.SchedulingResult + g.Go(func() error { + defer close(jobSubmissionCh) + schedulingResult = client.ExecuteJobBuilderRequest(gCtx, workRequest, jobSubmissionCh, w.workerSaturation) + return nil + }) + + var builderMu sync.Mutex + + submissionWorkers := max(1, runtime.GOMAXPROCS(0)*builderJobSubmissionWorkerMultiplier) + for range submissionWorkers { + g.Go(func() error { + // Always drain jobSubmissionCh until it is closed, even after cancellation, so the + // producer side (ExecuteJobBuilderRequest) never blocks trying to send. + for jobRequest := range jobSubmissionCh { + if gCtx.Err() != nil { + continue + } + w.sendBuilderJobRequest(gCtx, &builderMu, builder, jobRequest) + } + return nil + }) + } + + if err := g.Wait(); err != nil { + w.updateBuilderJob(work, entry, &rst.SchedulingResult{Err: err}) + cleanupEntries = true return - case jobRequest, ok := <-jobSubmissionChan: - if !ok { - break processJobs - } + } - if err := w.beeRemoteClient.SubmitJobRequest(work.ctx, jobRequest); err != nil { - totalErrors += 1 - } - total++ + if schedulingResult == nil { + schedulingResult = &rst.SchedulingResult{Err: rst.MarkBuilderFailed(fmt.Errorf("job builder returned unexpected scheduling result"))} } - } - if err != nil { - status.SetState(flex.Work_CANCELLED) - status.SetMessage("job builder failed to complete: " + err.Error()) - } else if reschedule { - status.SetState(flex.Work_RESCHEDULED) - message := "waiting for builder job to continue" - if totalErrors > 0 { - message = fmt.Sprintf("%s: %d job request(s) failed! See `beegfs remote status/job list` for details", message, totalErrors) + bulkOperations := builder.GetBulkOperations() + if bulkOperations == nil { + bulkOperations = []*flex.BulkOperation{} + } + workResult.Work.JobBuilderInfo = &flex.Work_JobBuilderInfo{BulkOperations: bulkOperations} + + // While the queue has spare capacity, keep building/submitting immediately instead of + // paying the cost of persisting reschedule state and waiting for the manager to poll us + // back in. The builder carries its own resume cursor, so it's safe to just loop. Bounded to + // workDelayMinimum (not a true tight spin) so this doesn't hammer the RST backend -- e.g. a + // bulk-retrieve builder whose session/batch isn't ready yet would otherwise re-invoke + // execute() as fast as the CPU allows, ignoring the backend's own reschedule delay -- and + // select on work.ctx.Done() so shutdown isn't blocked waiting on this loop to notice + // cancellation. + if schedulingResult.Reschedule && len(w.workerSaturation) > 0 { + if w.workerSaturation[0]() < 100 { + select { + case <-time.After(workDelayMinimum): + continue + case <-work.ctx.Done(): + } + } } - status.SetMessage(message) - entry.ExecuteAfter = time.Now() - w.sendWorkResult(work, result.Work) - w.rescheduleWork(work.submissionID, entry.ExecuteAfter) - w.metrics.workRequests.Add(context.Background(), 1, - metric.WithAttributes( - attrState.String("rescheduled"), - attrPriority.Int(normalizedPriority(request.GetPriority())), - ), - ) - return - } else if totalErrors > 0 { - status.SetState(flex.Work_CANCELLED) - status.SetMessage(fmt.Sprintf("%d job request(s) failed! See `beegfs remote status/job list` for details", totalErrors)) - } else { - status.SetState(flex.Work_COMPLETED) - status.SetMessage("all jobs were submitted") - } - if w.sendWorkResult(work, result.Work) { - cleanupEntries = true + cleanupEntries = w.updateBuilderJob(work, entry, schedulingResult) + return } - return } // Returns true if the work result was sent, or for some reason cannot be sent but the overall state @@ -506,7 +530,176 @@ func (w *worker) sendWorkResult(work workAssignment, workResult *flex.Work) bool case <-work.ctx.Done(): return false } + } + } +} + +// sendBuilderJobRequest submits request to BeeRemote, retrying indefinitely while it's unavailable. +// builder's counters are mutated under mu since this is called concurrently by multiple submission +// workers sharing the same builder job. +func (w *worker) sendBuilderJobRequest(ctx context.Context, mu *sync.Mutex, builder *flex.BuilderJob, request *pbr.JobRequest) { + const maxSendBuilderJobDelay = 60 * time.Second + delay := 1 * time.Second + + for { + if err := w.beeRemoteClient.SubmitJobRequest(ctx, request); err != nil { + if errors.Is(err, beeremote.ErrUnavailable) { + // Retry with an exponential backoff until remote is available again. + select { + case <-time.After(delay): + delay *= 2 + if delay > maxSendBuilderJobDelay { + delay = maxSendBuilderJobDelay + } + case <-ctx.Done(): + return + } + continue + } + mu.Lock() + if errors.Is(err, rst.ErrJobAlreadyComplete) { + builder.JobsAlreadyComplete++ + } else if errors.Is(err, rst.ErrJobAlreadyOffloaded) { + builder.JobsAlreadyOffloaded++ + } else if errors.Is(err, rst.ErrJobAlreadyExists) { + builder.JobsAlreadyExist++ + } else if errors.Is(err, rst.ErrJobNotAllowed) { + builder.JobsNotAllowed++ + } else { + builder.Errors++ + } + mu.Unlock() + return } + + mu.Lock() + builder.Submitted++ + mu.Unlock() } } + +// updateBuilderJob uses result to update the builder job's work status and state. +func (w *worker) updateBuilderJob(work workAssignment, entry *workEntry, result *rst.SchedulingResult) (cleanupEntries bool) { + request := entry.WorkRequest + workRequest := request.WorkRequest + workResult := entry.WorkResult + builder := workRequest.GetBuilder() + status := workResult.GetStatus() + + builderErr := getBuilderResults(builder) + defer func() { + status.SetMessage(appendMessage(status.Message, builderErr.Error())) + if w.sendWorkResult(work, workResult.Work) && !result.Reschedule { + cleanupEntries = true + } + + fmt.Println(status.Message) + + }() + + if result.Err != nil { + // Builder-level termination is driven by the classification of result.Err and builderErr. + // Individual request errors should already have been reported on the submitted requests via + // GenerationStatus and accounted for in the builder counters rather than forcing builder + // termination here. + message := result.Err.Error() + if errors.Is(builderErr, rst.ErrBuilderFailed) { + status.SetState(flex.Work_FAILED) + status.SetMessage("job builder failed to complete: " + message) + } else if errors.Is(builderErr, rst.ErrBuilderCancelled) { + status.SetState(flex.Work_CANCELLED) + status.SetMessage("job builder failed to complete: " + message) + } else { + status.SetState(flex.Work_FAILED) + status.SetMessage("job builder returned unclassified error: " + message) + } + } else if result.Reschedule { + status.SetState(flex.Work_RESCHEDULED) + status.SetMessage("waiting for builder job to continue") + entry.ExecuteAfter = time.Now().Add(result.Delay) + w.rescheduleWork(work.submissionID, entry.ExecuteAfter) + w.metrics.workRequests.Add(context.Background(), 1, + metric.WithAttributes( + attrState.String("rescheduled"), + attrPriority.Int(normalizedPriority(request.GetPriority())), + ), + ) + } else if errors.Is(builderErr, rst.ErrBuilderFailed) { + status.SetState(flex.Work_FAILED) + status.SetMessage("completed with errors") + } else if errors.Is(builderErr, rst.ErrBuilderCancelled) { + status.SetState(flex.Work_CANCELLED) + status.SetMessage("completed with errors") + } else { + status.SetState(flex.Work_COMPLETED) + status.SetMessage("completed successfully") + } + + return +} + +// getBuilderResults generates a status message based on the builder submission counters. +func getBuilderResults(builder *flex.BuilderJob) (err error) { + cfg := builder.GetCfg() + jobsSubmitted := builder.GetSubmitted() + jobsErrors := builder.GetErrors() + jobsNotAllowed := builder.GetJobsNotAllowed() + jobsAlreadyComplete := builder.GetJobsAlreadyComplete() + jobsAlreadyOffloaded := builder.GetJobsAlreadyOffloaded() + jobsAlreadyExist := builder.GetJobsAlreadyExist() + + var parts []string + markCancelled := false + markedFailed := false + + for _, bulkOperation := range builder.BulkOperations { + if bulkOperation.Failed { + markedFailed = true + } + } + + jobsProcessed := jobsSubmitted + jobsErrors + jobsNotAllowed + jobsAlreadyComplete + jobsAlreadyOffloaded + jobsAlreadyExist + failures := jobsErrors + jobsNotAllowed + if jobsProcessed == 0 { + if cfg.Download { + if rst.WalkLocalPathInsteadOfRemote(cfg) { + parts = append(parts, fmt.Sprintf("walked local path since --%s was not provided; No matches found in path: %s", rst.RemotePathFlag, cfg.Path)) + } else { + parts = append(parts, fmt.Sprintf("no matches found in remote path: %s", cfg.RemotePath)) + } + } else { + parts = append(parts, fmt.Sprintf("no matches found in local path: %s", cfg.Path)) + } + markCancelled = true + } else if failures > 0 { + markCancelled = true + } + + if jobsSubmitted > 0 { + parts = append(parts, fmt.Sprintf("%d job request(s) submitted", jobsSubmitted)) + } + if jobsAlreadyComplete > 0 { + parts = append(parts, fmt.Sprintf("%d already complete", jobsAlreadyComplete)) + } + if jobsAlreadyOffloaded > 0 { + parts = append(parts, fmt.Sprintf("%d already offloaded", jobsAlreadyOffloaded)) + } + if jobsAlreadyExist > 0 { + parts = append(parts, fmt.Sprintf("%d already exist", jobsAlreadyExist)) + } + if jobsNotAllowed > 0 { + parts = append(parts, fmt.Sprintf("%d not allowed", jobsNotAllowed)) + } + if jobsErrors > 0 { + parts = append(parts, fmt.Sprintf("%d submitted with errors", jobsErrors)) + } + + err = fmt.Errorf("%s", strings.Join(parts, "; ")) + if markedFailed { + err = rst.MarkBuilderFailed(err) + } else if markCancelled { + err = rst.MarkBuilderCancelled(err) + } + return +}