Swartzn/feat/add xtreemstore rst

common/filesystem/fs.go

@@ -46,11 +46,23 @@ type Provider interface {
 	// entire file, but should use optimized methods like fallocate() or truncate(). This also means
 	// it is not safe to rely on CreatePreallocatedFile to securely wipe an existing file.
 	CreatePreallocatedFile(name string, size int64, overwrite bool) error
+	// Creates or resizes a file at the specified path, returning an error if the file already
+	// exists unless overwrite is true. If overwrite is true and the file already exists, the file
+	// will be resized to the specified size without first zeroing or truncating it to zero. If the
+	// file is extended, the new region may be sparse and may not reserve physical disk space. If
+	// the file is reduced, data beyond the specified size is discarded and cannot be restored by
+	// later extending the file. This is useful for setting the logical size of a file before
+	// subsequent operations, such as a multipart download, but callers must still handle write-time
+	// space errors. Note that expanding a file with overwrite==true will not cause and error to be
+	// returned.
+	CreateOrResizeFile(name string, size int64, overwrite bool) error
 	// CreateWriteClose creates the file specified by name and immediately writes the specified buf
 	// as the file contents then closes the file.
 	CreateWriteClose(name string, buf []byte, mode uint32, overwrite bool) error
 	// Removes the file specified by name.
 	Remove(name string) error
+	// Removes the specified path and any children it contains.
+	RemoveAll(name string) error
 	// Opens the file specified by name and returns it as an io.ReadCloser. The caller must close the
 	// file when it is no longer required.
 	Open(name string) (io.ReadCloser, error)
@@ -82,6 +94,8 @@ type Provider interface {
 	CopyOwnerAndMode(fromStat fs.FileInfo, dstPath string) error
 	// CopyTimestamps sets the atime/mtime in fromStat on dstPath.
 	CopyTimestamps(fromStat fs.FileInfo, dstPath string) error
+	// Chtimes changes the access and modification times of the named file.
+	Chtimes(path string, atime time.Time, mtime time.Time) error
 	// Atomically renames srcPath to dstPath overwriting the dstPath with srcPath's contents.
 	OverwriteFile(srcPath, dstPath string) error
 	Readlink(path string) (string, error)
@@ -206,6 +220,26 @@ func (fs BeeGFS) CreatePreallocatedFile(path string, size int64, overwrite bool)
 	return file.Close()
 }
 
+func (fs BeeGFS) CreateOrResizeFile(path string, size int64, overwrite bool) error {
+	absPath := filepath.Join(fs.MountPoint, path)
+	flags := os.O_RDWR | os.O_CREATE
+	if !overwrite {
+		flags |= os.O_EXCL
+	}
+
+	file, err := os.OpenFile(absPath, flags, 0666)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	if err := file.Truncate(size); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (fs BeeGFS) CreateWriteClose(path string, buf []byte, mode uint32, overwrite bool) error {
 	var file *os.File
 	var err error
@@ -231,6 +265,10 @@ func (fs BeeGFS) Remove(path string) error {
 	return os.Remove(filepath.Join(fs.MountPoint, path))
 }
 
+func (fs BeeGFS) RemoveAll(path string) error {
+	return os.RemoveAll(filepath.Join(fs.MountPoint, path))
+}
+
 func (fs BeeGFS) Open(path string) (io.ReadCloser, error) {
 	return os.Open(filepath.Join(fs.MountPoint, path))
 }
@@ -434,6 +472,11 @@ func (fs BeeGFS) CopyTimestamps(fromStat fs.FileInfo, dstPath string) error {
 	return nil
 }
 
+func (fs BeeGFS) Chtimes(path string, atime time.Time, mtime time.Time) error {
+	absPath := filepath.Join(fs.MountPoint, path)
+	return os.Chtimes(absPath, atime, mtime)
+}
+
 func (fs BeeGFS) OverwriteFile(srcPath, dstPath string) error {
 	srcPath = filepath.Join(fs.MountPoint, srcPath)
 	dstPath = filepath.Join(fs.MountPoint, dstPath)

common/filesystem/fs_test.go

@@ -2,6 +2,7 @@ package filesystem
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -88,3 +89,20 @@ func TestBeeGFSWriteAndReadFileParts(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, expectedFileLen, readBytes)
 }
+
+func TestBeeGFSRemoveAll(t *testing.T) {
+	testDir, cleanup, err := tempPathForTesting(baseTestDir)
+	require.NoError(t, err)
+	defer cleanup(t)
+
+	mount := BeeGFS{MountPoint: testDir}
+	targetDir := filepath.Join(testDir, testFileName, "nested")
+	targetFile := filepath.Join(targetDir, "data")
+	require.NoError(t, os.MkdirAll(targetDir, 0755))
+	require.NoError(t, os.WriteFile(targetFile, []byte("test"), 0644))
+
+	require.NoError(t, mount.RemoveAll(testFileName))
+
+	_, err = os.Stat(filepath.Join(testDir, testFileName))
+	require.ErrorIs(t, err, os.ErrNotExist)
+}

common/filesystem/mock.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/spf13/afero"
 )
@@ -47,6 +48,16 @@ func (fs MockFS) CreatePreallocatedFile(path string, size int64, overwrite bool)
 	return nil
 }
 
+func (fs MockFS) CreateOrResizeFile(path string, size int64, overwrite bool) error {
+	file, err := fs.Fs.Create(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	file.Truncate(size)
+	return nil
+}
+
 func (fs MockFS) CreateWriteClose(path string, buf []byte, mode uint32, overwrite bool) error {
 	file, err := fs.Fs.Create(path)
 	if err != nil {
@@ -65,6 +76,10 @@ func (fs MockFS) Remove(path string) error {
 	return fs.Fs.Remove(path)
 }
 
+func (fs MockFS) RemoveAll(path string) error {
+	return fs.Fs.RemoveAll(path)
+}
+
 func (fs MockFS) Open(path string) (io.ReadCloser, error) {
 	return fs.Fs.Open(path)
 }
@@ -117,6 +132,10 @@ func (fs MockFS) CopyTimestamps(fromStat fs.FileInfo, dstPath string) error {
 	return fmt.Errorf("not implemented")
 }
 
+func (fs MockFS) Chtimes(path string, atime time.Time, mtime time.Time) error {
+	return fs.Fs.Chtimes(path, atime, mtime)
+}
+
 func (fs MockFS) OverwriteFile(srcPath, dstPath string) error {
 	return fmt.Errorf("not implemented")
 }

common/filesystem/unmounted.go

@@ -5,6 +5,7 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"time"
 )
 
 // UnmountedFS can be used whenever a mounted file system may not actually be required. This allows
@@ -40,6 +41,10 @@ func (fs UnmountedFS) CreatePreallocatedFile(path string, size int64, overwrite
 	return ErrUnmounted
 }
 
+func (fs UnmountedFS) CreateOrResizeFile(path string, size int64, overwrite bool) error {
+	return ErrUnmounted
+}
+
 func (fs UnmountedFS) CreateWriteClose(path string, buf []byte, mode uint32, overwrite bool) error {
 	return ErrUnmounted
 }
@@ -48,6 +53,10 @@ func (fs UnmountedFS) Remove(path string) error {
 	return ErrUnmounted
 }
 
+func (fs UnmountedFS) RemoveAll(path string) error {
+	return ErrUnmounted
+}
+
 func (fs UnmountedFS) Open(path string) (io.ReadCloser, error) {
 	return nil, ErrUnmounted
 }
@@ -85,6 +94,10 @@ func (fs UnmountedFS) CopyTimestamps(fromStat fs.FileInfo, dstPath string) error
 	return ErrUnmounted
 }
 
+func (fs UnmountedFS) Chtimes(path string, atime time.Time, mtime time.Time) error {
+	return ErrUnmounted
+}
+
 func (fs UnmountedFS) OverwriteFile(srcPath, dstPath string) error {
 	return ErrUnmounted
 }

common/filesystem/walk.go

@@ -108,24 +108,34 @@ type StreamPathResult struct {
 	Err         error
 }
 
-// StreamPathsLexicographically returns a *StreamPathResult channel that returns the pattern's paths in a
-// lexicographically increasing order. If startAfter != "" then only files lexically greater than
-// will be considered. maxPaths limits the number of paths returned and can be set to -1 for all
-// paths. chanSize is the buffer size for the returned *StreamPathResult channel.
-func StreamPathsLexicographically(ctx context.Context, mountPoint Provider, pattern string, startAfter string, maxPaths int, chanSize int, filter FileInfoFilter) (<-chan *StreamPathResult, error) {
-	return streamPathsLexicographically(ctx, mountPoint, pattern, startAfter, maxPaths, chanSize, filter, false)
+type PathStreamFunc func(
+	ctx context.Context,
+	mountPoint Provider,
+	pattern string,
+	startAfter string,
+	maxFiles int,
+	chanSize int,
+	filter FileInfoFilter,
+) (<-chan *StreamPathResult, error)
+
+// StreamPathsLexicographically returns a *StreamPathResult channel that returns the pattern's paths
+// in a lexicographically increasing order. If startAfter != "" then only files lexically greater
+// than will be considered. maxFiles limits the number of files returned and can be set to -1 for
+// all files. chanSize is the buffer size for the returned *StreamPathResult channel.
+func StreamPathsLexicographically(ctx context.Context, mountPoint Provider, pattern string, startAfter string, maxFiles int, chanSize int, filter FileInfoFilter) (<-chan *StreamPathResult, error) {
+	return streamPathsLexicographically(ctx, mountPoint, pattern, startAfter, maxFiles, chanSize, filter, false)
 }
 
-// StreamPathsLexicographicallyWithDirs behaves like StreamPathsLexicographically but also emits
-// directories that match the filter (if provided). Directories are still traversed even if they
-// don't match the filter.
-func StreamPathsLexicographicallyWithDirs(ctx context.Context, mountPoint Provider, pattern string, startAfter string, maxPaths int, chanSize int, filter FileInfoFilter) (<-chan *StreamPathResult, error) {
-	return streamPathsLexicographically(ctx, mountPoint, pattern, startAfter, maxPaths, chanSize, filter, true)
+// StreamPathsLexicographicallyWithDirs behaves like StreamPathsLexicographically but also emit
+// directories. Emitted directories are not counted against maxFiles. Also, the ResumeToken returned
+// will only be file paths.
+func StreamPathsLexicographicallyWithDirs(ctx context.Context, mountPoint Provider, pattern string, startAfter string, maxFiles int, chanSize int, filter FileInfoFilter) (<-chan *StreamPathResult, error) {
+	return streamPathsLexicographically(ctx, mountPoint, pattern, startAfter, maxFiles, chanSize, filter, true)
 }
 
-func streamPathsLexicographically(ctx context.Context, mountPoint Provider, pattern string, startAfter string, maxPaths int, chanSize int, filter FileInfoFilter, includeDirs bool) (<-chan *StreamPathResult, error) {
-	if maxPaths != -1 && maxPaths <= 0 {
-		return nil, fmt.Errorf("maxPaths must be greater than zero or -1")
+func streamPathsLexicographically(ctx context.Context, mountPoint Provider, pattern string, startAfter string, maxFiles int, chanSize int, filter FileInfoFilter, includeDirs bool) (<-chan *StreamPathResult, error) {
+	if maxFiles != -1 && maxFiles <= 0 {
+		return nil, fmt.Errorf("maxFiles must be greater than zero or -1")
 	}
 
 	preparePath := func(path string) string {
@@ -203,20 +213,31 @@ func streamPathsLexicographically(ctx context.Context, mountPoint Provider, patt
 				return true
 			}
 		}
-		emitPath := func(path string, resumeToken string) bool {
-			if maxPaths == 0 {
-				send(&StreamPathResult{ResumeToken: resumeToken})
+
+		// Only files count against maxFiles or anchor the resume token: a directory never produces
+		// a job request on its own (see jobRequestBuilder.Process), and reapplying its RST config on
+		// a later pass is a no-op, so it's always sent immediately regardless of the remaining
+		// budget.
+		var lastSent string
+		emitFile := func(path string) bool {
+			if maxFiles == 0 {
+				send(&StreamPathResult{ResumeToken: lastSent})
 				return false
 			}
 			if !send(&StreamPathResult{Path: path}) {
 				return false
 			}
-			if maxPaths > 0 {
-				maxPaths--
+			lastSent = path
+			if maxFiles > 0 {
+				maxFiles--
 			}
 			return true
 		}
 
+		emitDir := func(path string) bool {
+			return send(&StreamPathResult{Path: path})
+		}
+
 		var walkDir func(string) bool
 		walkDir = func(directory string) bool {
 			if err := ctx.Err(); err != nil {
@@ -233,29 +254,25 @@ func streamPathsLexicographically(ctx context.Context, mountPoint Provider, patt
 				return false
 			}
 
-			lastPath := directory
 			for _, entry := range entries {
 				path := filepath.Join(directory, entry.Name())
 				inMountPath := "/" + path
 
 				if entry.IsDir() {
 					if includeDirs {
-						emitDir := false
+						shouldEmitDir := false
 						if !isGlob {
-							emitDir = path > startAfter
+							shouldEmitDir = path > startAfter
 						} else if match := doublestar.MatchUnvalidated(pattern, path); match {
-							emitDir = path > startAfter
+							shouldEmitDir = path > startAfter
 						}
 
-						if emitDir {
+						if shouldEmitDir {
 							if keep, err := ApplyFilter(inMountPath, filter, mountPoint); err != nil {
 								send(&StreamPathResult{Err: fmt.Errorf("unable to filter files: %w", err)})
 								return false
-							} else if keep {
-								if !emitPath(inMountPath, lastPath) {
-									return false
-								}
-								lastPath = path
+							} else if keep && !emitDir(inMountPath) {
+								return false
 							}
 						}
 					}
@@ -281,10 +298,9 @@ func streamPathsLexicographically(ctx context.Context, mountPoint Provider, patt
 					continue
 				}
 
-				if !emitPath(inMountPath, lastPath) {
+				if !emitFile(inMountPath) {
 					return false
 				}
-				lastPath = path
 			}
 
 			return true
@@ -298,7 +314,7 @@ func streamPathsLexicographically(ctx context.Context, mountPoint Provider, patt
 					send(&StreamPathResult{Err: fmt.Errorf("unable to filter files: %w", err)})
 					return
 				} else if keep {
-					if !emitPath(inMountPath, root) {
+					if !emitDir(inMountPath) {
 						return
 					}
 				}