diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index bce442931be..f06ef079fd7 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -130,6 +130,29 @@ jobs: - name: Run dataset tests run: uv run pytest packages/syft-datasets/tests/ -v + rds-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install the project + run: uv sync --all-extras + + - name: Install just + run: | + curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to /usr/local/bin + + - name: Run RDS tests + run: just test-unit-rds + enclave-tests: runs-on: ubuntu-latest steps: diff --git a/Justfile b/Justfile index 74fcb344912..b16fedf3bbd 100644 --- a/Justfile +++ b/Justfile @@ -38,6 +38,10 @@ test-unit-migration: #!/bin/bash uv run pytest -n auto ./packages/syft-migration/tests +test-unit-rds: + #!/bin/bash + uv run pytest -n auto ./packages/syft-rds/tests + test-unit-enclave: #!/bin/bash @@ -49,7 +53,7 @@ test-unit-enclave-model-api: test-unit-fast: #!/bin/bash - uv run pytest ./tests/unit --ignore=tests/unit/test_job_auto_approval.py --ignore=tests/unit/test_version_mismatch_flow.py --ignore=tests/unit/syft_bg/test_email_auto_approve_flow.py --ignore=tests/unit/syft_bg/test_email_approval_flow.py --ignore=tests/unit/test_sync_file_lock.py -k "not (test_jobs or job_flow_with_dataset)" + uv run pytest ./tests/unit --ignore=tests/unit/syft_bg/test_email_auto_approve_flow.py --ignore=tests/unit/syft_bg/test_email_approval_flow.py --ignore=tests/unit/test_sync_file_lock.py -k "not (test_jobs or job_flow_with_dataset)" test-integration-mock-mode: diff --git a/packages/syft-bg/pyproject.toml b/packages/syft-bg/pyproject.toml index 8b6a98fde30..70cc208ecd7 100644 --- a/packages/syft-bg/pyproject.toml +++ b/packages/syft-bg/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "pydantic>=2.0.0", "google-cloud-pubsub>=2.18.0", "syft-job==0.1.39", + "syft-rds", ] [project.scripts] @@ -24,6 +25,7 @@ syft-bg = "syft_bg.cli:main" [tool.uv.sources] "syft-job" = { workspace = true } +"syft-rds" = { workspace = true } [build-system] requires = ["hatchling"] diff --git a/packages/syft-bg/src/syft_bg/approve/handlers/job.py b/packages/syft-bg/src/syft_bg/approve/handlers/job.py index fb661ad4bf1..2819cc36ff7 100644 --- a/packages/syft-bg/src/syft_bg/approve/handlers/job.py +++ b/packages/syft-bg/src/syft_bg/approve/handlers/job.py @@ -18,7 +18,7 @@ ) if TYPE_CHECKING: - from syft_client.sync.syftbox_manager import SyftboxManager + from syft_rds import SyftRDSClient class StateManager(Protocol): @@ -33,7 +33,7 @@ class JobApprovalHandler: def __init__( self, - client: SyftboxManager, + client: SyftRDSClient, config_path: Optional[Path] = None, state: Optional[StateManager] = None, on_approve: Optional[Callable[[JobInfo], None]] = None, diff --git a/packages/syft-bg/src/syft_bg/approve/handlers/peer.py b/packages/syft-bg/src/syft_bg/approve/handlers/peer.py index d9fda86e026..5fd9dd45165 100644 --- a/packages/syft-bg/src/syft_bg/approve/handlers/peer.py +++ b/packages/syft-bg/src/syft_bg/approve/handlers/peer.py @@ -7,7 +7,7 @@ from syft_bg.approve.config import PeerApprovalConfig if TYPE_CHECKING: - from syft_client.sync.syftbox_manager import SyftboxManager + from syft_rds import SyftRDSClient class StateManager(Protocol): @@ -22,7 +22,7 @@ class PeerApprovalHandler: def __init__( self, - client: SyftboxManager, + client: SyftRDSClient, config: PeerApprovalConfig, state: Optional[StateManager] = None, on_approve: Optional[Callable[[str], None]] = None, diff --git a/packages/syft-bg/src/syft_bg/approve/monitors/job.py b/packages/syft-bg/src/syft_bg/approve/monitors/job.py index d0fbd8a1ecf..1a42a1476af 100644 --- a/packages/syft-bg/src/syft_bg/approve/monitors/job.py +++ b/packages/syft-bg/src/syft_bg/approve/monitors/job.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: from syft_job.job import JobInfo - from syft_client.sync.syftbox_manager import SyftboxManager + from syft_rds import SyftRDSClient class JobMonitor(Monitor): @@ -18,7 +18,7 @@ class JobMonitor(Monitor): def __init__( self, - client: SyftboxManager, + client: SyftRDSClient, config_path: Optional[Path] = None, state: Optional[StateManager] = None, on_approve: Optional[Callable[[JobInfo], None]] = None, diff --git a/packages/syft-bg/src/syft_bg/approve/monitors/peer.py b/packages/syft-bg/src/syft_bg/approve/monitors/peer.py index 752936268dc..cfd591c91ec 100644 --- a/packages/syft-bg/src/syft_bg/approve/monitors/peer.py +++ b/packages/syft-bg/src/syft_bg/approve/monitors/peer.py @@ -9,7 +9,7 @@ if TYPE_CHECKING: from syft_bg.approve.config import PeerApprovalConfig - from syft_client.sync.syftbox_manager import SyftboxManager + from syft_rds import SyftRDSClient class PeerMonitor(Monitor): @@ -17,7 +17,7 @@ class PeerMonitor(Monitor): def __init__( self, - client: SyftboxManager, + client: SyftRDSClient, config: PeerApprovalConfig, state: Optional[StateManager] = None, on_approve: Optional[Callable[[str], None]] = None, diff --git a/packages/syft-bg/src/syft_bg/approve/orchestrator.py b/packages/syft-bg/src/syft_bg/approve/orchestrator.py index 20db094644d..cd1ece02802 100644 --- a/packages/syft-bg/src/syft_bg/approve/orchestrator.py +++ b/packages/syft-bg/src/syft_bg/approve/orchestrator.py @@ -12,7 +12,7 @@ from syft_bg.common.state import JsonStateManager if TYPE_CHECKING: - from syft_client.sync.syftbox_manager import SyftboxManager + from syft_rds import SyftRDSClient class ApprovalOrchestrator(BaseOrchestrator): @@ -20,7 +20,7 @@ class ApprovalOrchestrator(BaseOrchestrator): def __init__( self, - client: SyftboxManager, + client: SyftRDSClient, config: AutoApproveConfig, config_path: Optional[Path] = None, ): @@ -40,10 +40,10 @@ def setup(self) -> None: @classmethod def from_client( cls, - client: SyftboxManager, + client: SyftRDSClient, interval: int = 5, ) -> ApprovalOrchestrator: - """Create orchestrator from a SyftboxManager client.""" + """Create orchestrator from a SyftRDSClient.""" if not client.has_do_role: raise ValueError( "ApprovalOrchestrator should only run on Data Owner (DO) side." @@ -67,31 +67,35 @@ def from_config( if not config.syftbox_root: raise ValueError("Config missing 'syftbox_root' field") - # Wait for sync to seed the cache before building SyftboxManager — - # SyftboxManager.from_config triggers _load_file_hashes_from_disk, - # which races sync's identical replay if both run cold-start. + # Wait for sync to seed the cache before building the RDS client — + # its nested SyftboxManager.from_config triggers + # _load_file_hashes_from_disk, which races sync's identical replay if + # both run cold-start. cls._wait_for_sync_ready(label="Approve") - from syft_client.sync.environments.environment import Environment - from syft_client.sync.syftbox_manager import SyftboxManager - from syft_client.sync.utils.syftbox_utils import check_env + from syft_rds import ( + Environment, + SyftRDSClient, + SyftRDSClientConfig, + check_env, + ) - env = check_env() - if env == Environment.COLAB: - client = SyftboxManager.for_colab( + if check_env() == Environment.COLAB: + rds_config = SyftRDSClientConfig.for_colab( email=config.do_email, has_do_role=True, skip_peer_on_patch_version_diff=config.skip_peer_on_patch_version_diff, force_ignore_peer_version=config.force_ignore_peer_version, ) else: - client = SyftboxManager.for_jupyter( + rds_config = SyftRDSClientConfig.for_jupyter( email=config.do_email, has_do_role=True, token_path=config.drive_token_path, skip_peer_on_patch_version_diff=config.skip_peer_on_patch_version_diff, force_ignore_peer_version=config.force_ignore_peer_version, ) + client = SyftRDSClient.from_config(rds_config) return cls(client=client, config=config) diff --git a/packages/syft-bg/src/syft_bg/cli/init/flow.py b/packages/syft-bg/src/syft_bg/cli/init/flow.py index 26f65792c03..253c49d048c 100644 --- a/packages/syft-bg/src/syft_bg/cli/init/flow.py +++ b/packages/syft-bg/src/syft_bg/cli/init/flow.py @@ -100,7 +100,7 @@ def _resolve_common_settings(config: UserPassedConfig, result: SyftBgConfig) -> "Data Owner email address", default=default_email or None ) - from syft_client.sync.syftbox_manager import get_jupyter_default_syftbox_folder + from syft_rds import get_jupyter_default_syftbox_folder default_syftbox = result.syftbox_root or str( get_jupyter_default_syftbox_folder(result.do_email) diff --git a/packages/syft-bg/src/syft_bg/common/syft_bg_config.py b/packages/syft-bg/src/syft_bg/common/syft_bg_config.py index 6647bf67a91..a0c2e1e2aa4 100644 --- a/packages/syft-bg/src/syft_bg/common/syft_bg_config.py +++ b/packages/syft-bg/src/syft_bg/common/syft_bg_config.py @@ -38,9 +38,7 @@ def _set_default_syftbox_root(cls, data: dict) -> dict: if not isinstance(data, dict): return data if not data.get("syftbox_root") and data.get("do_email"): - from syft_client.sync.syftbox_manager import ( - get_jupyter_default_syftbox_folder, - ) + from syft_rds import get_jupyter_default_syftbox_folder data = dict(data) data["syftbox_root"] = str( diff --git a/packages/syft-bg/src/syft_bg/notify/handlers/job.py b/packages/syft-bg/src/syft_bg/notify/handlers/job.py index 16182327573..a7358766b2b 100644 --- a/packages/syft-bg/src/syft_bg/notify/handlers/job.py +++ b/packages/syft-bg/src/syft_bg/notify/handlers/job.py @@ -5,7 +5,7 @@ from syft_bg.common.state import JsonStateManager from syft_bg.notify.gmail.sender import GmailSender -from syft_client.sync.utils.path_filters import is_normal_syncable_path +from syft_rds import is_normal_syncable_path if TYPE_CHECKING: from syft_job.client import JobClient diff --git a/packages/syft-bg/src/syft_bg/sync/orchestrator.py b/packages/syft-bg/src/syft_bg/sync/orchestrator.py index b2258c4d1e5..e4f4195ae03 100644 --- a/packages/syft-bg/src/syft_bg/sync/orchestrator.py +++ b/packages/syft-bg/src/syft_bg/sync/orchestrator.py @@ -13,7 +13,7 @@ from syft_bg.sync.snapshot import PeerVersionInfo, SyncSnapshot if TYPE_CHECKING: - from syft_client.sync.syftbox_manager import SyftboxManager + from syft_rds import SyftRDSClient def sync_ready_path() -> Path: @@ -24,7 +24,7 @@ def sync_ready_path() -> Path: class SyncOrchestrator(BaseOrchestrator): def __init__( self, - client: SyftboxManager, + client: SyftRDSClient, state: JsonStateManager, config: SyncConfig, ): @@ -42,26 +42,29 @@ def from_config(cls, config: SyncConfig) -> SyncOrchestrator: if not config.syftbox_root: raise ValueError("SyncConfig missing 'syftbox_root'") - from syft_client.sync.environments.environment import Environment - from syft_client.sync.syftbox_manager import SyftboxManager - from syft_client.sync.utils.syftbox_utils import check_env + from syft_rds import ( + Environment, + SyftRDSClient, + SyftRDSClientConfig, + check_env, + ) - env = check_env() - if env == Environment.COLAB: - client = SyftboxManager.for_colab( + if check_env() == Environment.COLAB: + rds_config = SyftRDSClientConfig.for_colab( email=config.do_email, has_do_role=True, skip_peer_on_patch_version_diff=config.skip_peer_on_patch_version_diff, force_ignore_peer_version=config.force_ignore_peer_version, ) else: - client = SyftboxManager.for_jupyter( + rds_config = SyftRDSClientConfig.for_jupyter( email=config.do_email, has_do_role=True, token_path=config.drive_token_path, skip_peer_on_patch_version_diff=config.skip_peer_on_patch_version_diff, force_ignore_peer_version=config.force_ignore_peer_version, ) + client = SyftRDSClient.from_config(rds_config) state = JsonStateManager(state_file=config.sync_state_path) diff --git a/packages/syft-enclave/docs/terraform.md b/packages/syft-enclave/docs/terraform.md index 48e6b769930..070b1743e3e 100644 --- a/packages/syft-enclave/docs/terraform.md +++ b/packages/syft-enclave/docs/terraform.md @@ -53,6 +53,7 @@ Do **not** set `dev_mode` in tfvars — pass it on the command line (`-var=dev_m Hardened image — no SSH, TEE enforcement, encryption on, container restart policy `Never`. From syft-enclave dir, run + ```bash terraform -chdir=terraform init # once: download providers, set up local state terraform -chdir=terraform apply -var=dev_mode=false # provision APIs, SA, IAM, secret, and the VM diff --git a/packages/syft-enclave/pyproject.toml b/packages/syft-enclave/pyproject.toml index 55a93237862..087eab59ad1 100644 --- a/packages/syft-enclave/pyproject.toml +++ b/packages/syft-enclave/pyproject.toml @@ -8,6 +8,7 @@ requires-python = ">=3.10" dependencies = [ "syft-client", + "syft-rds", "pydantic-settings>=2.11.0", "requests>=2.32.0", "google-auth[pyjwt]>=2.22.0", @@ -19,6 +20,7 @@ build-backend = "hatchling.build" [tool.uv.sources] "syft-client" = { workspace = true } +"syft-rds" = { workspace = true } [tool.hatch.build.targets.wheel] packages = ["src/syft_enclaves"] diff --git a/packages/syft-enclave/src/syft_enclaves/client.py b/packages/syft-enclave/src/syft_enclaves/client.py index 6bdac8af9af..144fc47d617 100644 --- a/packages/syft-enclave/src/syft_enclaves/client.py +++ b/packages/syft-enclave/src/syft_enclaves/client.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import Optional -from syft_client.sync.syftbox_manager import SyftboxManager, SyftboxManagerConfig +from syft_rds import SyftRDSClient, SyftRDSClientConfig from syft_client.sync.version.peer_manager import CompatAction from syft_client.sync.peers.peer import Peer from syft_client.sync.peers.peer_list import PeerList @@ -39,31 +39,34 @@ class SyftEnclaveClient: def __init__( self, - manager: SyftboxManager, + rds: SyftRDSClient, data_owners: list[str] | None = None, ): - self._manager = manager + # The Remote Data Science product client. It OWNS the job + dataset + # surface (job_client / job_runner / dataset_manager) and composes the + # generic sync engine (a SyftboxManager) as ``rds.sync_engine``. + self._rds = rds # Data owners whose approval gates every job run on this enclave. # Fixed at launch/deploy time; stored in memory. self.data_owners = list(data_owners or []) @property def email(self) -> str: - return self._manager.email + return self._rds.email @property def syftbox_folder(self) -> Path: - return self._manager.syftbox_folder + return self._rds.syftbox_folder @property def peers(self) -> PeerList: - return self._manager.peers + return self._rds.peers def add_peer(self, peer_email: str, force: bool = False, verbose: bool = True): - self._manager.add_peer(peer_email, force=force, verbose=verbose) + self._rds.add_peer(peer_email, force=force, verbose=verbose) def load_peers(self): - self._manager.load_peers() + self._rds.load_peers() def approve_peer_request( self, @@ -71,12 +74,12 @@ def approve_peer_request( verbose: bool = True, peer_must_exist: bool = True, ): - self._manager.approve_peer_request( + self._rds.approve_peer_request( email_or_peer, verbose=verbose, peer_must_exist=peer_must_exist ) def reject_peer_request(self, email_or_peer: str | Peer): - self._manager.reject_peer_request(email_or_peer) + self._rds.reject_peer_request(email_or_peer) def attest_peer( self, @@ -103,10 +106,8 @@ def attest_peer( if expected_image_digest is not None: policy = AppraisalPolicy(expected_image_digest=expected_image_digest) - version_info = ( - self._manager.peer_manager.connection_router.read_peer_version_file( - peer_email - ) + version_info = self._rds.peer_manager.connection_router.read_peer_version_file( + peer_email ) if version_info is None: print( @@ -122,29 +123,29 @@ def attest_peer( return verify_attestation_token(version_info.attestation_token, policy=policy) def sync(self): - self._manager.sync() + self._rds.sync() def delete_syftbox( self, verbose: bool = True, broadcast_delete_events: bool = True ): """Delete all SyftBox state (Drive files + local caches/folder).""" - self._manager.delete_syftbox( + self._rds.delete_syftbox( verbose=verbose, broadcast_delete_events=broadcast_delete_events ) def create_dataset(self, *args, **kwargs): - return self._manager.create_dataset(*args, **kwargs) + return self._rds.create_dataset(*args, **kwargs) def share_private_dataset(self, tag: str, enclave_email: str): - self._manager.share_private_dataset(tag, enclave_email) + self._rds.share_private_dataset(tag, enclave_email) @property def datasets(self) -> SyftDatasetManager: - return self._manager.dataset_manager + return self._rds.dataset_manager @property def jobs(self) -> JobsList: - jobs_list = self._manager.job_client.jobs + jobs_list = self._rds.job_client.jobs wrapped = [ EnclaveJobInfo.from_job_info(j) if j.job_headers.get("job_type") == "enclave" @@ -172,14 +173,14 @@ def submit_python_job( otherwise surfaces much later as a stuck, never-distributed job). """ if not force_submission: - result = self._manager.peer_manager.get_peer_compatibility_status( + result = self._rds.peer_manager.get_peer_compatibility_status( enclave_email, action=CompatAction.SUBMIT, ignore_peer_version=ignore_peer_version, ) result.raise_on_skip(operation="submit job") result.maybe_warn() - job_dir = self._manager.job_client.submit_python_job( + job_dir = self._rds.job_client.submit_python_job( enclave_email, code_path, job_name, @@ -187,7 +188,7 @@ def submit_python_job( share_results_with_do=share_results_with_do, **kwargs, ) - self._manager.push_job_files(job_dir) + self._rds.sync_engine.push_job_files(job_dir) def run_jobs(self) -> None: """Run approved enclave jobs.""" @@ -201,7 +202,7 @@ def run_jobs(self) -> None: state.status = JobStatus.APPROVED state.save(job.job_review_path / "state.yaml") - self._manager.process_approved_jobs( + self._rds.process_approved_jobs( force_execution=True, share_outputs_with_submitter=True, share_logs_with_submitter=True, @@ -229,14 +230,14 @@ def distribute_results(self) -> None: results_shared_marker.write_text("shared") - self._manager.sync() + self._rds.sync() def _read_state_file(self, job: JobInfo) -> dict[Path, bytes]: """Read the job state.yaml as a {path_in_datasite: bytes} dict.""" state_file = job.job_review_path / "state.yaml" if not state_file.exists(): return {} - datasite_dir = self._manager.syftbox_folder / self._manager.email + datasite_dir = self._rds.syftbox_folder / self._rds.email state_rel = state_file.relative_to(datasite_dir) return {state_rel: state_file.read_bytes()} @@ -249,24 +250,23 @@ def _forward_results_to_recipients(self, job: JobInfo, recipients: list[str]): files_by_datasite_path.update(self._read_state_file(job)) if not files_by_datasite_path: return - events_message = ( - self._manager.datasite_owner_syncer.event_cache.create_events_for_files( - files_by_datasite_path - ) + syncer = self._rds.sync_engine.datasite_owner_syncer + events_message = syncer.event_cache.create_events_for_files( + files_by_datasite_path ) - self._manager.datasite_owner_syncer.queue_event_for_syftbox( + syncer.queue_event_for_syftbox( recipients=recipients, file_change_events_message=events_message, ) - self._manager.datasite_owner_syncer.process_syftbox_events_queue() + syncer.process_syftbox_events_queue() def approve_job(self, job: JobInfo) -> None: """Approve an enclave job and push the approval state file to the enclave.""" job.approve() file_name = enclave_approval_file_name(self.email) approval_file = job.job_review_path / file_name - relative_path = approval_file.relative_to(self._manager.syftbox_folder) - self._manager.datasite_watcher_syncer.on_file_change( + relative_path = approval_file.relative_to(self._rds.syftbox_folder) + self._rds.sync_engine.datasite_watcher_syncer.on_file_change( relative_path, process_now=True ) @@ -278,14 +278,14 @@ def receive_jobs(self): 3. Creates JobState with PartyApprovalStatus per DO 4. Sets permissions and marks as distributed """ - self._manager.job_client.scan_inbox() - job_manager = self._manager.job_client.manager - for ref in job_manager.iter_submission_refs(self._manager.email): + self._rds.job_client.scan_inbox() + job_manager = self._rds.job_client.manager + for ref in job_manager.iter_submission_refs(self._rds.email): self._try_distribute_job(ref) def _try_distribute_job(self, ref: JobRef): """Distribute a single enclave job to relevant DOs if not yet distributed.""" - job_manager = self._manager.job_client.manager + job_manager = self._rds.job_client.manager job_dir = job_manager.submission_dir(ref) config = job_manager.read_submission(ref) if config.job_type != "enclave" or not config.datasets: @@ -315,39 +315,37 @@ def _try_distribute_job(self, ref: JobRef): def _forward_job_to_dos(self, job_dir: Path, do_emails: list[str]): """Forward job files to DOs via the event-based outbox mechanism.""" files_by_datasite_path = self._get_files_in_dir(job_dir) - events_message = ( - self._manager.datasite_owner_syncer.event_cache.create_events_for_files( - files_by_datasite_path - ) + syncer = self._rds.sync_engine.datasite_owner_syncer + events_message = syncer.event_cache.create_events_for_files( + files_by_datasite_path ) - self._manager.datasite_owner_syncer.queue_event_for_syftbox( + syncer.queue_event_for_syftbox( recipients=do_emails, file_change_events_message=events_message, ) - self._manager.datasite_owner_syncer.process_syftbox_events_queue() + syncer.process_syftbox_events_queue() def _forward_approval_files_to_dos(self, review_dir: Path, do_emails: list[str]): """Forward each DO's approval state file to them individually.""" - datasite_dir = self._manager.syftbox_folder / self._manager.email + datasite_dir = self._rds.syftbox_folder / self._rds.email + syncer = self._rds.sync_engine.datasite_owner_syncer for do_email in do_emails: file_name = enclave_approval_file_name(do_email) approval_file = review_dir / file_name path_in_datasite = approval_file.relative_to(datasite_dir) files_by_datasite_path = {path_in_datasite: approval_file.read_bytes()} - events_message = ( - self._manager.datasite_owner_syncer.event_cache.create_events_for_files( - files_by_datasite_path - ) + events_message = syncer.event_cache.create_events_for_files( + files_by_datasite_path ) - self._manager.datasite_owner_syncer.queue_event_for_syftbox( + syncer.queue_event_for_syftbox( recipients=[do_email], file_change_events_message=events_message, ) - self._manager.datasite_owner_syncer.process_syftbox_events_queue() + syncer.process_syftbox_events_queue() def _get_files_in_dir(self, directory: Path) -> dict[Path, bytes]: """Read all files under directory, keyed by path relative to the datasite root.""" - datasite_dir = self._manager.syftbox_folder / self._manager.email + datasite_dir = self._rds.syftbox_folder / self._rds.email files_by_datasite_path = {} for f in directory.rglob("*"): if not f.is_file(): @@ -385,8 +383,8 @@ def _set_job_permissions( ): """Grant inbox read to everyone who needs to see the job (referenced + approving DOs), and approval-file write to the approving DOs.""" - job_manager = self._manager.job_client.manager - datasite = self._manager.syftbox_folder / self._manager.email + job_manager = self._rds.job_client.manager + datasite = self._rds.syftbox_folder / self._rds.email ctx = SyftPermContext(datasite=datasite) inbox_rel = job_manager.submission_dir(ref).relative_to(datasite) review_rel = job_manager.review_dir(ref).relative_to(datasite) @@ -401,25 +399,32 @@ def _set_job_permissions( @classmethod def from_config( cls, - config: SyftboxManagerConfig, + config: SyftRDSClientConfig, data_owners: list[str] | None = None, ) -> "SyftEnclaveClient": - """Build a SyftEnclaveClient from a manager config with a wrapped job_client. + """Build a SyftEnclaveClient from an RDS config with a wrapped job_client. + + The enclave client composes a ``SyftRDSClient`` (the RDS product client), + which OWNS the job + dataset surface and holds the generic sync engine at + ``rds.sync_engine``. We wrap the RDS-owned job_client with + ``EnclaveJobClient`` (settable on the RDS client) and install the private + dataset immutability filter on the nested sync engine's watcher cache. - Encryption rides along on ``config`` via + Encryption rides along on ``config.sync`` via ``peer_manager_config.use_encryption``; ``SyftboxManager.from_config`` - loads/generates this datasite's keys from its per-datasite key file, so no - extra step is needed. + (invoked inside ``SyftRDSClient.from_config``) loads/generates this + datasite's keys from its per-datasite key file, so no extra step is needed. """ - manager = SyftboxManager.from_config(config) - manager.job_client = EnclaveJobClient(manager.job_client) + rds = SyftRDSClient.from_config(config) + rds.job_client = EnclaveJobClient(rds.job_client) - if manager.datasite_watcher_syncer: - manager.datasite_watcher_syncer.datasite_watcher_cache.pre_write_filter = ( - make_private_dataset_immutability_filter(manager.syftbox_folder) + sync_engine = rds.sync_engine + if sync_engine.datasite_watcher_syncer: + sync_engine.datasite_watcher_syncer.datasite_watcher_cache.pre_write_filter = make_private_dataset_immutability_filter( + sync_engine.syftbox_folder ) - return cls(manager, data_owners=data_owners) + return cls(rds, data_owners=data_owners) @classmethod def for_enclave( @@ -436,7 +441,7 @@ def for_enclave( data_owners: Emails whose approval gates every job on this enclave. encryption: Enable end-to-end drive encryption. """ - config = SyftboxManagerConfig.for_jupyter( + config = SyftRDSClientConfig.for_jupyter( email=email, has_ds_role=True, has_do_role=True, @@ -483,7 +488,12 @@ def quad_with_mock_drive_service_connection( clients = create_clients(configs) enclave, do1, do2, ds = clients enclave.data_owners = [do1.email, do2.email] - managers = tuple(c._manager for c in clients) + # The setup helpers manipulate the generic sync engine directly + # (connections, file-watcher callbacks, version files, encryption, + # peering). Each client's sync engine lives at ``rds.sync_engine``; the + # RDS-owned job/dataset managers react via the peer-lifecycle callbacks + # registered on that same engine during SyftRDSClient.__init__. + managers = tuple(c._rds.sync_engine for c in clients) setup_connections(managers) setup_callbacks(managers) write_versions(managers) diff --git a/packages/syft-enclave/src/syft_enclaves/login.py b/packages/syft-enclave/src/syft_enclaves/login.py index 1018dd35505..6c2847bc8fc 100644 --- a/packages/syft-enclave/src/syft_enclaves/login.py +++ b/packages/syft-enclave/src/syft_enclaves/login.py @@ -5,8 +5,8 @@ from syft_client.sync.environments.environment import Environment from syft_client.sync.login import _init_client_login, _resolve_login_params from syft_client.sync.login_utils import handle_potential_version_mismatches_on_login -from syft_client.sync.syftbox_manager import SyftboxManagerConfig from syft_client.sync.utils.syftbox_utils import check_env +from syft_rds import SyftRDSClientConfig from syft_enclaves.client import SyftEnclaveClient @@ -38,7 +38,7 @@ def _login( handle_potential_version_mismatches_on_login(email, token_path) if env == Environment.COLAB: - config = SyftboxManagerConfig.for_colab( + config = SyftRDSClientConfig.for_colab( email=email, has_do_role=has_do_role, has_ds_role=has_ds_role, @@ -47,7 +47,7 @@ def _login( skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, ) else: - config = SyftboxManagerConfig.for_jupyter( + config = SyftRDSClientConfig.for_jupyter( email=email, has_do_role=has_do_role, has_ds_role=has_ds_role, @@ -62,8 +62,9 @@ def _login( client = SyftEnclaveClient.from_config(config) # Reuses syft-client's login init: verifies the token authenticates as - # `email`, writes the local version, then syncs / loads peers. - _init_client_login(client._manager, sync=sync, load_peers=load_peers) + # `email`, writes the local version, then syncs / loads peers. Operates on + # the generic sync engine nested inside the RDS client. + _init_client_login(client._rds.sync_engine, sync=sync, load_peers=load_peers) return client diff --git a/packages/syft-enclave/src/syft_enclaves/runner.py b/packages/syft-enclave/src/syft_enclaves/runner.py index ffbe33ed713..96a071c7f9a 100644 --- a/packages/syft-enclave/src/syft_enclaves/runner.py +++ b/packages/syft-enclave/src/syft_enclaves/runner.py @@ -112,7 +112,7 @@ def _on_initializing(self) -> None: "fresh_state=true — wiping ALL SyftBox state " "(local folder + Google Drive files) before init" ) - self.client._manager.delete_syftbox() + self.client.delete_syftbox() logger.info("State wipe complete — enclave starts with a clean slate") def _on_attesting(self) -> None: @@ -133,8 +133,9 @@ def _publish_attestation(self) -> None: """Fetch attestation JWT from the TEE and write it into the version file.""" eat_nonce = build_eat_nonce() token = fetch_attestation_token(eat_nonce=eat_nonce) - self.client._manager.peer_manager.get_own_version().attestation_token = token - self.client._manager.peer_manager.write_own_version() + peer_manager = self.client._rds.peer_manager + peer_manager.get_own_version().attestation_token = token + peer_manager.write_own_version() logger.info("Attestation token published to SYFT_version.json") def _on_peering(self) -> None: diff --git a/packages/syft-enclave/src/syft_enclaves/utils.py b/packages/syft-enclave/src/syft_enclaves/utils.py index 393df737263..8ccabeef540 100644 --- a/packages/syft-enclave/src/syft_enclaves/utils.py +++ b/packages/syft-enclave/src/syft_enclaves/utils.py @@ -1,4 +1,4 @@ -from syft_client.sync.syftbox_manager import SyftboxManagerConfig +from syft_rds import SyftRDSClientConfig from syft_client.sync.connections.drive.mock_drive_service import ( MockDriveBackingStore, MockDriveService, @@ -9,25 +9,25 @@ def create_configs( enclave_email, do1_email, do2_email, ds_email, use_in_memory_cache ) -> tuple: - enclave_config = SyftboxManagerConfig._base_config_for_testing( + enclave_config = SyftRDSClientConfig._base_config_for_testing( email=enclave_email, has_do_role=True, has_ds_role=True, use_in_memory_cache=use_in_memory_cache, ) - do1_config = SyftboxManagerConfig._base_config_for_testing( + do1_config = SyftRDSClientConfig._base_config_for_testing( email=do1_email, has_do_role=True, has_ds_role=True, use_in_memory_cache=use_in_memory_cache, ) - do2_config = SyftboxManagerConfig._base_config_for_testing( + do2_config = SyftRDSClientConfig._base_config_for_testing( email=do2_email, has_do_role=True, has_ds_role=True, use_in_memory_cache=use_in_memory_cache, ) - ds_config = SyftboxManagerConfig._base_config_for_testing( + ds_config = SyftRDSClientConfig._base_config_for_testing( email=ds_email, has_ds_role=True, use_in_memory_cache=use_in_memory_cache, diff --git a/packages/syft-enclave/tests/test_enclave_client.py b/packages/syft-enclave/tests/test_enclave_client.py index 4ad5c7743ec..909630906df 100644 --- a/packages/syft-enclave/tests/test_enclave_client.py +++ b/packages/syft-enclave/tests/test_enclave_client.py @@ -15,21 +15,21 @@ def test_quad_initialization(): assert all(isinstance(c, SyftEnclaveClient) for c in clients) # Correct roles - assert enclave._manager.has_do_role is True - assert enclave._manager.has_ds_role is True + assert enclave._rds.has_do_role is True + assert enclave._rds.has_ds_role is True - assert do1._manager.has_do_role is True - assert do1._manager.has_ds_role is True + assert do1._rds.has_do_role is True + assert do1._rds.has_ds_role is True - assert do2._manager.has_do_role is True - assert do2._manager.has_ds_role is True + assert do2._rds.has_do_role is True + assert do2._rds.has_ds_role is True - assert ds._manager.has_do_role is False - assert ds._manager.has_ds_role is True + assert ds._rds.has_do_role is False + assert ds._rds.has_ds_role is True - # Helper to get approved peer emails for a manager + # Helper to get approved peer emails for a client def approved_emails(client): - return {p.email for p in client._manager.peer_manager.approved_peers} + return {p.email for p in client._rds.peer_manager.approved_peers} # Enclave (DO-only): approved DS, DO1, DO2 assert approved_emails(enclave) == {ds.email, do1.email, do2.email} diff --git a/packages/syft-enclave/tests/test_enclave_datasets.py b/packages/syft-enclave/tests/test_enclave_datasets.py index 77ef5c992ca..7fdae2d83ff 100644 --- a/packages/syft-enclave/tests/test_enclave_datasets.py +++ b/packages/syft-enclave/tests/test_enclave_datasets.py @@ -38,7 +38,7 @@ def test_share_private_dataset_with_enclave(): # DO1 + DS sync -> DS can see mock data do1.sync() - ds._manager.sync() + ds._rds.sync() ds_datasets = ds.datasets.get_all() assert len(ds_datasets) == 1 assert ds_datasets[0].name == "testdataset" @@ -51,11 +51,7 @@ def test_share_private_dataset_with_enclave(): assert mock_content == "Hello, world!" non_existing_ds_private_dir = ( - ds._manager.syftbox_folder - / do1.email - / "private" - / "syft_datasets" - / "testdataset" + ds._rds.syftbox_folder / do1.email / "private" / "syft_datasets" / "testdataset" ) assert not non_existing_ds_private_dir.exists() @@ -63,12 +59,12 @@ def test_share_private_dataset_with_enclave(): do1.share_private_dataset("testdataset", enclave.email) # Enclave syncs (as DS) and pulls the private data events from DO1's outbox - enclave._manager.sync() + enclave._rds.sync() # Enclave can see the dataset via mock data (shared with DS and enclave shares peers) # But more importantly, enclave can access private files via shared_private_dir enclave_private_dir = ( - enclave._manager.syftbox_folder + enclave._rds.syftbox_folder / do1.email / "private" / "syft_datasets" diff --git a/packages/syft-enclave/tests/test_encryption.py b/packages/syft-enclave/tests/test_encryption.py index 056adab7816..226d78cf039 100644 --- a/packages/syft-enclave/tests/test_encryption.py +++ b/packages/syft-enclave/tests/test_encryption.py @@ -34,21 +34,21 @@ def test_encrypted_quad_keys_and_bundles(): ) for client in (enclave, do1, do2, ds): - store = client._manager._peer_store + store = client._rds.sync_engine._peer_store assert store is not None assert store.use_encryption is True assert store.has_my_keys() is True # The enclave peers with all three; it should hold each of their bundles # after wiring, so it can encrypt to and verify from them. - enclave_store = enclave._manager._peer_store + enclave_store = enclave._rds.sync_engine._peer_store for peer_email in (do1.email, do2.email, ds.email): assert enclave_store.has_peer_bundle(peer_email), ( f"enclave missing encryption bundle for {peer_email}" ) # And the DS holds the enclave's bundle (the link we care most about). - assert ds._manager._peer_store.has_peer_bundle(enclave.email) + assert ds._rds.sync_engine._peer_store.has_peer_bundle(enclave.email) def test_encrypted_message_to_enclave_roundtrips(): diff --git a/packages/syft-enclave/tests/test_immutability.py b/packages/syft-enclave/tests/test_immutability.py index 15ea205ada1..7057f965511 100644 --- a/packages/syft-enclave/tests/test_immutability.py +++ b/packages/syft-enclave/tests/test_immutability.py @@ -122,10 +122,10 @@ def test_enclave_blocks_reshare_of_private_dataset(): # First share + sync — enclave receives private data do1.share_private_dataset("testdataset", enclave.email) - enclave._manager.sync() + enclave._rds.sync() enclave_private_dir = ( - enclave._manager.syftbox_folder + enclave._rds.syftbox_folder / do1.email / "private" / "syft_datasets" @@ -137,7 +137,7 @@ def test_enclave_blocks_reshare_of_private_dataset(): # DO modifies private data locally and re-shares private_path.write_text("TAMPERED DATA") - do1._manager.dataset_manager.delete("testdataset", require_confirmation=False) + do1._rds.dataset_manager.delete("testdataset", require_confirmation=False) do1.create_dataset( name="testdataset", mock_path=mock_path, @@ -148,7 +148,7 @@ def test_enclave_blocks_reshare_of_private_dataset(): sync=False, ) do1.share_private_dataset("testdataset", enclave.email) - enclave._manager.sync() + enclave._rds.sync() # Enclave still has original content — overwrite was blocked assert (enclave_private_dir / "private.txt").read_bytes() == original_content diff --git a/packages/syft-enclave/tests/test_runner.py b/packages/syft-enclave/tests/test_runner.py index c51a9c1c48d..c9ce52edca8 100644 --- a/packages/syft-enclave/tests/test_runner.py +++ b/packages/syft-enclave/tests/test_runner.py @@ -26,7 +26,7 @@ def test_fresh_state_true_invokes_delete_syftbox(tmp_path, monkeypatch): runner = EnclaveRunner(client=client, fresh_state=True) runner.init() - client._manager.delete_syftbox.assert_called_once_with() + client.delete_syftbox.assert_called_once_with() def test_fresh_state_false_skips_delete_syftbox(tmp_path, monkeypatch): @@ -39,7 +39,7 @@ def test_fresh_state_false_skips_delete_syftbox(tmp_path, monkeypatch): runner = EnclaveRunner(client=client, fresh_state=False) runner.init() - client._manager.delete_syftbox.assert_not_called() + client.delete_syftbox.assert_not_called() def test_fresh_state_default_is_true(tmp_path, monkeypatch): @@ -52,7 +52,7 @@ def test_fresh_state_default_is_true(tmp_path, monkeypatch): runner = EnclaveRunner(client=client) # no fresh_state arg assert runner.fresh_state is True runner.init() - client._manager.delete_syftbox.assert_called_once_with() + client.delete_syftbox.assert_called_once_with() def test_fresh_state_uses_default_kwargs_on_delete(tmp_path, monkeypatch): @@ -66,6 +66,6 @@ def test_fresh_state_uses_default_kwargs_on_delete(tmp_path, monkeypatch): # Must be called with no positional or keyword args — let the method's # own defaults handle broadcast_delete_events and verbose. - call = client._manager.delete_syftbox.call_args + call = client.delete_syftbox.call_args assert call.args == () assert call.kwargs == {} diff --git a/packages/syft-rds/pyproject.toml b/packages/syft-rds/pyproject.toml new file mode 100644 index 00000000000..57e82968466 --- /dev/null +++ b/packages/syft-rds/pyproject.toml @@ -0,0 +1,25 @@ +[project] +name = "syft-rds" +version = "0.1.0" +description = "Remote Data Science product: datasets + jobs composed on top of the syft-client sync engine" +authors = [{ name = "OpenMined", email = "info@openmined.org" }] +license = { text = "Apache-2.0" } +requires-python = ">=3.10" + +dependencies = [ + "syft-client", + "syft-job", + "syft-dataset", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.uv.sources] +"syft-client" = { workspace = true } +"syft-job" = { workspace = true } +"syft-dataset" = { workspace = true } + +[tool.hatch.build.targets.wheel] +packages = ["src/syft_rds"] diff --git a/packages/syft-rds/src/syft_rds/__init__.py b/packages/syft-rds/src/syft_rds/__init__.py new file mode 100644 index 00000000000..bd434a50625 --- /dev/null +++ b/packages/syft-rds/src/syft_rds/__init__.py @@ -0,0 +1,27 @@ +"""syft-rds: Remote Data Science product composed on top of syft-client.""" + +from syft_rds.client import SyftRDSClient +from syft_rds.config import SyftRDSClientConfig +from syft_rds.job_auto_approval import auto_approve_and_run_jobs, job_matches_criteria +from syft_rds.login import login_do, login_ds + +# Generic sync-stack helpers re-exported so consumers that compose the RDS product +# (e.g. syft-bg) have a single integration surface and need not import from +# syft_client.sync internals directly. +from syft_client.sync.syftbox_manager import get_jupyter_default_syftbox_folder +from syft_client.sync.utils.path_filters import is_normal_syncable_path +from syft_client.sync.environments.environment import Environment +from syft_client.sync.utils.syftbox_utils import check_env + +__all__ = [ + "SyftRDSClient", + "SyftRDSClientConfig", + "login_do", + "login_ds", + "auto_approve_and_run_jobs", + "job_matches_criteria", + "get_jupyter_default_syftbox_folder", + "is_normal_syncable_path", + "Environment", + "check_env", +] diff --git a/packages/syft-rds/src/syft_rds/client.py b/packages/syft-rds/src/syft_rds/client.py new file mode 100644 index 00000000000..ffb6cd50148 --- /dev/null +++ b/packages/syft-rds/src/syft_rds/client.py @@ -0,0 +1,670 @@ +"""The Remote Data Science client.""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path +from typing import Any + +import yaml +from pydantic import BaseModel, ConfigDict + +from syft_client.sync.syftbox_manager import SyftboxManager +from syft_datasets.dataset_manager import ( + DATASET_COLLECTION_PREFIX, + PRIVATE_DATASET_COLLECTION_PREFIX, +) +from syft_client.sync.utils.pre_submit_scan import run_pre_submit_check +from syft_client.sync.version.peer_manager import CompatAction +from syft_job.client import JobClient +from syft_job.job_runner import SyftJobRunner +from syft_datasets.dataset_manager import SyftDatasetManager +from syft_rds.config import DATASET_COLLECTION_SPECS, SyftRDSClientConfig + +logger = logging.getLogger(__name__) + + +class SyftRDSClient(BaseModel): + # Holds live service objects (sync engine + RDS-owned managers), not + # serializable data, so arbitrary types are allowed. + model_config = ConfigDict(arbitrary_types_allowed=True) + + # The nested generic sync engine (composition). + sync_engine: SyftboxManager + # RDS-owned domain managers, exposed so consumers (e.g. syft-enclave, which + # wraps the job client) can read/replace them. + job_client: JobClient + job_runner: SyftJobRunner | None = None + dataset_manager: SyftDatasetManager + + def model_post_init(self, __context: Any) -> None: + # React to sync-engine peer lifecycle events with RDS-owned logic. + self.sync_engine.on("peer_approved", self._on_peer_approved) + self.sync_engine.on("peers_loaded", self._on_peers_loaded) + # React to collection restores so RDS can run dataset-specific post-processing + # (the private dataset's data_dir fixup) without the sync core knowing datasets. + if self.sync_engine.datasite_owner_syncer is not None: + self.sync_engine.datasite_owner_syncer.on( + "collection_restored", self._on_collection_restored + ) + + @classmethod + def from_config(cls, config: "SyftRDSClientConfig") -> "SyftRDSClient": + sync_engine = SyftboxManager.from_config(config.sync) + job_client = JobClient.from_config(config.job) + job_runner = ( + SyftJobRunner.from_config(config.job) if config.sync.has_do_role else None + ) + dataset_manager = SyftDatasetManager.from_config(config.dataset) + return cls( + sync_engine=sync_engine, + job_client=job_client, + job_runner=job_runner, + dataset_manager=dataset_manager, + ) + + @classmethod + def _build_rds_pair_from_managers( + cls, ds_mgr: SyftboxManager, do_mgr: SyftboxManager + ) -> tuple["SyftRDSClient", "SyftRDSClient"]: + """Wrap a paired ``(ds, do)`` ``SyftboxManager`` tuple into RDS clients. + + Peers were already approved/loaded during pairing (before our callbacks + were registered), so we replay the DO-side setup so its owned + ``JobClient`` has the DS job folders + install sources. + """ + + def _build(mgr: SyftboxManager) -> "SyftRDSClient": + assert mgr.config is not None, "paired managers are built from a config" + # Compose via the config so the sub-configs are derived (and + # alignment-checked) exactly as they are on the from_config path. + config = SyftRDSClientConfig._compose(mgr.config) + return cls( + sync_engine=mgr, + job_client=JobClient.from_config(config.job), + job_runner=( + SyftJobRunner.from_config(config.job) if mgr.has_do_role else None + ), + dataset_manager=SyftDatasetManager.from_config(config.dataset), + ) + + ds_rds = _build(ds_mgr) + do_rds = _build(do_mgr) + do_rds._on_peers_loaded() + for peer in do_mgr.peer_manager.approved_peers: + do_rds._on_peer_approved(peer.email) + ds_rds._on_peers_loaded() + return ds_rds, do_rds + + @classmethod + def pair_with_mock_drive_service_connection( + cls, **kwargs: Any + ) -> tuple["SyftRDSClient", "SyftRDSClient"]: + """(ds, do) pair of self-contained RDS clients sharing one mock Drive.""" + ds_mgr, do_mgr = SyftboxManager.pair_with_mock_drive_service_connection( + collection_specs=DATASET_COLLECTION_SPECS, **kwargs + ) + return cls._build_rds_pair_from_managers(ds_mgr, do_mgr) + + @classmethod + def _pair_with_google_drive_testing_connection( + cls, **kwargs: Any + ) -> tuple["SyftRDSClient", "SyftRDSClient"]: + """(ds, do) pair of self-contained RDS clients sharing a REAL Google Drive.""" + ds_mgr, do_mgr = SyftboxManager._pair_with_google_drive_testing_connection( + collection_specs=DATASET_COLLECTION_SPECS, **kwargs + ) + return cls._build_rds_pair_from_managers(ds_mgr, do_mgr) + + def __dir__(self) -> list[str]: + """Public API only, without pydantic's machinery.""" + names = set(type(self).model_fields) + for klass in type(self).__mro__: + if klass is BaseModel: + break + names.update( + name + for name, value in vars(klass).items() + if not name.startswith(("_", "model_")) + and not isinstance(value, (classmethod, staticmethod)) + ) + return sorted(names) + + # ------------------------------------------------------------------ # + # delegated identity + sync surface (owned by the generic core) + # ------------------------------------------------------------------ # + @property + def email(self) -> str: + return self.sync_engine.email + + @property + def syftbox_folder(self) -> Path: + return self.sync_engine.syftbox_folder + + @property + def has_do_role(self) -> bool: + return self.sync_engine.has_do_role + + @property + def has_ds_role(self) -> bool: + return self.sync_engine.has_ds_role + + @property + def peer_manager(self) -> Any: + return self.sync_engine.peer_manager + + @property + def peers(self) -> Any: + """Combined peer list (approved + requests). Delegates to the sync engine. + + Auto-syncs first when ``PRE_SYNC`` is enabled (the engine's behavior). + """ + return self.sync_engine.peers + + def sync(self, *args: Any, **kwargs: Any) -> Any: + return self.sync_engine.sync(*args, **kwargs) + + def add_peer(self, *args: Any, **kwargs: Any) -> Any: + return self.sync_engine.add_peer(*args, **kwargs) + + def approve_peer_request(self, *args: Any, **kwargs: Any) -> Any: + return self.sync_engine.approve_peer_request(*args, **kwargs) + + def reject_peer_request(self, *args: Any, **kwargs: Any) -> Any: + return self.sync_engine.reject_peer_request(*args, **kwargs) + + def load_peers(self, *args: Any, **kwargs: Any) -> Any: + return self.sync_engine.load_peers(*args, **kwargs) + + def delete_syftbox(self, *args: Any, **kwargs: Any) -> Any: + return self.sync_engine.delete_syftbox(*args, **kwargs) + + # ------------------------------------------------------------------ # + # peer lifecycle callbacks (RDS-owned reactions to sync events) + # ------------------------------------------------------------------ # + def _on_peer_approved(self, peer_email: str) -> None: + """A peer was approved: set up their DS job folder and share datasets.""" + if self.has_do_role: + self.job_client.setup_ds_job_folder_as_do(peer_email) + self._share_any_datasets_with_peer(peer_email) + + def _on_collection_restored(self, prefix: str, tag: str, local_dir: Any) -> None: + """A collection was restored from the backend by the owner-syncer. + + For the PRIVATE dataset collection, rewrite ``private_metadata.yaml``'s + machine-specific ``data_dir`` to the current local path, so job execution + can locate the real data after a restore onto a new machine/path. This is + dataset domain knowledge, so it lives here (rds) rather than the sync core. + """ + if prefix != PRIVATE_DATASET_COLLECTION_PREFIX: + return + + metadata_path = Path(local_dir) / "private_metadata.yaml" + if not metadata_path.exists(): + return + data = yaml.safe_load(metadata_path.read_text()) + if not data: + # Missing or empty file: nothing to rewrite (and guards against + # yaml.safe_load returning None on an empty file). + return + expected_dir = str(local_dir) + if data.get("data_dir") != expected_dir: + data["data_dir"] = expected_dir + metadata_path.write_text(yaml.safe_dump(data, indent=2, sort_keys=False)) + + def _on_peers_loaded(self, *args: Any, **kwargs: Any) -> None: + """Peers were (re)loaded: copy each peer's advertised install source + into our owned job client so submitted run.sh references the DO's path.""" + for peer in self.sync_engine.peer_manager.syncable_peers: + if peer.version: + self.job_client.peer_install_sources[peer.email] = ( + peer.version.syft_client_install_source + ) + + def _share_any_datasets_with_peer(self, peer_email: str) -> None: + """Share all datasets tagged 'any' with a specific peer. + + Google Drive "anyone with link" files are not discoverable via search, + so explicit user sharing is added. Reads the cache populated during + ``pull_initial_state()`` in the nested DatasiteOwnerSyncer. + """ + for ( + tag, + content_hash, + ) in self.sync_engine.datasite_owner_syncer.any_shared_collections: + try: + self.sync_engine.share_collection( + DATASET_COLLECTION_PREFIX, tag, content_hash, [peer_email] + ) + except Exception: + # One collection failing (missing folder, quota, network) must + # not stop us sharing the rest with this peer. "alreadyShared" + # is already handled in _batch_add_permissions, so anything + # reaching here is a real failure worth a traceback. + logger.exception( + "Failed to share collection %r with %s", tag, peer_email + ) + + # ------------------------------------------------------------------ # + # job product surface (RDS-owned) + # ------------------------------------------------------------------ # + def submit_bash_job( + self, + user: str, + script: str, + job_name: str = "", + force_submission: bool = False, + ignore_peer_version: bool = False, + ): + # Check version compatibility before submission (uses cached versions) + if not force_submission: + result = self.sync_engine.peer_manager.get_peer_compatibility_status( + user, + action=CompatAction.SUBMIT, + ignore_peer_version=ignore_peer_version, + ) + result.raise_on_skip(operation="submit job") + result.maybe_warn() + job_dir = self.job_client.submit_bash_job(user, script, job_name=job_name) + self.sync_engine.push_job_files(job_dir) + + def submit_python_job( + self, + user: str, + code_path: str, + job_name: str | None = "", + dependencies: list[str] | None = None, + entrypoint: str | None = None, + force_submission: bool = False, + ignore_peer_version: bool = False, + ): + peer_emails = {p.email for p in self.sync_engine.peer_manager.syncable_peers} + if user not in peer_emails: + print(f"⚠️ {user} is not in your peer list.") + print(f" Add them first with: client.add_peer('{user}')") + return + + if not force_submission: + if not run_pre_submit_check(Path(code_path)): + print("Submission aborted.") + return + + print(f"📤 Submitting '{code_path}' to {user}...") + if job_name: + print(f" Job name : {job_name}") + if dependencies: + print(f" Dependencies : {', '.join(dependencies)}") + + # Check version compatibility before submission (uses cached versions) + if not force_submission: + result = self.sync_engine.peer_manager.get_peer_compatibility_status( + user, + action=CompatAction.SUBMIT, + ignore_peer_version=ignore_peer_version, + ) + result.raise_on_skip(operation="submit job") + result.maybe_warn() + job_dir = self.job_client.submit_python_job( + user, + code_path, + job_name=job_name, + dependencies=dependencies, + entrypoint=entrypoint, + ) + self.sync_engine.push_job_files(job_dir) + + print("\n✅ Job submitted successfully!") + print(" Status : inbox (waiting for DO to review)") + print(f"\n⏳ Next step: wait for {user} to approve and run it.") + print(" Check progress with: client.jobs") + + @property + def _pre_sync_enabled(self) -> bool: + """Whether accessors auto-sync (disabled by setting ``PRE_SYNC=false``).""" + return os.environ.get("PRE_SYNC", "true").lower() == "true" + + @property + def jobs(self) -> Any: + """List of jobs. Auto-syncs first unless PRE_SYNC=false.""" + if self._pre_sync_enabled: + self.sync_engine.sync() + return self.job_client.jobs + + def process_approved_jobs( + self, + stream_output: bool = True, + timeout: int | None = None, + force_execution: bool = False, + share_outputs_with_submitter: bool = False, + share_logs_with_submitter: bool = False, + ignore_peer_version: bool = False, + ) -> None: + """Process approved jobs (DO only). Auto-syncs after unless PRE_SYNC=false.""" + if not self.has_do_role: + raise ValueError("Only dataset owners can process approved jobs") + if self.job_runner is None: + raise ValueError("Job runner is not configured for this client") + + skip_job_names = [] + + if not force_execution: + approved_jobs = [ + job for job in self.job_client.jobs if job.status == "approved" + ] + for job in approved_jobs: + result = self.sync_engine.peer_manager.get_peer_compatibility_status( + job.submitted_by, + action=CompatAction.EXECUTE, + ignore_peer_version=ignore_peer_version, + ) + result.maybe_warn() + if result.should_skip: + skip_job_names.append(job.name) + + self.job_runner.process_approved_jobs( + stream_output=stream_output, + timeout=timeout, + skip_job_names=skip_job_names if skip_job_names else None, + share_outputs_with_submitter=share_outputs_with_submitter, + share_logs_with_submitter=share_logs_with_submitter, + ) + + if self._pre_sync_enabled: + self.sync_engine.sync() + + # ------------------------------------------------------------------ # + # dataset product surface (RDS-owned) + # ------------------------------------------------------------------ # + def create_dataset( + self, + name: str, + mock_path: str | Path, + private_path: str | Path, + summary: str | None = None, + readme_path: Path | None = None, + location: str | None = None, + tags: list[str] | None = None, + users: list[str] | str | None = None, + upload_private: bool = False, + sync=True, + ): + if self.dataset_manager is None: + raise ValueError("Dataset manager is not set") + + # Only DO can create datasets + if not self.has_do_role: + raise ValueError("Only dataset owners can create datasets") + + # Convert None to empty list + if users is None: + users = [] + + dataset_name = None + created_local = False + mock_folder_id = None + private_folder_id = None + + try: + # Create dataset locally + dataset = self.dataset_manager.create( + name=name, + mock_path=mock_path, + private_path=private_path, + summary=summary, + readme_path=readme_path, + location=location, + tags=tags, + users=users, + ) + created_local = True + dataset_name = dataset.name + + # Upload mock data to collection folder + mock_folder_id = self._upload_dataset_to_collection(dataset, users) + + # Upload private data to a separate owner-only collection + if upload_private: + private_folder_id = self._upload_private_dataset_to_collection(dataset) + + if sync: + self.sync() + + return dataset + + except Exception: + logger.error( + "Failed to create dataset%s, cleaning up", + f" '{dataset_name}'" if dataset_name else "", + ) + self._cleanup_failed_dataset_creation( + dataset_name, created_local, mock_folder_id, private_folder_id + ) + raise + + def _cleanup_failed_dataset_creation( + self, + dataset_name: str | None, + created_local: bool, + mock_folder_id: str | None, + private_folder_id: str | None, + ) -> None: + """Best-effort cleanup after a failed create_dataset, in reverse order.""" + if private_folder_id is not None: + try: + self.sync_engine.delete_file_by_id(private_folder_id) + except Exception: + logger.warning( + "Cleanup: failed to delete private GDrive folder %s", + private_folder_id, + ) + + if mock_folder_id is not None: + try: + self.sync_engine.delete_file_by_id(mock_folder_id) + except Exception: + logger.warning( + "Cleanup: failed to delete mock GDrive folder %s", + mock_folder_id, + ) + + if created_local and dataset_name is not None: + try: + self.dataset_manager.delete(dataset_name, require_confirmation=False) + except Exception: + logger.warning( + "Cleanup: failed to delete local dataset '%s'", + dataset_name, + ) + + def _create_and_upload_collection( + self, prefix: str, tag: str, files: dict[str, bytes] + ) -> tuple[str, str]: + """Create a hash-named collection folder and upload its files. + + Returns ``(folder_id, content_hash)``. + """ + from syft_client.sync.connections.drive.gdrive_transport import ( + CollectionFolder, + ) + + content_hash = CollectionFolder.compute_hash(files) + folder_id = self.sync_engine.create_collection_folder( + prefix, tag=tag, content_hash=content_hash + ) + self.sync_engine.upload_collection_files(prefix, tag, content_hash, files) + return folder_id, content_hash + + def _collect_mock_files(self, dataset) -> dict[str, bytes]: + """Read a dataset's mock files, metadata and readme into a name->bytes map.""" + files = {} + for mock_file in dataset.mock_files: + if mock_file.exists(): + files[mock_file.name] = mock_file.read_bytes() + + metadata_path = dataset.mock_dir / "dataset.yaml" + if metadata_path.exists(): + files["dataset.yaml"] = metadata_path.read_bytes() + + if dataset.readme_path and dataset.readme_path.exists(): + files[dataset.readme_path.name] = dataset.readme_path.read_bytes() + return files + + def _share_dataset_collection( + self, tag: str, content_hash: str, users: list[str] | str + ) -> None: + """Share a dataset collection with ``users``, or tag it ``"any"`` and + share with all already-approved peers.""" + if users == "any": + self.sync_engine.tag_collection_as_any( + DATASET_COLLECTION_PREFIX, tag, content_hash + ) + self.sync_engine.datasite_owner_syncer.register_any_shared_collection( + tag, content_hash + ) + peer_emails = [ + p.email for p in self.sync_engine.peer_manager.approved_peers + ] + if peer_emails: + self.sync_engine.share_collection( + DATASET_COLLECTION_PREFIX, tag, content_hash, peer_emails + ) + else: + if isinstance(users, str): + users = [users] + self.sync_engine.share_collection( + DATASET_COLLECTION_PREFIX, tag, content_hash, users + ) + + def _upload_dataset_to_collection(self, dataset, users: list[str] | str) -> str: + """Upload dataset files to collection folder. Returns the folder ID.""" + files = self._collect_mock_files(dataset) + folder_id, content_hash = self._create_and_upload_collection( + DATASET_COLLECTION_PREFIX, dataset.name, files + ) + self._share_dataset_collection(dataset.name, content_hash, users) + return folder_id + + def _upload_private_dataset_to_collection(self, dataset) -> str | None: + """Upload private dataset files to a separate owner-only collection folder. + Returns the folder ID, or None if no files to upload.""" + collection_tag = dataset.name + + # Collect all files in private dir (data, metadata, permissions) + files = {} + for f in dataset.private_dir.iterdir(): + if f.is_file(): + files[f.name] = f.read_bytes() + + if not files: + return None + + # Private collection: no sharing step. + folder_id, _ = self._create_and_upload_collection( + PRIVATE_DATASET_COLLECTION_PREFIX, collection_tag, files + ) + return folder_id + + def delete_dataset( + self, + name: str, + datasite: str | None = None, + require_confirmation: bool = True, + sync=True, + ): + if self.dataset_manager is None: + raise ValueError("Dataset manager is not set") + self.dataset_manager.delete( + name=name, + datasite=datasite, + require_confirmation=require_confirmation, + ) + # Delete collection folders from Google Drive so DS peers + # pick up the deletion on their next sync. + try: + self.sync_engine.delete_collection(DATASET_COLLECTION_PREFIX, name) + except Exception: + logger.warning("Failed to delete dataset collection '%s' from Drive", name) + try: + self.sync_engine.delete_collection(PRIVATE_DATASET_COLLECTION_PREFIX, name) + except Exception: + logger.warning( + "Failed to delete private dataset collection '%s' from Drive", + name, + ) + if sync: + self.sync() + + def share_dataset(self, tag: str, users: list[str] | str, sync=True): + """ + Share an existing dataset with additional users. + + Args: + tag: Dataset name + users: List of email addresses or "any" + sync: Whether to sync after sharing + """ + from syft_client.sync.connections.drive.gdrive_transport import ( + CollectionFolder, + ) + + if self.dataset_manager is None: + raise ValueError("Dataset manager is not set") + + if not self.has_do_role: + raise ValueError("Only dataset owners can share datasets") + + # Verify dataset exists + dataset = self.dataset_manager.get(name=tag, datasite=self.email) + if dataset is None: + raise ValueError(f"Dataset {tag} not found") + + # Compute current content hash from local files, then share. + files = self._collect_mock_files(dataset) + content_hash = CollectionFolder.compute_hash(files) + self._share_dataset_collection(tag, content_hash, users) + + if sync: + self.sync() + + def share_private_dataset(self, tag: str, enclave_email: str): + """Share private dataset files with an enclave via outbox events.""" + if not self.has_do_role: + raise ValueError("Only data owners can share private datasets") + + with self.sync_engine.sync_file_lock(): + files = self.dataset_manager.get_private_dataset_files(tag) + events_message = self.sync_engine.datasite_owner_syncer.event_cache.create_events_for_files( + files + ) + self.sync_engine.datasite_owner_syncer.queue_event_for_syftbox( + recipients=[enclave_email], + file_change_events_message=events_message, + ) + self.sync_engine.datasite_owner_syncer.process_syftbox_events_queue() + + @property + def datasets(self) -> Any: + """The dataset manager. Auto-syncs first unless PRE_SYNC=false.""" + if self.dataset_manager is None: + raise ValueError("Dataset manager is not set") + + if self._pre_sync_enabled: + self.sync_engine.sync() + + return self.dataset_manager + + def _resolve_dataset_owners_for_name(self, dataset_name: str) -> list: + """Resolve which datasite(s) own a dataset of the given name. + + Used by syft_client.utils.resolve_dataset_files_path when a client is + passed. Owned here (drives the RDS dataset manager). + """ + return [ + dataset.owner + for dataset in self.dataset_manager.get_all() + if dataset.name == dataset_name + ] + + def __repr__(self) -> str: + return f"SyftRDSClient(email={self.sync_engine.email!r})" diff --git a/packages/syft-rds/src/syft_rds/config.py b/packages/syft-rds/src/syft_rds/config.py new file mode 100644 index 00000000000..8a76babf2fa --- /dev/null +++ b/packages/syft-rds/src/syft_rds/config.py @@ -0,0 +1,199 @@ +"""Configuration for the Remote Data Science product. + +``SyftRDSClientConfig`` COMPOSES the three sub-configs the RDS layer owns: + +* ``sync`` – the (domain-free) ``SyftboxManagerConfig`` sync engine config, +* ``job`` – the ``SyftJobConfig`` for the RDS-owned ``JobClient``/``SyftJobRunner``, +* ``dataset`` – the ``SyftBoxConfig`` for the RDS-owned ``SyftDatasetManager``. + +All three are derived from the SAME primitives (email, syftbox folder, role) +so their paths always line up. The RDS layer is also the place where the +dataset collection sync specs are *supplied* into the generic sync core; the +core itself stays domain-free. +""" + +from __future__ import annotations + +from pathlib import Path + +from pydantic import BaseModel, model_validator + +from syft_client.sync.syftbox_manager import ( + SyftboxManagerConfig, +) +from syft_client.sync.sync.collection_spec import CollectionSyncSpec +from syft_job import SyftJobConfig +from syft_datasets.config import SyftBoxConfig +from syft_datasets.dataset_manager import ( + DATASET_COLLECTION_PREFIX, + PRIVATE_DATASET_COLLECTION_PREFIX, +) + +# The RDS layer owns the local subpaths; the on-wire prefixes come from +# syft_datasets (imported above), mirrored in syft_client for login-time cleanup. +COLLECTION_SUBPATH = Path("public/syft_datasets") +PRIVATE_COLLECTION_SUBPATH = Path("private/syft_datasets") + +# The RDS layer OWNS the dataset collection specs (syft-client core stays domain-free). +# Two specs, distinguished purely by the two generic behavioral flags: +# * public (mock) – mirror + shareable → peers' watchers pull it. +# * private (real) – restore-only + owner-only → the owner restores it for itself; +# peer-facing watchers skip it; it is never shared. +DATASET_COLLECTION_SPECS = [ + CollectionSyncSpec.public(DATASET_COLLECTION_PREFIX, COLLECTION_SUBPATH), + CollectionSyncSpec.private( + PRIVATE_DATASET_COLLECTION_PREFIX, PRIVATE_COLLECTION_SUBPATH + ), +] + + +class SyftRDSClientConfig(BaseModel): + sync: SyftboxManagerConfig + job: SyftJobConfig + dataset: SyftBoxConfig + + @model_validator(mode="after") + def _assert_sub_configs_aligned(self) -> "SyftRDSClientConfig": + """Enforce the invariant the ``_compose`` docstring promises: all three + sub-configs are derived from the SAME primitives, so their paths line up. + + Guards against a hand-built config (or a future factory) drifting the + email / folder / role across the sync engine, job client, and dataset + manager, which would silently point them at mismatched datasite paths. + """ + emails = { + "sync": self.sync.email, + "job": self.job.current_user_email, + "dataset": self.dataset.email, + } + if len(set(emails.values())) > 1: + raise ValueError(f"sub-config emails must all match, got {emails}") + + folders = { + "sync": self.sync.syftbox_folder, + "job": self.job.syftbox_folder, + "dataset": self.dataset.syftbox_folder, + } + if len(set(folders.values())) > 1: + raise ValueError( + f"sub-config syftbox_folders must all match, got {folders}" + ) + + if self.sync.has_do_role != self.job.has_do_role: + raise ValueError( + "sync.has_do_role and job.has_do_role must match, got " + f"{self.sync.has_do_role} vs {self.job.has_do_role}" + ) + return self + + # ------------------------------------------------------------------ # + # private helper: build all three sub-configs from shared primitives + # ------------------------------------------------------------------ # + @staticmethod + def _compose(sync: SyftboxManagerConfig) -> "SyftRDSClientConfig": + """Build the composed config using ``sync`` as the single source of truth. + + ``sync`` is built by the caller (it differs per environment); ``job`` and + ``dataset`` are derived here from the sync engine's OWN email/folder/role + rather than from separately-passed primitives, so the sub-configs cannot + drift apart. The ``_assert_sub_configs_aligned`` validator enforces this. + """ + job = SyftJobConfig( + syftbox_folder=sync.syftbox_folder, + current_user_email=sync.email, + has_do_role=sync.has_do_role, + ) + dataset = SyftBoxConfig(syftbox_folder=sync.syftbox_folder, email=sync.email) + return SyftRDSClientConfig(sync=sync, job=job, dataset=dataset) + + @classmethod + def for_jupyter( + cls, + email, + has_do_role=False, + has_ds_role=False, + token_path=None, + **kw, + ) -> "SyftRDSClientConfig": + sync = SyftboxManagerConfig.for_jupyter( + email=email, + has_do_role=has_do_role, + has_ds_role=has_ds_role, + token_path=token_path, + collection_specs=DATASET_COLLECTION_SPECS, + **kw, + ) + return cls._compose(sync) + + @classmethod + def for_colab( + cls, + email, + has_do_role=False, + has_ds_role=False, + **kw, + ) -> "SyftRDSClientConfig": + sync = SyftboxManagerConfig.for_colab( + email=email, + has_do_role=has_do_role, + has_ds_role=has_ds_role, + collection_specs=DATASET_COLLECTION_SPECS, + **kw, + ) + return cls._compose(sync) + + @classmethod + def _base_config_for_testing( + cls, + email=None, + syftbox_folder=None, + has_do_role=False, + has_ds_role=False, + **kw, + ) -> "SyftRDSClientConfig": + """Build a composed config over ``SyftboxManagerConfig._base_config_for_testing``. + + The sync sub-config is the base testing config (in-/out-of-memory caches, + mock connections wired in later); ``job`` and ``dataset`` are derived from + the SAME email + folder that the sync config resolved, so paths align. + """ + sync = SyftboxManagerConfig._base_config_for_testing( + email=email, + syftbox_folder=syftbox_folder, + has_do_role=has_do_role, + has_ds_role=has_ds_role, + collection_specs=DATASET_COLLECTION_SPECS, + **kw, + ) + # _base_config_for_testing may have generated a random email/folder; + # _compose reads them back off the resolved sync config. + return cls._compose(sync) + + @classmethod + def for_google_drive_testing_connection( + cls, + email, + token_path, + syftbox_folder=None, + has_do_role=False, + has_ds_role=False, + **kw, + ) -> "SyftRDSClientConfig": + """Build a composed config over ``SyftboxManagerConfig.for_google_drive_testing_connection``. + + Same shape as :meth:`_base_config_for_testing`, but the sync sub-config is + wired to a REAL Google Drive connection (via ``token_path``) instead of the + in-memory mock — for integration tests that exercise the actual transport. + ``job`` and ``dataset`` are derived from the SAME email + folder the sync + config resolved, so paths align. + """ + sync = SyftboxManagerConfig.for_google_drive_testing_connection( + email=email, + token_path=token_path, + syftbox_folder=syftbox_folder, + has_do_role=has_do_role, + has_ds_role=has_ds_role, + collection_specs=DATASET_COLLECTION_SPECS, + **kw, + ) + return cls._compose(sync) diff --git a/syft_client/job_auto_approval.py b/packages/syft-rds/src/syft_rds/job_auto_approval.py similarity index 95% rename from syft_client/job_auto_approval.py rename to packages/syft-rds/src/syft_rds/job_auto_approval.py index 711199f916a..a8d491018f5 100644 --- a/syft_client/job_auto_approval.py +++ b/packages/syft-rds/src/syft_rds/job_auto_approval.py @@ -10,7 +10,7 @@ from syft_job.job import JobInfo if TYPE_CHECKING: - from syft_client.sync.syftbox_manager import SyftboxManager + from syft_rds.client import SyftRDSClient def _get_non_empty_lines(content: str) -> list[str]: @@ -143,7 +143,7 @@ def job_matches_criteria( def auto_approve_and_run_jobs( - client: SyftboxManager, + client: SyftRDSClient, *, required_file_contents: Dict[str, str], required_file_paths: List[str], @@ -164,7 +164,7 @@ def auto_approve_and_run_jobs( 5. (Optional) Were submitted by an approved peer Args: - client: SyftboxManager instance + client: SyftRDSClient instance required_file_contents: Dict mapping filename to expected content. Content is compared after stripping trailing whitespace. Example: {"main.py": "print('hello')"} @@ -180,8 +180,8 @@ def auto_approve_and_run_jobs( List of JobInfo objects that were approved. Example: - >>> from syft_client.sync.syftbox_manager import SyftboxManager - >>> client = SyftboxManager.for_jupyter(email="me@example.com", ...) + >>> from syft_rds import login_do + >>> client = login_do(email="me@example.com", ...) >>> approved = auto_approve_and_run_jobs( ... client, ... required_file_contents={"main.py": EXPECTED_SCRIPT}, @@ -193,7 +193,7 @@ def auto_approve_and_run_jobs( approved_peers = None if peers_only: client.load_peers() - approved_peers = [p.email for p in client._approved_peers] + approved_peers = [p.email for p in client.peer_manager.approved_peers] approved_jobs = [] jobs = client.jobs diff --git a/packages/syft-rds/src/syft_rds/login.py b/packages/syft-rds/src/syft_rds/login.py new file mode 100644 index 00000000000..7850d4bf178 --- /dev/null +++ b/packages/syft-rds/src/syft_rds/login.py @@ -0,0 +1,102 @@ +"""Entry points for the Remote Data Science product. + + from syft_rds import login_do + rds_client = login_do(email, token_path) + rds_client.datasets + rds_client.jobs + +These build a composed ``SyftRDSClientConfig`` (which owns the sync + job + +dataset sub-configs), construct a self-contained ``SyftRDSClient`` from it +""" + +from __future__ import annotations + +from pathlib import Path + +from syft_client.sync.environments.environment import Environment +from syft_client.sync.login import _init_client_login, _resolve_login_params +from syft_client.sync.login_utils import handle_potential_version_mismatches_on_login +from syft_client.sync.utils.syftbox_utils import check_env + +from syft_rds.client import SyftRDSClient +from syft_rds.config import SyftRDSClientConfig + + +def _login( + *, + email: str | None, + token_path: str | Path | None, + sync: bool, + load_peers: bool, + skip_peer_on_patch_version_diff: bool | None, + has_do_role: bool, + has_ds_role: bool, +) -> SyftRDSClient: + """Shared RDS login. + + Mirrors the pre-split ``syft_client`` login flow: detect the environment, + resolve login params + """ + env = check_env() + email, token_path = _resolve_login_params(email, token_path) + handle_potential_version_mismatches_on_login(email, token_path) + + if env == Environment.COLAB: + config = SyftRDSClientConfig.for_colab( + email=email, + has_do_role=has_do_role, + has_ds_role=has_ds_role, + skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, + ) + else: + config = SyftRDSClientConfig.for_jupyter( + email=email, + has_do_role=has_do_role, + has_ds_role=has_ds_role, + token_path=Path(token_path) if token_path is not None else None, + skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, + ) + + client = SyftRDSClient.from_config(config) + _init_client_login(client.sync_engine, sync=sync, load_peers=load_peers) + return client + + +def login_do( + email: str | None = None, + token_path: str | Path | None = None, + *, + sync: bool = True, + load_peers: bool = True, + skip_peer_on_patch_version_diff: bool | None = None, +) -> SyftRDSClient: + """Log in as a Data Owner and return a self-contained RDS client.""" + return _login( + email=email, + token_path=token_path, + sync=sync, + load_peers=load_peers, + skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, + has_do_role=True, + has_ds_role=False, + ) + + +def login_ds( + email: str | None = None, + token_path: str | Path | None = None, + *, + sync: bool = True, + load_peers: bool = True, + skip_peer_on_patch_version_diff: bool | None = None, +) -> SyftRDSClient: + """Log in as a Data Scientist and return a self-contained RDS client.""" + return _login( + email=email, + token_path=token_path, + sync=sync, + load_peers=load_peers, + skip_peer_on_patch_version_diff=skip_peer_on_patch_version_diff, + has_do_role=False, + has_ds_role=True, + ) diff --git a/packages/syft-rds/tests/conftest.py b/packages/syft-rds/tests/conftest.py new file mode 100644 index 00000000000..ee302387819 --- /dev/null +++ b/packages/syft-rds/tests/conftest.py @@ -0,0 +1,23 @@ +"""Pytest configuration for the syft-rds test suite. + +Mirrors the top-level ``tests/conftest.py`` so RDS product tests behave the same +whether run here or as part of the full monorepo suite. +""" + +import os + +import pytest + + +@pytest.fixture(scope="session", autouse=True) +def disable_pre_sync_for_tests(): + """Disable PRE_SYNC by default for all tests (explicit sync control).""" + original_value = os.environ.get("PRE_SYNC") + os.environ["PRE_SYNC"] = "false" + + yield + + if original_value is not None: + os.environ["PRE_SYNC"] = original_value + else: + os.environ.pop("PRE_SYNC", None) diff --git a/packages/syft-rds/tests/dataset_test_utils.py b/packages/syft-rds/tests/dataset_test_utils.py new file mode 100644 index 00000000000..694abbe7afc --- /dev/null +++ b/packages/syft-rds/tests/dataset_test_utils.py @@ -0,0 +1,144 @@ +"""Shared helpers for the syft-rds product tests. + +Uniquely named (not ``utils.py``) to avoid a pytest module-name collision with +``tests/unit/utils.py`` when the full monorepo suite is collected together. +""" + +import random +from pathlib import Path + + +def create_tmp_dataset_files(): + tmp_dir = Path("/tmp/syft-datasets-testing") / str(random.randint(1, 1000000)) + tmp_dir.mkdir(parents=True, exist_ok=True) + mock_path = tmp_dir / "mock.txt" + private_path = tmp_dir / "private.txt" + readme_path = tmp_dir / "readme.md" + mock_path.write_text("Hello, world!") + private_path.write_text("Hello, world private!") + readme_path.write_text("Hello, world!") + return mock_path, private_path, readme_path + + +def create_tmp_dataset_files_with_parquet(): + """Create temporary dataset files with parquet files (binary format).""" + import pandas as pd + + tmp_dir = Path("/tmp/syft-datasets-testing") / str(random.randint(1, 1000000)) + tmp_dir.mkdir(parents=True, exist_ok=True) + + # Create parquet files (binary format) + mock_df = pd.DataFrame( + { + "id": [1, 2, 3, 4, 5], + "name": ["Alice", "Bob", "Charlie", "Diana", "Eve"], + "age": [25, 30, 35, 28, 32], + "score": [85.5, 90.0, 88.5, 92.0, 87.5], + } + ) + mock_path = tmp_dir / "mock_data.parquet" + mock_df.to_parquet(mock_path, index=False) + + private_df = pd.DataFrame( + { + "id": [1, 2, 3], + "sensitive_data": ["secret1", "secret2", "secret3"], + "value": [100, 200, 300], + } + ) + private_path = tmp_dir / "private_data.parquet" + private_df.to_parquet(private_path, index=False) + + readme_path = tmp_dir / "readme.md" + readme_path.write_text( + "# Dataset with Parquet Files\n\nThis dataset contains parquet files." + ) + + return mock_path, private_path, readme_path + + +def create_test_project_folder( + with_pyproject: bool = False, + multiplier: int = 2, + prefix: str = "test_project_", +) -> Path: + """Create a test project folder with helpers package and main.py. + + Creates folder structure: + project_dir/ + ├── pyproject.toml # only if with_pyproject=True + ├── main.py # entrypoint, imports from helpers.helper + └── helpers/ + ├── __init__.py # package marker + └── helper.py # contains process_data() and get_multiplier() + + Args: + with_pyproject: If True, creates pyproject.toml in the folder + multiplier: Value returned by get_multiplier() in helper.py + prefix: Prefix for the temp directory name + + Returns: + Path to the created project directory + """ + import tempfile + + project_dir = Path(tempfile.mkdtemp(prefix=prefix)) + + # Create pyproject.toml if requested + if with_pyproject: + pyproject_path = project_dir / "pyproject.toml" + pyproject_path.write_text(""" +[project] +name = "test-project" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [] +""") + + # Create nested helpers package + helpers_dir = project_dir / "helpers" + helpers_dir.mkdir(parents=True) + + # Create __init__.py to make it a package + init_path = helpers_dir / "__init__.py" + init_path.write_text("# helpers package\n") + + # Create helper module + helper_path = helpers_dir / "helper.py" + helper_path.write_text(f''' +def process_data(data): + """Helper function to process data.""" + return f"Processed: {{data}}" + +def get_multiplier(): + return {multiplier} +''') + + # Create main.py that imports from nested helpers package + main_path = project_dir / "main.py" + main_path.write_text(""" +import json +import syft_client as sc +from helpers.helper import process_data, get_multiplier + +# Read data from dataset +data_path = sc.resolve_dataset_file_path("my dataset") + +with open(data_path, "r") as data_file: + data = data_file.read() + +# Use helper functions +processed = process_data(data) +multiplier = get_multiplier() + +result = { + "original": data, + "processed": processed, + "multiplier": multiplier +} + +with open("outputs/result.json", "w") as f: + f.write(json.dumps(result)) +""") + + return project_dir diff --git a/packages/syft-rds/tests/test_client.py b/packages/syft-rds/tests/test_client.py new file mode 100644 index 00000000000..46691542ba8 --- /dev/null +++ b/packages/syft-rds/tests/test_client.py @@ -0,0 +1,181 @@ +"""Tests for the self-contained SyftRDSClient product.""" + +from syft_rds import SyftRDSClient + + +def test_rds_layer_supplies_collection_specs(): + """The composed DS sync engine's watcher cache received the dataset spec + from the RDS layer (the RDS -> generic-engine spec-injection seam).""" + from syft_datasets.dataset_manager import DATASET_COLLECTION_PREFIX + + ds, do = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False + ) + + watcher_cache = ds.sync_engine.datasite_watcher_syncer.datasite_watcher_cache + specs = watcher_cache.collection_specs + + assert len(specs) > 0 + assert any(spec.prefix == DATASET_COLLECTION_PREFIX for spec in specs) + + +def test_dataset_creation_and_sync(): + """Datasets created by the DO are visible to the DS via mock drive.""" + from dataset_test_utils import create_tmp_dataset_files + + ds, do = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + ) + + mock_path, private_path, readme_path = create_tmp_dataset_files() + do.create_dataset( + name="mock drive dataset", + mock_path=mock_path, + private_path=private_path, + summary="Test dataset via mock drive", + readme_path=readme_path, + tags=["test"], + users=[ds.email], + ) + + assert len(do.datasets.get_all()) == 1 + + ds.sync() + + assert len(ds.datasets.get_all()) == 1 + dataset = ds.datasets.get("mock drive dataset", datasite=do.email) + assert dataset is not None + assert len(dataset.mock_files) > 0 + + +def test_delete_unversioned_state_removes_dataset_collections(): + """delete_unversioned_state clears both dataset collection folders.""" + from syft_datasets.dataset_manager import ( + DATASET_COLLECTION_PREFIX, + PRIVATE_DATASET_COLLECTION_PREFIX, + ) + from dataset_test_utils import create_tmp_dataset_files + + def query(conn, name_contains): + results = ( + conn.drive_service.files() + .list( + q=f"name contains '{name_contains}' and trashed=false", + fields="files(id, name)", + ) + .execute() + ) + return results.get("files", []) + + ds, do = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + sync_automatically=False, + encryption=True, + ) + + mock_path, private_path, readme_path = create_tmp_dataset_files() + do.create_dataset( + name="my dataset", + mock_path=mock_path, + private_path=private_path, + summary="Test", + readme_path=readme_path, + users=[ds.email], + upload_private=True, + ) + do.sync() + + conn = do.peer_manager.connection_router.connections[0] + assert len(query(conn, DATASET_COLLECTION_PREFIX)) > 0 + assert len(query(conn, PRIVATE_DATASET_COLLECTION_PREFIX)) > 0 + + conn.delete_unversioned_state() + + assert len(query(conn, DATASET_COLLECTION_PREFIX)) == 0 + assert len(query(conn, PRIVATE_DATASET_COLLECTION_PREFIX)) == 0 + + +def test_dir_returns_only_public_api(): + _ds, do = SyftRDSClient.pair_with_mock_drive_service_connection() + + public_names = dir(do) + + # Identity + sync surface + assert "email" in public_names + assert "sync" in public_names + assert "peers" in public_names + assert "add_peer" in public_names + + # Composed engine and managers + assert "sync_engine" in public_names + assert "job_client" in public_names + assert "dataset_manager" in public_names + + # Dataset/job surface + assert "jobs" in public_names + assert "datasets" in public_names + assert "create_dataset" in public_names + assert "submit_python_job" in public_names + assert "submit_bash_job" in public_names + assert "process_approved_jobs" in public_names + + # Hides Pydantic internals + assert "model_dump" not in public_names + assert "model_fields" not in public_names + assert "model_validate" not in public_names + + # Hides internal helpers + assert "model_post_init" not in public_names + assert "from_config" not in public_names + + # Hidden attributes are still accessible + assert do.email is not None + assert callable(do.model_dump) + + +def test_encrypted_dataset_collection_syncs(): + """Dataset-collection sync-down works under encryption.""" + from dataset_test_utils import create_tmp_dataset_files + from syft_datasets.dataset_manager import DATASET_COLLECTION_PREFIX + + ds, do = SyftRDSClient.pair_with_mock_drive_service_connection( + encryption=True, + ) + + mock_path, private_path, readme_path = create_tmp_dataset_files() + do.create_dataset( + name="demo", + mock_path=mock_path, + private_path=private_path, + summary="demo dataset", + readme_path=readme_path, + users=[ds.email], + upload_private=True, + sync=False, + ) + do.sync() + + ds.sync() + + cr = ds.peer_manager.connection_router + collections = cr.watcher_list_collections(DATASET_COLLECTION_PREFIX) + do_collections = [c for c in collections if c["owner_email"] == do.email] + assert do_collections, "DS does not see the DO's dataset collection" + + c = do_collections[0] + files = cr.watcher_download_collection( + DATASET_COLLECTION_PREFIX, c["tag"], c["content_hash"], do.email + ) + assert files, "DS could not download the dataset collection files" + + +def test_collection_prefixes_match_syft_datasets(): + """The sync core mirrors the prefixes rather than importing the domain.""" + from syft_client.sync.connections import collection_prefixes as core + from syft_datasets import dataset_manager as domain + + assert core.DATASET_COLLECTION_PREFIX == domain.DATASET_COLLECTION_PREFIX + assert ( + core.PRIVATE_DATASET_COLLECTION_PREFIX + == domain.PRIVATE_DATASET_COLLECTION_PREFIX + ) diff --git a/tests/unit/test_create_dataset_cleanup.py b/packages/syft-rds/tests/test_create_dataset_cleanup.py similarity index 93% rename from tests/unit/test_create_dataset_cleanup.py rename to packages/syft-rds/tests/test_create_dataset_cleanup.py index e1f89401e8e..a016b24d9db 100644 --- a/tests/unit/test_create_dataset_cleanup.py +++ b/packages/syft-rds/tests/test_create_dataset_cleanup.py @@ -6,14 +6,16 @@ import pytest from syft_client.sync.syftbox_manager import SyftboxManager -from tests.unit.utils import create_tmp_dataset_files +from syft_rds import SyftRDSClient +from syft_datasets.dataset_manager import DATASET_COLLECTION_PREFIX +from dataset_test_utils import create_tmp_dataset_files class TestCreateDatasetCleanup: """Tests that create_dataset cleans up partial state on failure.""" def _make_do_manager(self): - _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + _, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) return do_manager @@ -95,7 +97,9 @@ def test_cleanup_on_private_upload_failure(self): ) # Mock GDrive folder should have been cleaned up too - collections = do_manager._connection_router.owner_list_dataset_collections() + collections = do_manager.sync_engine._connection_router.owner_list_collections( + DATASET_COLLECTION_PREFIX + ) assert "testdataset" not in collections def test_cleanup_on_sync_failure(self): @@ -147,9 +151,7 @@ def test_cleanup_logs_warning_on_failure(self, caplog): side_effect=OSError("delete broken"), ), ): - with caplog.at_level( - logging.WARNING, logger="syft_client.sync.syftbox_manager" - ): + with caplog.at_level(logging.WARNING, logger="syft_rds.client"): with pytest.raises(RuntimeError, match="upload failed"): do_manager.create_dataset(**self._dataset_kwargs()) diff --git a/tests/unit/test_dataset_upload_private.py b/packages/syft-rds/tests/test_dataset_upload_private.py similarity index 76% rename from tests/unit/test_dataset_upload_private.py rename to packages/syft-rds/tests/test_dataset_upload_private.py index c877f9d824d..659de3034cc 100644 --- a/tests/unit/test_dataset_upload_private.py +++ b/packages/syft-rds/tests/test_dataset_upload_private.py @@ -1,14 +1,17 @@ from syft_client.sync.connections.drive.gdrive_transport import GDriveConnection -from syft_client.sync.syftbox_manager import SyftboxManager -from syft_datasets.dataset_manager import PRIVATE_DATASET_COLLECTION_PREFIX -from tests.unit.utils import create_tmp_dataset_files +from syft_rds import SyftRDSClient +from syft_datasets.dataset_manager import ( + DATASET_COLLECTION_PREFIX, + PRIVATE_DATASET_COLLECTION_PREFIX, +) +from dataset_test_utils import create_tmp_dataset_files class TestDatasetUploadPrivate: def test_ds_cannot_see_private_data(self): """When DO creates a dataset with upload_private=True, DS should see mock data but NOT private data.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) @@ -36,14 +39,14 @@ def test_ds_cannot_see_private_data(self): assert len(dataset.mock_files) > 0 # DS should NOT see any private collections via the connection - private_collections = ( - ds_manager._connection_router.owner_list_private_dataset_collections() + private_collections = ds_manager.sync_engine._connection_router.owner_list_all_collections_with_permissions( + PRIVATE_DATASET_COLLECTION_PREFIX ) assert len(private_collections) == 0 # Verify private collection folder exists on GDrive (visible to DO) - do_private_collections = ( - do_manager._connection_router.owner_list_private_dataset_collections() + do_private_collections = do_manager.sync_engine._connection_router.owner_list_all_collections_with_permissions( + PRIVATE_DATASET_COLLECTION_PREFIX ) assert len(do_private_collections) == 1 assert do_private_collections[0].tag == "testdataset" @@ -51,7 +54,7 @@ def test_ds_cannot_see_private_data(self): def test_do_cold_start_restores_private_data(self): """When DO creates a dataset with upload_private=True, then loses local data, a fresh sync should restore the private files.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) @@ -79,7 +82,7 @@ def test_do_cold_start_restores_private_data(self): shutil.rmtree(private_dir) assert not private_dir.exists() - do_manager.datasite_owner_syncer.initial_sync_done = False + do_manager.sync_engine.datasite_owner_syncer.initial_sync_done = False # Sync again — should restore private data from GDrive do_manager.sync() @@ -91,7 +94,7 @@ def test_do_cold_start_restores_private_data(self): def test_default_behavior_no_private_upload(self): """When upload_private is not set, no private collection should be created.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) @@ -107,20 +110,22 @@ def test_default_behavior_no_private_upload(self): ) # No private collection should exist - private_collections = ( - do_manager._connection_router.owner_list_private_dataset_collections() + private_collections = do_manager.sync_engine._connection_router.owner_list_all_collections_with_permissions( + PRIVATE_DATASET_COLLECTION_PREFIX ) assert len(private_collections) == 0 # Mock collection should still exist mock_collections = ( - do_manager._connection_router.owner_list_dataset_collections() + do_manager.sync_engine._connection_router.owner_list_collections( + DATASET_COLLECTION_PREFIX + ) ) assert "testdataset" in mock_collections def test_ds_cannot_find_private_folders_via_gdrive_query(self): """DS searching GDrive directly for private collection prefix should find nothing.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) @@ -137,7 +142,9 @@ def test_ds_cannot_find_private_folders_via_gdrive_query(self): ) # Get the DS's raw GDrive connection and search for private folders - ds_connection: GDriveConnection = ds_manager._connection_router.connections[0] + ds_connection: GDriveConnection = ( + ds_manager.sync_engine._connection_router.connections[0] + ) results = ( ds_connection.drive_service.files() .list( @@ -152,7 +159,9 @@ def test_ds_cannot_find_private_folders_via_gdrive_query(self): assert len(results.get("files", [])) == 0 # DO should find it with the same query - do_connection: GDriveConnection = do_manager._connection_router.connections[0] + do_connection: GDriveConnection = ( + do_manager.sync_engine._connection_router.connections[0] + ) do_results = ( do_connection.drive_service.files() .list( diff --git a/tests/unit/test_datasets_jobs_repr.py b/packages/syft-rds/tests/test_datasets_jobs_repr.py similarity index 93% rename from tests/unit/test_datasets_jobs_repr.py rename to packages/syft-rds/tests/test_datasets_jobs_repr.py index 42dd28b8f9d..8d63b88cfca 100644 --- a/tests/unit/test_datasets_jobs_repr.py +++ b/packages/syft-rds/tests/test_datasets_jobs_repr.py @@ -1,15 +1,15 @@ """Tests for SyftDatasetManager and JobsList repr and indexing.""" import pytest -from syft_client.sync.syftbox_manager import SyftboxManager -from syft_job.job import JobInfo, JobsList -from tests.unit.utils import create_tmp_dataset_files +from syft_rds import SyftRDSClient +from syft_job.job import JobInfo, JobsList +from dataset_test_utils import create_tmp_dataset_files def _create_manager_with_dataset(): """Create a pair of managers and a dataset, return (ds_manager, do_manager, dataset).""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -76,7 +76,7 @@ def test_dataset_manager_repr_html(): def test_dataset_manager_repr_html_with_tags(): - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -92,7 +92,7 @@ def test_dataset_manager_repr_html_with_tags(): def test_dataset_manager_repr_html_empty(): - _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + _, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -116,7 +116,7 @@ def test_dataset_manager_get_missing_lists_available(): def test_dataset_manager_get_missing_no_datasets(): - _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + _, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) diff --git a/packages/syft-rds/tests/test_install_source_advertise.py b/packages/syft-rds/tests/test_install_source_advertise.py new file mode 100644 index 00000000000..df43839a605 --- /dev/null +++ b/packages/syft-rds/tests/test_install_source_advertise.py @@ -0,0 +1,80 @@ +"""DS JobClient picks up the DO's advertised syft-client install source.""" + +from __future__ import annotations + +import warnings +from pathlib import Path + +import pytest + +from syft_rds import SyftRDSClient + + +@pytest.fixture(autouse=True) +def _clear_install_source_cache(): + """get_syft_client_install_source uses lru_cache; clear so env-var patches work.""" + from syft_job.install_source import get_syft_client_install_source + + get_syft_client_install_source.cache_clear() + yield + get_syft_client_install_source.cache_clear() + + +class TestEndToEndPeerInstallSourcePropagation: + def test_ds_job_client_learns_do_install_source(self, monkeypatch): + monkeypatch.setenv("SYFT_CLIENT_INSTALL_SOURCE", "/do/local/syft-client") + + ds, do = SyftRDSClient.pair_with_mock_drive_service_connection() + + assert do.email in ds.job_client.peer_install_sources + assert ds.job_client.peer_install_sources[do.email] == "/do/local/syft-client" + + def test_submit_python_job_bakes_do_source_into_run_sh(self, monkeypatch, tmp_path): + monkeypatch.setenv("SYFT_CLIENT_INSTALL_SOURCE", "/do/local/syft-client") + + ds, do = SyftRDSClient.pair_with_mock_drive_service_connection() + + # Minimal job: a single Python file + code = tmp_path / "main.py" + code.write_text("print('hello')\n") + + job_dir = ds.job_client.submit_python_job( + user=do.email, code_path=str(code), job_name="my-test-job" + ) + + run_sh = (Path(job_dir) / "run.sh").read_text() + # The DO's advertised source must appear in the uv pip install line, + # and the DS's local source must NOT be substituted instead. + assert "/do/local/syft-client" in run_sh + assert "uv pip install" in run_sh + + +class TestFallbackWhenDoDidNotAdvertise: + def test_falls_back_and_warns_when_peer_has_no_source(self, monkeypatch, tmp_path): + monkeypatch.setenv("SYFT_CLIENT_INSTALL_SOURCE", "/ds/local/syft-client") + + ds, do = SyftRDSClient.pair_with_mock_drive_service_connection() + + # Simulate an older DO: clear the advertised source from DS's JobClient. + ds.job_client.peer_install_sources.pop(do.email, None) + + code = tmp_path / "main.py" + code.write_text("print('hello')\n") + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + job_dir = ds.job_client.submit_python_job( + user=do.email, + code_path=str(code), + job_name="fallback-job", + ) + + # A prominent warning must have been emitted. + messages = [str(w.message) for w in caught] + assert any("No syft-client install source advertised" in m for m in messages), ( + f"Expected fallback warning, got: {messages}" + ) + + # The DS's local detection result should be used as the fallback. + run_sh = (Path(job_dir) / "run.sh").read_text() + assert "/ds/local/syft-client" in run_sh diff --git a/tests/unit/test_job_auto_approval.py b/packages/syft-rds/tests/test_job_auto_approval.py similarity index 91% rename from tests/unit/test_job_auto_approval.py rename to packages/syft-rds/tests/test_job_auto_approval.py index 342cd8ee754..b83e5444bef 100644 --- a/tests/unit/test_job_auto_approval.py +++ b/packages/syft-rds/tests/test_job_auto_approval.py @@ -10,8 +10,8 @@ from syft_bg.api import auto_approve_job from syft_bg.approve.config import AutoApproveConfig from syft_bg.common.config import get_default_paths -from syft_client.job_auto_approval import auto_approve_and_run_jobs -from syft_client.sync.syftbox_manager import SyftboxManager +from syft_rds import SyftRDSClient +from syft_rds.job_auto_approval import auto_approve_and_run_jobs @contextmanager @@ -41,7 +41,7 @@ def test_auto_approve_and_run_jobs(): - The exact Python script content (newline agnostic) - Exactly the required files (no more, no less) """ - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -134,7 +134,7 @@ def _create_project_dir(script_content="print('hello')\n", data_content='{"k": " def test_auto_approve_job_default_all_content_matched(): """Default behavior: all files are content-matched.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -156,7 +156,7 @@ def test_auto_approve_job_default_all_content_matched(): def test_auto_approve_job_file_paths_only(): """file_paths specified: those are name-only, rest are content-matched.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -175,7 +175,7 @@ def test_auto_approve_job_file_paths_only(): def test_auto_approve_job_contents_only(): """contents specified: only those files are content-matched, rest ignored.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -194,7 +194,7 @@ def test_auto_approve_job_contents_only(): def test_auto_approve_job_both_contents_and_file_paths(): """Both specified: contents are content-matched, file_paths are name-only.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -213,7 +213,7 @@ def test_auto_approve_job_both_contents_and_file_paths(): def test_auto_approve_job_overlap_error(): """Overlap between contents and file_paths should fail.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -227,7 +227,7 @@ def test_auto_approve_job_overlap_error(): def test_auto_approve_job_file_not_found_error(): """Referencing a non-existent file should fail.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -241,7 +241,7 @@ def test_auto_approve_job_file_not_found_error(): def test_auto_approve_job_nested_directory(): """Files in subdirectories are stored with relative paths.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -272,7 +272,7 @@ def test_auto_approve_job_nested_directory(): def test_auto_approve_job_default_no_special_treatment_for_non_params_json(): """Default behavior with non-params.json files: all are content-matched.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) @@ -294,7 +294,7 @@ def test_auto_approve_job_default_no_special_treatment_for_non_params_json(): def test_auto_approve_job_default_params_json_is_name_only(): """Default behavior: params.json is automatically name-only.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) diff --git a/tests/unit/test_load_dataset_code.py b/packages/syft-rds/tests/test_load_dataset_code.py similarity index 89% rename from tests/unit/test_load_dataset_code.py rename to packages/syft-rds/tests/test_load_dataset_code.py index d8efa55c271..ec2c1cc1192 100644 --- a/tests/unit/test_load_dataset_code.py +++ b/packages/syft-rds/tests/test_load_dataset_code.py @@ -5,7 +5,7 @@ import pytest import syft_client as sc -from syft_client.sync.syftbox_manager import SyftboxManager +from syft_rds import SyftRDSClient def _create_tmp_code_folder() -> Path: @@ -49,7 +49,7 @@ def _create_local_dataset(do_manager, name: str) -> None: def test_load_dataset_code_top_level_and_nested(): - _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + _, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) _create_local_dataset(do_manager, "my_code_dataset") @@ -68,7 +68,7 @@ def test_load_dataset_code_top_level_and_nested(): def test_load_dataset_code_explicit_module_name(): - _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + _, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) _create_local_dataset(do_manager, "my_code_dataset") @@ -82,7 +82,7 @@ def test_load_dataset_code_explicit_module_name(): def test_load_dataset_code_invalid_path_raises(): - _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + _, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) with pytest.raises(ValueError, match="Invalid path"): @@ -90,7 +90,7 @@ def test_load_dataset_code_invalid_path_raises(): def test_load_dataset_code_missing_file_raises(): - _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + _, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) _create_local_dataset(do_manager, "my_code_dataset") diff --git a/packages/syft-rds/tests/test_sync_manager.py b/packages/syft-rds/tests/test_sync_manager.py new file mode 100644 index 00000000000..49757bba562 --- /dev/null +++ b/packages/syft-rds/tests/test_sync_manager.py @@ -0,0 +1,1458 @@ +"""Dataset and job flows over the mock drive connection.""" + +from pathlib import Path +import json +import shutil +import tempfile + +import pytest + +from syft_client.sync.connections.drive import mock_drive_service +from syft_client.sync.connections.drive.gdrive_transport import GDriveConnection +from syft_datasets import Dataset +from syft_datasets.dataset_manager import DATASET_COLLECTION_PREFIX +from syft_rds import SyftRDSClient +from syft_rds.config import COLLECTION_SUBPATH, SyftRDSClientConfig +from dataset_test_utils import ( + create_test_project_folder, + create_tmp_dataset_files, + create_tmp_dataset_files_with_parquet, +) + + +def path_for_job(do_email: str, ds_email: str, filename: str = "test.job") -> str: + """Return the correct path for DS to submit a job file to DO.""" + return f"{do_email}/app_data/job/inbox/{ds_email}/{filename}" + + +def test_datasets(): + """Test dataset creation and sync between DO and DS.""" + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False + ) + + mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() + + # Create dataset with specific users + do_manager.create_dataset( + name="my dataset", + mock_path=mock_dset_path, + private_path=private_dset_path, + summary="This is a summary", + readme_path=readme_path, + tags=["tag1", "tag2"], + users=[ds_manager.email], # Share with specific user + ) + + # Verify collection created + collections = do_manager.sync_engine._connection_router.owner_list_collections( + DATASET_COLLECTION_PREFIX + ) + assert "my dataset" in collections + + datasets = do_manager.datasets.get_all() + assert len(datasets) == 1 + + # Retrieve dataset by name + dataset_do = do_manager.datasets["my dataset"] + assert isinstance(dataset_do, Dataset) + assert len(dataset_do.private_files) > 0 + assert len(dataset_do.mock_files) > 0 + + ds_manager.sync() + + # Verify DS can see collection + ds_collections = ds_manager.sync_engine._connection_router.watcher_list_collections( + DATASET_COLLECTION_PREFIX + ) + assert any(c["tag"] == "my dataset" for c in ds_collections) + + assert len(ds_manager.datasets.get_all()) == 1 + + dataset_ds = ds_manager.datasets.get("my dataset", datasite=do_manager.email) + + assert dataset_ds.mock_files[0].exists() + + mock_content_ds = (dataset_ds.mock_dir / "mock.txt").read_text() + assert len(mock_content_ds) > 0 + + # test getting it via resolve path + from syft_client import resolve_dataset_file_path + + mock_file_path = resolve_dataset_file_path("my dataset", client=ds_manager) + assert mock_file_path.exists() + + mock_content_ds = mock_file_path.read_text() + assert len(mock_content_ds) > 0 + + def has_file(root_dir, filename): + return any(p.name == filename for p in Path(root_dir).rglob("*")) + + assert has_file(ds_manager.syftbox_folder, "mock.txt") + assert not has_file(ds_manager.syftbox_folder, "private.txt") + # Confirm that "private.txt" does not exist anywhere in the DS syftbox folder + for path in Path(ds_manager.syftbox_folder).rglob("*"): + assert path.name != "private.txt" + + +def test_datasets_with_parquet(): + """Test dataset creation and sync with parquet files (binary format).""" + import pandas as pd + + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False + ) + + mock_dset_path, private_dset_path, readme_path = ( + create_tmp_dataset_files_with_parquet() + ) + + # This should work without errors even though parquet files are binary + do_manager.create_dataset( + name="parquet dataset", + mock_path=mock_dset_path, + private_path=private_dset_path, + summary="This is a dataset with parquet files", + readme_path=readme_path, + tags=["parquet", "binary"], + users=[ds_manager.email], + ) + + datasets = do_manager.datasets.get_all() + assert len(datasets) == 1 + + # Dataset files are synced via collections, not the event log. + # Only the permission file (syft.pub.yaml) should appear as an event. + cached_events = ( + do_manager.sync_engine.datasite_owner_syncer.event_cache.get_cached_events() + ) + assert all(str(e.path_in_datasite).endswith("syft.pub.yaml") for e in cached_events) + + # Retrieve dataset by name + dataset_do = do_manager.datasets["parquet dataset"] + assert isinstance(dataset_do, Dataset) + assert len(dataset_do.private_files) > 0 + assert len(dataset_do.mock_files) > 0 + + # Verify parquet files are present + mock_files = [f.name for f in dataset_do.mock_files] + assert "mock_data.parquet" in mock_files + + private_files = [f.name for f in dataset_do.private_files] + assert "private_data.parquet" in private_files + + # Sync to datasite + ds_manager.sync() + + assert len(ds_manager.datasets.get_all()) == 1 + + dataset_ds = ds_manager.datasets.get("parquet dataset", datasite=do_manager.email) + + # Verify the parquet file exists and can be read + mock_parquet_path = dataset_ds.mock_dir / "mock_data.parquet" + assert mock_parquet_path.exists() + + # Verify we can read the parquet file back + df = pd.read_parquet(mock_parquet_path) + assert len(df) == 5 + assert "name" in df.columns + assert "age" in df.columns + + def has_file(root_dir, filename): + return any(p.name == filename for p in Path(root_dir).rglob("*")) + + assert has_file(ds_manager.syftbox_folder, "mock_data.parquet") + assert not has_file(ds_manager.syftbox_folder, "private_data.parquet") + + +def test_dataset_empty_permissions_no_access(): + """Test that empty permissions list means no one can access the dataset collection.""" + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False + ) + + mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() + + # Create dataset with empty permissions list (share with no one) + do_manager.create_dataset( + name="private dataset", + mock_path=mock_dset_path, + private_path=private_dset_path, + summary="This is a private summary", + readme_path=readme_path, + tags=["private"], + users=[], # Empty list - no one has access + ) + + # Verify collection created + collections = do_manager.sync_engine._connection_router.owner_list_collections( + DATASET_COLLECTION_PREFIX + ) + assert "private dataset" in collections + + # DO should be able to see their own dataset + datasets = do_manager.datasets.get_all() + assert len(datasets) == 1 + + # DS syncs + ds_manager.sync() + + # DS should NOT see the collection (no permissions) + ds_collections = ds_manager.sync_engine._connection_router.watcher_list_collections( + DATASET_COLLECTION_PREFIX + ) + assert not any(c["tag"] == "private dataset" for c in ds_collections) + + # DS should not have downloaded any datasets + assert len(ds_manager.datasets.get_all()) == 0 + + +def test_dataset_only_mock_data_uploaded(): + """Test that only mock data is uploaded to the collection, not private data.""" + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False + ) + + mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() + + do_manager.create_dataset( + name="test dataset", + mock_path=mock_dset_path, + private_path=private_dset_path, + summary="Test summary", + readme_path=readme_path, + tags=["test"], + users=[ds_manager.email], + ) + + # Sync so DS receives the dataset + ds_manager.sync() + + files = ds_manager.sync_engine._connection_router.connections[ + 0 + ].drive_service._backing_store.files + list(files.keys()) + file_objs = list(files.values()) + file_objs_ex_dataset_yaml = [ + file_obj for file_obj in file_objs if file_obj.name != "dataset.yaml" + ] + + assert not any("private" in file_obj.name for file_obj in file_objs) + # dataset.yaml does mention "private", but thats just the path + assert not any( + b"private" in file_obj.content for file_obj in file_objs_ex_dataset_yaml + ) + + assert any("mock" in file_obj.name for file_obj in file_objs) + assert any(b"Hello, world" in file_obj.content for file_obj in file_objs) + + mock_file = next(file_obj for file_obj in file_objs if file_obj.name == "mock.txt") + + # Verify mock content is correct + mock_content = mock_file.content.decode("utf-8") + assert len(mock_content) > 0, "Mock file should have content" + assert "Hello" in mock_content, "Mock file should contain expected data" + + +def test_jobs(): + """Test basic job submission, approval, execution, and result sync.""" + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + sync_automatically=False, + ) + + test_py_path = "/tmp/test.py" + with open(test_py_path, "w") as f: + f.write(""" +with open("outputs/result.json", "w") as f: + f.write('{"result": 1}') +""") + + ds_manager.submit_python_job( + user=do_manager.email, + code_path=test_py_path, + job_name="test.job", + ) + + # We want to make sure that we only send one message for the multiple files in the job. + # this is to reduce the number of messages sent, which increases the speed of sync + # we do this by not always syncing on a file change, currently this logic is a bit of + # a short cut, but we could do this based on timing eventually (if there are items in the + # queue for longer than a certain time we start pushing) + connection_do = do_manager.sync_engine._connection_router.connections[0] + inbox_folder_id = connection_do._get_own_datasite_inbox_id(ds_manager.email) + inbox_file_metadatas = connection_do.get_file_metadatas_from_folder(inbox_folder_id) + assert len(inbox_file_metadatas) == 1 + + do_manager.sync() + + assert len(do_manager.job_client.jobs) == 1 + job = do_manager.job_client.jobs[0] + + job.approve() + + do_manager.job_runner.process_approved_jobs() + do_manager.job_runner.share_job_results( + "test.job", share_outputs=True, share_logs=False + ) + + do_manager.sync() + + ds_manager.sync() + + output_path = ds_manager.job_client.jobs[-1].output_paths[0] + with open(output_path, "r") as f: + json_content = json.loads(f.read()) + + assert json_content["result"] == 1 + + +def test_jobs_with_dataset(): + """Test job execution with dataset access using syft:// protocol.""" + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + sync_automatically=False, + ) + + mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() + + do_manager.create_dataset( + name="my dataset", + mock_path=mock_dset_path, + private_path=private_dset_path, + summary="This is a summary", + readme_path=readme_path, + tags=["tag1", "tag2"], + users=[ds_manager.email], + ) + do_manager.sync() + + ds_manager.sync() + assert len(ds_manager.datasets.get_all()) == 1 + + dataset_ds = ds_manager.datasets.get("my dataset", datasite=do_manager.email) + assert dataset_ds.mock_files[0].exists() + import syft_client as sc + + assert ( + sc.resolve_dataset_file_path("my dataset", client=ds_manager) + == dataset_ds.mock_files[0] + ) + + test_py_path = "/tmp/test.py" + with open(test_py_path, "w") as f: + f.write(""" +import syft_client as sc +import json + +data_path = sc.resolve_dataset_file_path("my dataset") +with open(data_path, "r") as f: + data = f.read() +result = {"result": len(data)} +with open("outputs/result.json", "w") as f: + f.write(json.dumps(result)) +""") + + ds_manager.submit_python_job( + user=do_manager.email, + code_path=test_py_path, + job_name="test.job", + ) + + do_manager.sync() + assert len(do_manager.job_client.jobs) == 1 + job = do_manager.job_client.jobs[0] + + job.approve() + + do_manager.job_runner.process_approved_jobs() + do_manager.job_runner.share_job_results( + "test.job", share_outputs=True, share_logs=False + ) + + do_manager.sync() + + ds_manager.sync() + + output_path = ds_manager.job_client.jobs[-1].output_paths[0] + with open(output_path, "r") as f: + json_content = json.loads(f.read()) + + with open(private_dset_path, "r") as f: + private_data_length = len(f.read()) + + assert json_content["result"] == private_data_length + + +def test_single_file_job_submission_without_pyproject(): + """Test that code files are placed in job_dir/code/ subdirectory. + + Verifies code is at: + job_dir/code/main.py + """ + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + sync_automatically=False, + ) + + # Test with single file submission + test_py_path = "/tmp/test_direct_copy.py" + with open(test_py_path, "w") as f: + f.write('print("hello")') + + ds_manager.submit_python_job( + user=do_manager.email, + code_path=test_py_path, + job_name="test.direct.copy", + ) + + do_manager.sync() + + assert len(do_manager.job_client.jobs) == 1 + job = do_manager.job_client.jobs[0] + job_dir = job.job_submission_path + + # Verify code is in job_dir/code/ + assert (job_dir / "code" / "test_direct_copy.py").exists(), ( + "Code should be in job_dir/code/" + ) + assert (job_dir / "run.sh").exists(), "run.sh should exist" + assert (job_dir / "config.yaml").exists(), "config.yaml should exist" + + +def test_folder_job_submission_without_pyproject(): + """Test folder submission without pyproject.toml uses uv venv + uv pip install. + + Verifies: + - Folder without pyproject.toml works + - Folder is preserved with its name (not dumped at root) + - Generated run.sh uses 'uv venv' (not 'uv sync') + - Entrypoint path includes folder name + """ + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + sync_automatically=False, + ) + + # Create a folder without pyproject.toml + project_dir = tempfile.mkdtemp(prefix="test_no_pyproject_") + Path(project_dir).name + + try: + # Create main.py + main_path = Path(project_dir) / "main.py" + with open(main_path, "w") as f: + f.write(""" +with open("outputs/result.txt", "w") as f: + f.write("success") +""") + + # Create a helper module + helper_path = Path(project_dir) / "helper.py" + with open(helper_path, "w") as f: + f.write("VALUE = 42\n") + + # Submit folder (no pyproject.toml) + ds_manager.submit_python_job( + user=do_manager.email, + code_path=project_dir, + job_name="test.no.pyproject", + entrypoint="main.py", + ) + + do_manager.sync() + + assert len(do_manager.job_client.jobs) == 1 + job = do_manager.job_client.jobs[0] + job_dir = job.job_submission_path + + # Verify folder structure - code is in code/ subdirectory + assert (job_dir / "code").exists(), "code/ should exist in job_dir" + assert (job_dir / "code" / "main.py").exists(), "main.py should be inside code/" + assert (job_dir / "code" / "helper.py").exists(), ( + "helper.py should be inside code/" + ) + assert (job_dir / "run.sh").exists(), "run.sh should be at job_dir root" + assert (job_dir / "config.yaml").exists(), ( + "config.yaml should be at job_dir root" + ) + + # Verify run.sh uses uv venv (not uv sync) and correct entrypoint path + run_script = (job_dir / "run.sh").read_text() + assert "uv venv" in run_script, ( + "Should use 'uv venv' for folders without pyproject.toml" + ) + assert "uv sync" not in run_script, ( + "Should NOT use 'uv sync' without pyproject.toml" + ) + assert "cd code" in run_script, "Should cd into code folder" + assert "python main.py" in run_script, "Should run main.py from code/" + + finally: + shutil.rmtree(project_dir, ignore_errors=True) + + +def test_folder_job_submission_with_pyproject(): + """Test folder submission with pyproject.toml uses uv sync. + + Verifies: + - Folder is preserved with its name inside job_dir + - pyproject.toml is inside the folder, not at job_dir root + - run.sh uses 'uv sync' inside the folder + - Entrypoint path includes folder name + """ + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + sync_automatically=False, + ) + + # Create a folder with pyproject.toml + project_dir = tempfile.mkdtemp(prefix="test_with_pyproject_") + Path(project_dir).name + + try: + # Create pyproject.toml + pyproject_path = Path(project_dir) / "pyproject.toml" + with open(pyproject_path, "w") as f: + f.write(""" +[project] +name = "test-project" +version = "0.1.0" +dependencies = [] +""") + + # Create main.py + main_path = Path(project_dir) / "main.py" + with open(main_path, "w") as f: + f.write('print("hello from pyproject project")') + + # Submit folder (with pyproject.toml) + ds_manager.submit_python_job( + user=do_manager.email, + code_path=project_dir, + job_name="test.with.pyproject", + entrypoint="main.py", + ) + + do_manager.sync() + + assert len(do_manager.job_client.jobs) == 1 + job = do_manager.job_client.jobs[0] + job_dir = job.job_submission_path + + # Verify folder structure - code is in code/ subdirectory + assert (job_dir / "code").exists(), "code/ should exist in job_dir" + assert (job_dir / "code" / "main.py").exists(), "main.py should be inside code/" + assert (job_dir / "code" / "pyproject.toml").exists(), ( + "pyproject.toml should be inside code/" + ) + assert (job_dir / "run.sh").exists(), "run.sh should be at job_dir root" + assert (job_dir / "config.yaml").exists(), ( + "config.yaml should be at job_dir root" + ) + + # Verify run.sh uses uv sync inside code folder and correct entrypoint path + run_script = (job_dir / "run.sh").read_text() + assert "uv sync" in run_script, ( + "Should use 'uv sync' for folders with pyproject.toml" + ) + assert "cd code" in run_script, "Should cd into code folder for uv sync" + assert "python main.py" in run_script, "Should run main.py from code/" + + finally: + shutil.rmtree(project_dir, ignore_errors=True) + + +def test_folder_job_auto_detect_main_py(): + """Test that entrypoint is auto-detected when main.py exists.""" + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + sync_automatically=False, + ) + + project_dir = tempfile.mkdtemp(prefix="test_auto_main_") + Path(project_dir).name + + try: + # Create main.py and another file + (Path(project_dir) / "main.py").write_text('print("main")') + (Path(project_dir) / "utils.py").write_text('print("utils")') + + # Submit without entrypoint - should auto-detect main.py + ds_manager.submit_python_job( + user=do_manager.email, + code_path=project_dir, + job_name="test.auto.main", + # No entrypoint specified + ) + + do_manager.sync() + job = do_manager.job_client.jobs[0] + job_dir = job.job_submission_path + + # Verify main.py was auto-detected + run_script = (job_dir / "run.sh").read_text() + assert "cd code" in run_script, "Should cd into code folder" + assert "python main.py" in run_script, ( + "Should auto-detect main.py as entrypoint" + ) + + finally: + shutil.rmtree(project_dir, ignore_errors=True) + + +def test_folder_job_auto_detect_single_py(): + """Test that entrypoint is auto-detected when only one .py file exists.""" + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + sync_automatically=False, + ) + + project_dir = tempfile.mkdtemp(prefix="test_auto_single_") + Path(project_dir).name + + try: + # Create only one .py file (not named main.py) + (Path(project_dir) / "script.py").write_text('print("script")') + (Path(project_dir) / "README.md").write_text("# Readme") + + # Submit without entrypoint - should auto-detect script.py + ds_manager.submit_python_job( + user=do_manager.email, + code_path=project_dir, + job_name="test.auto.single", + ) + + do_manager.sync() + job = do_manager.job_client.jobs[0] + job_dir = job.job_submission_path + + # Verify script.py was auto-detected + run_script = (job_dir / "run.sh").read_text() + assert "cd code" in run_script, "Should cd into code folder" + assert "python script.py" in run_script, ( + "Should auto-detect single .py file as entrypoint" + ) + + finally: + shutil.rmtree(project_dir, ignore_errors=True) + + +def test_folder_job_no_auto_detect_multiple_py(): + """Test that auto-detection fails when multiple .py files and no main.py.""" + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + sync_automatically=False, + ) + + project_dir = tempfile.mkdtemp(prefix="test_no_auto_") + + try: + # Create multiple .py files (no main.py) + (Path(project_dir) / "script1.py").write_text('print("1")') + (Path(project_dir) / "script2.py").write_text('print("2")') + + # Submit without entrypoint - should fail + with pytest.raises(ValueError, match="Could not auto-detect entrypoint"): + ds_manager.submit_python_job( + user=do_manager.email, + code_path=project_dir, + job_name="test.no.auto", + ) + + finally: + shutil.rmtree(project_dir, ignore_errors=True) + + +def test_single_file_job_flow_with_dataset(): + """Test complete job submission flow with dataset access. + + This test verifies the end-to-end flow of: + 1. Data Owner (DO) creates a dataset with mock and private data + 2. Data Scientist (DS) syncs and sees the dataset + 3. DS submits a Python job that accesses the private dataset + 4. DO approves and runs the job + 5. Job reads private data using syft:// protocol + 6. Results sync back to DS + + Test flow: + DO: create_dataset("my dataset") with private.txt containing "Hello, world!" + ↓ + DS: sync() → sees dataset + ↓ + DS: submit_python_job() with code that reads syft://private/... + ↓ + DO: sync() → receives job + ↓ + DO: job.approve() + process_approved_jobs() + ↓ + Job executes: reads private data → writes outputs/result.json + ↓ + DO: sync() → sends results + ↓ + DS: sync() → receives results + ↓ + Assert: result.json contains {"result": "Hello, world!"} + + Verifies: + - Dataset creation and sync between DO and DS + - Job submission with syft:// path resolution + - Job approval and execution workflow + - Output file sync back to DS + """ + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False + ) + + mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() + do_manager.create_dataset( + name="my dataset", + mock_path=mock_dset_path, + private_path=private_dset_path, + summary="This is a summary", + readme_path=readme_path, + tags=["tag1", "tag2"], + users=[ds_manager.email], # Share with DS so they can access the dataset + ) + + datasets = do_manager.datasets.get_all() + assert len(datasets) == 1 + + ds_manager.sync() + + assert len(ds_manager.datasets.get_all()) == 1 + + test_py_path = "/tmp/test.py" + with open(test_py_path, "w") as f: + f.write(""" +import json +import syft_client as sc + +data_path = sc.resolve_dataset_file_path("my dataset") + +with open(data_path, "r") as data_file: + data = data_file.read() + +result = {"result": data} + +with open("outputs/result.json", "w") as f: + f.write(json.dumps(result)) +""") + + ds_manager.submit_python_job( + user=do_manager.email, + code_path=test_py_path, + job_name="test.job", + ) + do_manager.sync() + + assert len(do_manager.job_client.jobs) == 1 + job = do_manager.job_client.jobs[0] + + job.approve() + + do_manager.job_runner.process_approved_jobs() + + # Before sharing: DS should not see outputs + do_manager.sync() + ds_manager.sync() + assert len(ds_manager.job_client.jobs[-1].output_paths) == 0 + + # After sharing: DS should see outputs + do_manager.job_runner.share_job_results( + "test.job", share_outputs=True, share_logs=False + ) + do_manager.sync() + ds_manager.sync() + + output_path = ds_manager.job_client.jobs[-1].output_paths[0] + with open(output_path, "r") as f: + json_content = json.loads(f.read()) + + assert json_content["result"] == "Hello, world private!" + + +def test_folder_job_flow_with_dataset(): + """Test job submission with a folder containing multiple Python files. + + Tests folder structure: + project_dir/ + ├── main.py # entrypoint, imports from helpers.helper + └── helpers/ + ├── __init__.py # package marker + └── helper.py # helper functions + + Verifies: + - Folder submission with entrypoint parameter works + - Nested package imports work (from helpers.helper import ...) + - Outputs created at job root (not inside code/) + - End-to-end flow: submit → approve → run → sync → verify output + """ + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False + ) + + mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() + do_manager.create_dataset( + name="my dataset", + mock_path=mock_dset_path, + private_path=private_dset_path, + summary="This is a summary", + readme_path=readme_path, + tags=["tag1", "tag2"], + users=[ds_manager.email], # Share with DS so they can access the dataset + ) + + ds_manager.sync() + assert len(ds_manager.datasets.get_all()) == 1 + + # Create test project folder (no pyproject.toml, multiplier=2) + project_dir = create_test_project_folder(with_pyproject=False, multiplier=2) + + try: + ds_manager.submit_python_job( + user=do_manager.email, + code_path=str(project_dir), + job_name="test.folder.job", + entrypoint="main.py", + ) + do_manager.sync() + + assert len(do_manager.job_client.jobs) == 1 + job = do_manager.job_client.jobs[0] + + job.approve() + do_manager.job_runner.process_approved_jobs() + + # Before sharing: DS should not see outputs + do_manager.sync() + ds_manager.sync() + assert len(ds_manager.job_client.jobs[-1].output_paths) == 0 + + # After sharing: DS should see outputs + do_manager.job_runner.share_job_results( + "test.folder.job", share_outputs=True, share_logs=False + ) + do_manager.sync() + ds_manager.sync() + + # Verify the job completed and produced output + output_path = ds_manager.job_client.jobs[-1].output_paths[0] + with open(output_path, "r") as f: + json_content = json.loads(f.read()) + + # Verify the helper module was imported and used correctly + assert json_content["original"] == "Hello, world private!" + assert json_content["processed"] == "Processed: Hello, world private!" + assert json_content["multiplier"] == 2 + + finally: + shutil.rmtree(project_dir, ignore_errors=True) + + +def test_pyproject_folder_job_flow_with_dataset(): + """Test job submission with a folder containing pyproject.toml. + + Tests folder structure: + project_dir/ + ├── pyproject.toml # project config with dependencies + ├── main.py # entrypoint, imports from helpers.helper + └── helpers/ + ├── __init__.py # package marker + └── helper.py # helper functions + + Verifies: + - Folder with pyproject.toml uses 'uv sync' (not 'uv venv') + - Folder is preserved with its name in job_dir + - .venv is created inside the code folder by uv sync + - Nested package imports work + - End-to-end flow: submit → approve → run → sync → verify output + """ + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False + ) + + mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() + do_manager.create_dataset( + name="my dataset", + mock_path=mock_dset_path, + private_path=private_dset_path, + summary="This is a summary", + readme_path=readme_path, + tags=["tag1", "tag2"], + users=[ds_manager.email], # Share with DS so they can access the dataset + ) + + ds_manager.sync() + assert len(ds_manager.datasets.get_all()) == 1 + + # Create test project folder with pyproject.toml, multiplier=3 + project_dir = create_test_project_folder( + with_pyproject=True, multiplier=3, prefix="test_pyproject_" + ) + + try: + ds_manager.submit_python_job( + user=do_manager.email, + code_path=str(project_dir), + job_name="test.pyproject.job", + entrypoint="main.py", + ) + + do_manager.sync() + assert len(do_manager.job_client.jobs) == 1 + job = do_manager.job_client.jobs[0] + job_dir = job.job_submission_path + + # Verify folder structure before running - code is in code/ subdirectory + assert (job_dir / "code").exists(), "code/ should exist in job_dir" + assert (job_dir / "code" / "pyproject.toml").exists(), ( + "pyproject.toml should be inside code/" + ) + assert (job_dir / "code" / "main.py").exists(), "main.py should be inside code/" + assert (job_dir / "run.sh").exists(), "run.sh should be at job_dir root" + + # Verify run.sh uses uv sync (pyproject.toml case) + run_script = (job_dir / "run.sh").read_text() + assert "uv sync" in run_script, ( + "Should use 'uv sync' for folders with pyproject.toml" + ) + assert "cd code" in run_script, "Should cd into code folder for uv sync" + assert "python main.py" in run_script, "Should run main.py from code/" + + # Run the job + job.approve() + do_manager.job_runner.process_approved_jobs() + + # Verify .venv was created inside the code folder (by uv sync) + assert (job_dir / "code" / ".venv").exists(), ( + ".venv should be created inside code/ folder by uv sync" + ) + + # Before sharing: DS should not see outputs + do_manager.sync() + ds_manager.sync() + assert len(ds_manager.job_client.jobs[-1].output_paths) == 0 + + # After sharing: DS should see outputs + do_manager.job_runner.share_job_results( + "test.pyproject.job", share_outputs=True, share_logs=False + ) + do_manager.sync() + ds_manager.sync() + + # Verify the job completed and produced output + output_path = ds_manager.job_client.jobs[-1].output_paths[0] + with open(output_path, "r") as f: + json_content = json.loads(f.read()) + + # Verify the helper module was imported and used correctly + assert json_content["original"] == "Hello, world private!" + assert json_content["processed"] == "Processed: Hello, world private!" + assert json_content["multiplier"] == 3 + + finally: + shutil.rmtree(project_dir, ignore_errors=True) + + +def test_ds_dataset_cache_aware_sync(): + """Test that DS loads dataset hashes from disk and skips re-download on restart. + + This test verifies cache-aware dataset syncing: + 1. Create pair 1, DO creates dataset, DS syncs and downloads + 2. Create pair 2 with same directories and backing store (simulates restart) + 3. Verify hash is loaded from disk on startup and matches remote hash + 4. This ensures sync_down_datasets will skip downloading (hash comparison passes) + """ + from unittest.mock import patch + + # Create first pair + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False + ) + + mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() + + do_manager.create_dataset( + name="cached dataset", + mock_path=mock_dset_path, + private_path=private_dset_path, + summary="This is a cached dataset", + readme_path=readme_path, + tags=["cache", "test"], + users=[ds_manager.email], + ) + + # DS syncs and receives dataset + ds_manager.sync() + + # Verify dataset was downloaded + assert len(ds_manager.datasets.get_all()) == 1 + dataset = ds_manager.datasets.get("cached dataset", datasite=do_manager.email) + assert dataset.mock_files[0].exists() + + # Get the original hash from the collection + collections = ds_manager.sync_engine._connection_router.watcher_list_collections( + DATASET_COLLECTION_PREFIX + ) + remote_hash = None + for c in collections: + if c["tag"] == "cached dataset": + remote_hash = c["content_hash"] + break + assert remote_hash is not None + + # Get the mock backing store and directories for creating second pair + mock_backing_store = ds_manager.sync_engine._connection_router.connections[ + 0 + ].drive_service._backing_store + ds_folder = ds_manager.syftbox_folder + do_folder = do_manager.syftbox_folder + ds_email = ds_manager.email + do_email = do_manager.email + + # Create second pair with same directories (simulates restart) + ds_manager2, do_manager2 = SyftRDSClient.pair_with_mock_drive_service_connection( + email1=do_email, + email2=ds_email, + base_path1=do_folder, + base_path2=ds_folder, + use_in_memory_cache=False, + add_peers=False, + ) + + # Replace mock backing store to share dataset collections + ds_manager2.sync_engine._connection_router.connections[ + 0 + ].drive_service._backing_store = mock_backing_store + do_manager2.sync_engine._connection_router.connections[ + 0 + ].drive_service._backing_store = mock_backing_store + + # Load peers (already approved in shared backing store) + ds_manager2.load_peers() + do_manager2.load_peers() + + # Verify hash was loaded from disk on startup + ds_cache = ds_manager2.sync_engine.datasite_watcher_syncer.datasite_watcher_cache + # Cache uses full path as key: syftbox_folder / owner_email / collection_subpath / tag + cache_key = ds_cache.get_collection_path( + do_email, "cached dataset", COLLECTION_SUBPATH + ) + assert cache_key in ds_cache.collection_hashes, ( + "Hash should be loaded from disk on startup" + ) + + # Verify the loaded hash matches the remote hash + # This ensures sync_down_datasets will skip the download (hash comparison at line 283-284) + loaded_hash = ds_cache.collection_hashes[cache_key] + assert loaded_hash == remote_hash, ( + "Loaded hash should match remote hash, ensuring no re-download is needed" + ) + + # Patch the download method to verify it's NOT called (hash match should skip download) + syncer = ds_manager2.sync_engine.datasite_watcher_syncer + original_method = syncer.download_collection_file_with_new_connection + + with patch( + "syft_client.sync.sync.datasite_watcher_syncer.DatasiteWatcherSyncer.download_collection_file_with_new_connection", + wraps=original_method, + ) as mock_download: + # Sync - no download should happen because hash matches + ds_manager2.sync() + + # Verify download_collection_file_with_new_connection was NOT called + assert mock_download.call_count == 0, ( + "Should not download files when local hash matches remote" + ) + + # Verify dataset still accessible + assert len(ds_manager2.datasets.get_all()) == 1 + + +def test_do_dataset_cache_aware_sync(): + """Test that DO doesn't re-download datasets on restart when hash matches. + + This test verifies cache-aware dataset syncing for DO side: + 1. Create pair 1, DO creates dataset + 2. Create pair 2 with same directories and backing store (simulates restart) + 3. Verify _download_collections_parallel is NOT called on sync + """ + from unittest.mock import patch + + # Create first pair + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False + ) + + mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() + + do_manager.create_dataset( + name="do cached dataset", + mock_path=mock_dset_path, + private_path=private_dset_path, + summary="This is a DO cached dataset", + readme_path=readme_path, + tags=["cache", "do", "test"], + users=[ds_manager.email], + ) + + # Verify dataset was created locally + assert len(do_manager.datasets.get_all()) == 1 + + # Get the mock backing store and directories for creating second pair + mock_backing_store = ds_manager.sync_engine._connection_router.connections[ + 0 + ].drive_service._backing_store + ds_folder = ds_manager.syftbox_folder + do_folder = do_manager.syftbox_folder + ds_email = ds_manager.email + do_email = do_manager.email + + # Create second pair with same directories (simulates restart) + ds_manager2, do_manager2 = SyftRDSClient.pair_with_mock_drive_service_connection( + email1=do_email, + email2=ds_email, + base_path1=do_folder, + base_path2=ds_folder, + use_in_memory_cache=False, + add_peers=False, + ) + + # Replace mock backing store to share dataset collections + ds_manager2.sync_engine._connection_router.connections[ + 0 + ].drive_service._backing_store = mock_backing_store + do_manager2.sync_engine._connection_router.connections[ + 0 + ].drive_service._backing_store = mock_backing_store + + # Load peers (already approved in shared backing store) + ds_manager2.load_peers() + do_manager2.load_peers() + + # Patch the download method to verify it's NOT called (hash match should skip download) + syncer = do_manager2.sync_engine.datasite_owner_syncer + with patch.object( + syncer, + "_download_file_with_new_connection", + wraps=syncer._download_file_with_new_connection, + ) as mock_download: + # Sync - should NOT trigger download since local hash matches remote + do_manager2.sync() + + # Verify _download_file_with_new_connection was NOT called + assert mock_download.call_count == 0, ( + "Should not download files when local hash matches remote" + ) + + # Verify dataset still accessible + assert len(do_manager2.datasets.get_all()) == 1 + + +def test_in_memory_connection_load_state(): + """Test state persistence and loading with mock drive connection. + + Unit test equivalent of integration test_google_drive_connection_load_state. + + Workflow (matches integration test): + 1. Pair 1: Create peers, make changes, create dataset + 2. Pair 2: Load peers, sync DO → verify events processed + 3. Pair 3: Load peers, sync both → verify state loaded from storage + """ + + # Get shared backing store and directories that will persist across pairs + ds_manager1, do_manager1 = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + add_peers=True, + ) + + # Get the backing store that will persist across "restarts" + backing_store = do_manager1.sync_engine._connection_router.connections[ + 0 + ].drive_service._backing_store + ds_folder = ds_manager1.syftbox_folder + do_folder = do_manager1.syftbox_folder + ds_email = ds_manager1.email + do_email = do_manager1.email + ds_manager1.sync_engine._connection_router.connections[0] + do_manager1.sync_engine._connection_router.connections[0] + + # Make some changes (submit to DS's job folder where DS has write access) + ds_manager1.sync_engine._send_file_change( + path_for_job(do_manager1.email, ds_manager1.email, "my.job"), "Hello, world!" + ) + ds_manager1.sync_engine._send_file_change( + path_for_job(do_manager1.email, ds_manager1.email, "my_second.job"), + "Hello, world!", + ) + + # Create a dataset with "any" permission + mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() + do_manager1.create_dataset( + name="load_state_dataset", + mock_path=mock_dset_path, + private_path=private_dset_path, + summary="Dataset for load state test", + readme_path=readme_path, + tags=["test"], + users="any", + ) + + # Verify dataset was created and cache populated + assert ( + len( + do_manager1.sync_engine._connection_router.owner_list_collections( + DATASET_COLLECTION_PREFIX + ) + ) + == 1 + ) + assert ( + len(do_manager1.sync_engine.datasite_owner_syncer.any_shared_collections) == 1 + ) + + # Create second pair (simulates restart, tests loading peers and processing inbox) + do_config2 = SyftRDSClientConfig._base_config_for_testing( + email=do_email, + syftbox_folder=do_folder, + has_ds_role=False, + has_do_role=True, + use_in_memory_cache=False, + check_versions=False, + ) + ds_config2 = SyftRDSClientConfig._base_config_for_testing( + email=ds_email, + syftbox_folder=ds_folder, + has_ds_role=True, + has_do_role=False, + use_in_memory_cache=False, + check_versions=False, + ) + + do_manager2 = SyftRDSClient.from_config(do_config2) + ds_manager2 = SyftRDSClient.from_config(ds_config2) + + # Connect to the same backing store + service_do = mock_drive_service.MockDriveService(backing_store, do_email) + do_connection2 = GDriveConnection.from_service(do_email, service_do) + + service_ds = mock_drive_service.MockDriveService(backing_store, ds_email) + ds_connection2 = GDriveConnection.from_service(ds_email, service_ds) + + do_manager2.sync_engine._add_connection(do_connection2) + ds_manager2.sync_engine._add_connection(ds_connection2) + + # Load peers + do_manager2.load_peers() + assert len(do_manager2.peers) == 1 + + ds_manager2.load_peers() + assert len(ds_manager2.peers) == 1 + + # Sync DO so we have something in the syftbox and do outbox + do_manager2.sync() + ds_manager2.sync() + + # Verify events in DO cache (inbox was processed) + # 2 data events + 1 permission file event (syft.pub.yaml from approve_peer_request) + assert ( + len( + do_manager2.sync_engine.datasite_owner_syncer.event_cache.get_cached_events() + ) + == 4 + ) + + # verify events in DS cache + loaded_events_ds = ds_manager2.sync_engine.datasite_watcher_syncer.datasite_watcher_cache.get_cached_events() + assert len(loaded_events_ds) == 2 + + # Verify datasets were loaded + loaded_datasets = do_manager2.datasets.get_all() + assert len(loaded_datasets) == 1 + assert loaded_datasets[0].name == "load_state_dataset" + assert ( + len(do_manager2.sync_engine.datasite_owner_syncer.any_shared_collections) == 1 + ) + assert ( + do_manager2.sync_engine.datasite_owner_syncer.any_shared_collections[0][0] + == "load_state_dataset" + ) + + +def test_datasets_shared_with_any(): + """Test that datasets shared with 'any' become discoverable after peer approval. + + Unit test equivalent of integration test_datasets_shared_with_any. + """ + # Create managers WITHOUT auto peer setup + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + add_peers=False, + ) + + mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() + + # DO creates dataset with users='any' BEFORE peer is approved + do_manager.create_dataset( + name="any dataset", + mock_path=mock_dset_path, + private_path=private_dset_path, + summary="Dataset shared with anyone", + readme_path=readme_path, + tags=["any"], + users="any", + ) + + # DS should NOT see the dataset yet (not approved) + ds_collections = ds_manager.sync_engine._connection_router.watcher_list_collections( + DATASET_COLLECTION_PREFIX + ) + assert not any(c["tag"] == "any dataset" for c in ds_collections) + + # DS adds peer, DO approves (this should share 'any' datasets) + ds_manager.add_peer(do_manager.email) + do_manager.load_peers() + do_manager.approve_peer_request(ds_manager.email, peer_must_exist=False) + + # DS should now see the dataset + ds_collections = ds_manager.sync_engine._connection_router.watcher_list_collections( + DATASET_COLLECTION_PREFIX + ) + assert any(c["tag"] == "any dataset" for c in ds_collections) + + +def test_datasets_shared_with_any_after_peer_approved(): + """Test that creating a dataset with users='any' after peers are approved + grants those peers access immediately. + + Workflow: + 1. Create managers without auto peers + 2. DS adds peer, DO approves + 3. DO creates dataset with users='any' + 4. DS can see the dataset (without needing another approve_peer_request) + """ + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + add_peers=False, + ) + + # DS adds peer, DO approves + ds_manager.add_peer(do_manager.email) + do_manager.load_peers() + do_manager.approve_peer_request(ds_manager.email, peer_must_exist=False) + + mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() + + # DO creates dataset with users='any' AFTER peer is already approved + do_manager.create_dataset( + name="late any dataset", + mock_path=mock_dset_path, + private_path=private_dset_path, + summary="Dataset shared with anyone, created after peer approval", + readme_path=readme_path, + tags=["any", "late"], + users="any", + ) + + # DS should see the dataset immediately (shared at creation time) + ds_collections = ds_manager.sync_engine._connection_router.watcher_list_collections( + DATASET_COLLECTION_PREFIX + ) + assert any(c["tag"] == "late any dataset" for c in ds_collections) + + +def test_ds_stale_state_cleared_after_do_delete_syftbox(): + """Test that DS state is cleaned up after DO calls delete_syftbox(). + + Verifies: + 1. After initial setup, DS has datasets and file_hashes + 2. DO calls delete_syftbox() which broadcasts is_deleted events + 3. DS syncs and its file_hashes and datasets are empty + """ + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + ) + + # Create dataset on DO side + mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() + do_manager.create_dataset( + name="my dataset", + mock_path=mock_dset_path, + private_path=private_dset_path, + summary="Test dataset", + readme_path=readme_path, + users=[ds_manager.email], + ) + + # DS sends a file change to DO (use correct job path) + ds_manager.sync_engine._send_file_change( + path_for_job(do_manager.email, ds_manager.email), "print('hello')" + ) + do_manager.sync() + + # DS syncs to get dataset and file events + ds_manager.sync() + + # Verify initial state exists + ds_cache = ds_manager.sync_engine.datasite_watcher_syncer.datasite_watcher_cache + assert len(ds_manager.datasets.get_all()) == 1 + assert len(ds_cache.file_hashes) > 0 + + # DO deletes syftbox (broadcasts delete events to DS) + do_manager.delete_syftbox() + + # DS syncs again - should pick up delete events and stale dataset cleanup + ds_manager.sync() + + # Verify DS state is cleaned up + assert len(ds_cache.file_hashes) == 0, ( + "DS file_hashes should be empty after DO delete" + ) + assert len(ds_manager.datasets.get_all()) == 0, ( + "DS datasets should be empty after DO delete" + ) + + +def test_dataset_delete_propagates_to_ds(): + """Test that deleting a dataset on DO removes it from Drive and DS.""" + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False + ) + + mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() + + # DO creates dataset shared with DS + do_manager.create_dataset( + name="deleteme", + mock_path=mock_dset_path, + private_path=private_dset_path, + summary="To be deleted", + readme_path=readme_path, + users=[ds_manager.email], + ) + + # Verify collection exists on Drive + collections = do_manager.sync_engine._connection_router.owner_list_collections( + DATASET_COLLECTION_PREFIX + ) + assert "deleteme" in collections + + # DS syncs and sees the dataset + ds_manager.sync() + assert len(ds_manager.datasets.get_all()) == 1 + + # DO deletes the dataset + do_manager.delete_dataset(name="deleteme", require_confirmation=False, sync=True) + + # Verify collection is gone from Drive + collections_after = ( + do_manager.sync_engine._connection_router.owner_list_collections( + DATASET_COLLECTION_PREFIX + ) + ) + assert "deleteme" not in collections_after + + # DS syncs again — should pick up the deletion + ds_manager.sync() + assert len(ds_manager.datasets.get_all()) == 0 diff --git a/tests/unit/test_version_mismatch_flow.py b/packages/syft-rds/tests/test_version_mismatch_flow.py similarity index 93% rename from tests/unit/test_version_mismatch_flow.py rename to packages/syft-rds/tests/test_version_mismatch_flow.py index 471142d0e71..569a1efef59 100644 --- a/tests/unit/test_version_mismatch_flow.py +++ b/packages/syft-rds/tests/test_version_mismatch_flow.py @@ -11,10 +11,11 @@ from syft_client.sync.connections.drive.mock_drive_service import ( MockDriveService, ) -from syft_client.sync.syftbox_manager import SyftboxManager, SyftboxManagerConfig +from syft_rds import SyftRDSClient +from syft_rds.config import SyftRDSClientConfig from syft_client.version import SYFT_CLIENT_VERSION -from tests.unit.utils import create_test_project_folder, create_tmp_dataset_files +from dataset_test_utils import create_test_project_folder, create_tmp_dataset_files NEW_VERSION = "99.0.0" @@ -52,27 +53,27 @@ def _reinitialize_manager(email, backing_store, has_do_role, has_ds_role): single manager, reusing the same backing store so the new manager sees the same GDrive state. """ - config = SyftboxManagerConfig._base_config_for_testing( + config = SyftRDSClientConfig._base_config_for_testing( email=email, has_do_role=has_do_role, has_ds_role=has_ds_role, use_in_memory_cache=False, ) - manager = SyftboxManager.from_config(config) + manager = SyftRDSClient.from_config(config) mock_service = MockDriveService(backing_store, email) conn = GDriveConnection.from_service(email, mock_service) - manager._add_connection(conn) + manager.sync_engine._add_connection(conn) if has_ds_role: - manager.file_writer.add_callback( + manager.sync_engine.file_writer.add_callback( "write_file", - manager.datasite_watcher_syncer.on_file_change, + manager.sync_engine.datasite_watcher_syncer.on_file_change, ) if has_do_role: - manager.datasite_owner_syncer.event_cache.add_callback( + manager.sync_engine.datasite_owner_syncer.event_cache.add_callback( "on_event_local_write", - manager.job_file_change_handler._handle_file_change, + manager.sync_engine.job_file_change_handler._handle_file_change, ) manager.peer_manager.write_own_version() @@ -138,7 +139,7 @@ def test_version_mismatch_and_backup_flow(): """Full flow: create state on v1 -> upgrade to v2 -> old state preserved, new version works.""" # -- Step 1: Create DO/DS on current version -- - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) diff --git a/packages/syft-rds/tests/test_version_negotiation.py b/packages/syft-rds/tests/test_version_negotiation.py new file mode 100644 index 00000000000..f8753009830 --- /dev/null +++ b/packages/syft-rds/tests/test_version_negotiation.py @@ -0,0 +1,125 @@ +"""Version gating on the RDS job submission and execution paths.""" + +import pytest + +from syft_client.sync.version.exceptions import VersionUnknownError +from syft_client.sync.version.version_info import VersionInfo +from syft_rds import SyftRDSClient + + +def build_client_version(version: str) -> VersionInfo: + return VersionInfo( + syft_client_version=version, + min_supported_syft_client_version=version, + protocol_version="1.0.0", + min_supported_protocol_version="1.0.0", + ) + + +def _set_peer_version(client, peer_email: str, version: VersionInfo): + """Override the cached version for a peer without a Drive round-trip.""" + peer = client.peer_manager.get_cached_peer(peer_email) + assert peer is not None, f"peer {peer_email} not in store" + peer.version = version + + +class TestForceSubmission: + """Tests for force_submission parameter.""" + + def test_job_submission_blocked_without_version(self): + ds, do = SyftRDSClient.pair_with_mock_drive_service_connection( + add_peers=False, + sync_automatically=False, + check_versions=True, + ) + + ds.add_peer(do.email) + + test_py_path = "/tmp/test_version.py" + with open(test_py_path, "w") as f: + f.write('print("hello")') + + with pytest.raises(VersionUnknownError): + ds.submit_python_job( + user=do.email, + code_path=test_py_path, + job_name="test.job", + ) + + def test_job_submission_allowed_with_force(self): + ds, do = SyftRDSClient.pair_with_mock_drive_service_connection( + add_peers=False, + sync_automatically=False, + check_versions=True, + ) + + ds.add_peer(do.email) + + test_py_path = "/tmp/test_version_force.py" + with open(test_py_path, "w") as f: + f.write('print("hello")') + + with pytest.raises(VersionUnknownError): + ds.submit_python_job( + user=do.email, + code_path=test_py_path, + job_name="test.fail.job", + ) + + ds.submit_python_job( + user=do.email, + code_path=test_py_path, + job_name="test.force.job", + force_submission=True, + ) + + job_dir = ( + ds.syftbox_folder + / do.email + / "app_data" + / "job" + / "inbox" + / ds.email + / "v1" + / "test.force.job" + ) + assert job_dir.exists() + + +class TestVersionMismatchBehavior: + def test_job_execution_forced_with_incompatible_version(self): + ds, do = SyftRDSClient.pair_with_mock_drive_service_connection( + sync_automatically=False, + use_in_memory_cache=False, + ) + + test_py_path = "/tmp/test_exec_force.py" + with open(test_py_path, "w") as f: + f.write('print("hello")') + + ds.submit_python_job( + user=do.email, + code_path=test_py_path, + job_name="test.exec.force.job", + ) + + do.sync() + assert len(do.job_client.jobs) == 1 + job = do.job_client.jobs[0] + job.approve() + + _set_peer_version(do, ds.email, build_client_version("0.0.1")) + + executed_jobs = [] + + def mock_process_approved_jobs( + stream_output=True, timeout=None, skip_job_names=None, **kwargs + ): + executed_jobs.append(skip_job_names) + + do.job_runner.process_approved_jobs = mock_process_approved_jobs + + do.process_approved_jobs(force_execution=True) + + assert len(executed_jobs) == 1 + assert executed_jobs[0] is None # No jobs skipped when force=True diff --git a/pyproject.toml b/pyproject.toml index 40c9fb98c24..c687f6970c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,31 +30,15 @@ dependencies = [ "google-auth[pyjwt]>=2.22.0", "google-auth-oauthlib>=1.0.0", "rich>=13.0.0", - "pandas", "pydantic-settings>=2.11.0", - "pyyaml>=6.0", - "jinja2>=3.1.0", - "click>=8.0.0", "portalocker>=2.8.0", - "python-daemon>=3.0.0", - "syft-bg==0.3.11", - "syft-job==0.1.39", # TODO: move to optional-dependencies after adding lazy imports - "syft-dataset==0.1.20", # TODO: move to optional-dependencies after adding lazy imports "syft-permissions==0.1.14", "syft-perms==0.1.14", "syft-crypto-python>=0.1.2b2", -] - -[project.optional-dependencies] -datasets = [ "syft-dataset==0.1.20", ] -job = [ - "syft-job==0.1.39", -] -all = [ - "syft-client[datasets,job]", -] + + [tool.uv] default-groups = ["dev", "test"] @@ -125,6 +109,10 @@ exclude_lines = [ "if TYPE_CHECKING:" ] +[tool.ruff] +# Obfuscator output is deliberately not valid Python; keep it out of lint/format. +extend-exclude = ["*.obfuscated.py"] + [tool.isort] profile = "black" multi_line_output = 3 @@ -182,6 +170,7 @@ test = [ "isort>=5.12.0", "coverage>=7.0.0", "enclave-model-api-example[test]", # pulls fastapi/httpx for the inference server tests + "syft-bg", # tests/unit/syft_bg/ lives under this package's testpaths and imports syft_bg "syft-enclave", "syft-restrict", "syft-migration", diff --git a/scripts/auto_approve_jobs.py b/scripts/auto_approve_jobs.py index 67839f00603..531535c402b 100644 --- a/scripts/auto_approve_jobs.py +++ b/scripts/auto_approve_jobs.py @@ -9,8 +9,8 @@ import time from pathlib import Path -from syft_client.job_auto_approval import auto_approve_and_run_jobs -from syft_client.sync.syftbox_manager import SyftboxManager +from syft_rds import login_do +from syft_rds.job_auto_approval import auto_approve_and_run_jobs # Configuration - edit these values EMAIL = "your-email@example.com" @@ -37,9 +37,8 @@ def main(): - client = SyftboxManager.for_jupyter( + client = login_do( email=EMAIL, - has_do_role=True, token_path=TOKEN_PATH, ) diff --git a/syft_client/sync/callback_mixin.py b/syft_client/sync/callback_mixin.py index 0f979cb6b24..30319f7ae1c 100644 --- a/syft_client/sync/callback_mixin.py +++ b/syft_client/sync/callback_mixin.py @@ -1,6 +1,9 @@ +import logging from pydantic import BaseModel from typing import Dict, List, Callable +logger = logging.getLogger(__name__) + class BaseModelCallbackMixin(BaseModel): callbacks: Dict[str, List[Callable]] = {} @@ -9,3 +12,24 @@ def add_callback(self, on: str, callback: Callable): if on not in self.callbacks: self.callbacks[on] = [] self.callbacks[on].append(callback) + + def on(self, event: str, callback: Callable) -> None: + """Readable alias for registering a lifecycle callback.""" + self.add_callback(event, callback) + + def _emit(self, event: str, *args, **kwargs) -> None: + """Fire every callback registered for `event`, ISOLATED from each other + and from the emitting core. + + No-op when nothing is registered, so emitting is always safe. + """ + for callback in self.callbacks.get(event, []): + try: + callback(*args, **kwargs) + except Exception as e: + logger.exception( + "callback for event %r failed (%r) exception: %r", + event, + callback, + e, + ) diff --git a/syft_client/sync/connections/base_connection.py b/syft_client/sync/connections/base_connection.py index a1c5afc881f..b49ad55cbd1 100644 --- a/syft_client/sync/connections/base_connection.py +++ b/syft_client/sync/connections/base_connection.py @@ -32,58 +32,55 @@ def watcher_send_proposed_file_changes_message( def from_config(cls, config: ConnectionConfig): return config.connection_type.from_config(config) - def owner_create_dataset_collection_folder( - self, tag: str, content_hash: str, owner_email: str + # ========================================================================= + # GENERIC PREFIX-PARAMETERIZED COLLECTION METHODS + # ========================================================================= + + def owner_create_collection_folder( + self, prefix: str, tag: str, content_hash: str, owner_email: str ) -> str: raise NotImplementedError() - def owner_tag_dataset_collection_as_any(self, tag: str, content_hash: str) -> None: + def owner_tag_collection_as_any( + self, prefix: str, tag: str, content_hash: str + ) -> None: raise NotImplementedError() - def owner_share_dataset_collection( - self, tag: str, content_hash: str, users: list[str] + def owner_share_collection( + self, prefix: str, tag: str, content_hash: str, users: list[str] ) -> None: raise NotImplementedError() - def owner_upload_dataset_files( - self, tag: str, content_hash: str, files: dict[str, bytes] + def owner_upload_collection_files( + self, prefix: str, tag: str, content_hash: str, files: dict[str, bytes] ) -> None: raise NotImplementedError() - def owner_list_dataset_collections(self) -> list[str]: + def owner_list_collections(self, prefix: str) -> list[str]: raise NotImplementedError() - def owner_list_all_dataset_collections_with_permissions( - self, + def owner_list_all_collections_with_permissions( + self, prefix: str ) -> list[FileCollection]: - """Returns list of FileCollection objects for DO's dataset collections.""" - raise NotImplementedError() - - def watcher_list_dataset_collections(self) -> list[dict]: - """Returns list of dicts with keys: owner_email, tag, content_hash""" raise NotImplementedError() - def watcher_download_dataset_collection( - self, tag: str, content_hash: str, owner_email: str - ) -> dict[str, bytes]: + def owner_delete_collection(self, prefix: str, tag: str) -> None: raise NotImplementedError() - def owner_create_private_dataset_collection_folder( - self, tag: str, content_hash: str, owner_email: str - ) -> str: + def watcher_list_collections(self, prefix: str) -> list[dict]: raise NotImplementedError() - def owner_upload_private_dataset_files( - self, tag: str, content_hash: str, files: dict[str, bytes] - ) -> None: + def watcher_download_collection( + self, prefix: str, tag: str, content_hash: str, owner_email: str + ) -> dict[str, bytes]: raise NotImplementedError() - def owner_list_private_dataset_collections(self) -> list[FileCollection]: + def watcher_get_collection_file_metadatas( + self, prefix: str, tag: str, content_hash: str, owner_email: str + ) -> list[dict]: raise NotImplementedError() - def owner_get_private_collection_file_metadatas( - self, tag: str, content_hash: str, owner_email: str - ) -> list[dict]: + def watcher_download_collection_file(self, file_id: str) -> bytes: raise NotImplementedError() # ========================================================================= diff --git a/syft_client/sync/connections/collection_prefixes.py b/syft_client/sync/connections/collection_prefixes.py new file mode 100644 index 00000000000..eafed83bc21 --- /dev/null +++ b/syft_client/sync/connections/collection_prefixes.py @@ -0,0 +1,11 @@ +"""Wire-level collection folder-name prefixes.""" + +# Mirrors syft_datasets.dataset_manager. Duplicated rather than imported because +# delete_unversioned_state needs these at login time, before an RDS client exists, +# and the sync core must not import the domain. + +# Kept in sync by test_collection_prefixes_match_syft_datasets in +# packages/syft-rds/tests. + +DATASET_COLLECTION_PREFIX = "syft_datasetcollection" +PRIVATE_DATASET_COLLECTION_PREFIX = "syft_privatecollection" diff --git a/syft_client/sync/connections/connection_router.py b/syft_client/sync/connections/connection_router.py index e165ed86e21..b2c4a7b83d6 100644 --- a/syft_client/sync/connections/connection_router.py +++ b/syft_client/sync/connections/connection_router.py @@ -258,71 +258,70 @@ def read_peer_encryption_bundle(self, peer_email: str) -> str | None: return connection.read_peer_encryption_bundle(peer_email) # ========================================================================= - # DATASET METHODS (with encryption) + # GENERIC PREFIX-PARAMETERIZED COLLECTION METHODS (with encryption) # ========================================================================= - def owner_create_dataset_collection_folder( - self, tag: str, content_hash: str, owner_email: str + def owner_create_collection_folder( + self, prefix: str, tag: str, content_hash: str, owner_email: str ) -> str: connection = self.connection_for_send_message() - return connection.owner_create_dataset_collection_folder( - tag, content_hash, owner_email + return connection.owner_create_collection_folder( + prefix, tag, content_hash, owner_email ) - def owner_tag_dataset_collection_as_any(self, tag: str, content_hash: str) -> None: + def owner_tag_collection_as_any( + self, prefix: str, tag: str, content_hash: str + ) -> None: connection = self.connection_for_send_message() - connection.owner_tag_dataset_collection_as_any(tag, content_hash) + connection.owner_tag_collection_as_any(prefix, tag, content_hash) - def owner_share_dataset_collection( - self, tag: str, content_hash: str, users: list[str] + def owner_share_collection( + self, prefix: str, tag: str, content_hash: str, users: list[str] ) -> None: connection = self.connection_for_send_message() - connection.owner_share_dataset_collection(tag, content_hash, users) + connection.owner_share_collection(prefix, tag, content_hash, users) - def owner_upload_dataset_files( + def owner_upload_collection_files( self, + prefix: str, tag: str, content_hash: str, files: dict[str, bytes], recipient_email: str | None = None, ) -> None: - """Upload dataset files, encrypting each file if encryption is enabled.""" + """Upload collection files, encrypting each file if encryption is enabled.""" if recipient_email and self.peer_store: files = { name: self.peer_store.encrypt_if_needed(recipient_email, data) for name, data in files.items() } connection = self.connection_for_send_message() - connection.owner_upload_dataset_files(tag, content_hash, files) + connection.owner_upload_collection_files(prefix, tag, content_hash, files) - def owner_list_dataset_collections(self) -> list[str]: + def owner_list_collections(self, prefix: str) -> list[str]: connection = self.connection_for_send_message() - return connection.owner_list_dataset_collections() + return connection.owner_list_collections(prefix) - def owner_list_all_dataset_collections_with_permissions( - self, + def owner_list_all_collections_with_permissions( + self, prefix: str ) -> list[FileCollection]: connection = self.connection_for_send_message() - return connection.owner_list_all_dataset_collections_with_permissions() + return connection.owner_list_all_collections_with_permissions(prefix) - def owner_delete_dataset_collection(self, tag: str) -> None: + def owner_delete_collection(self, prefix: str, tag: str) -> None: connection = self.connection_for_send_message() - connection.owner_delete_dataset_collection(tag) + connection.owner_delete_collection(prefix, tag) - def owner_delete_private_dataset_collection(self, tag: str) -> None: - connection = self.connection_for_send_message() - connection.owner_delete_private_dataset_collection(tag) - - def watcher_list_dataset_collections(self) -> list[dict]: + def watcher_list_collections(self, prefix: str) -> list[dict]: connection = self.connection_for_receive_message() - return connection.watcher_list_dataset_collections() + return connection.watcher_list_collections(prefix) - def watcher_download_dataset_collection( - self, tag: str, content_hash: str, owner_email: str + def watcher_download_collection( + self, prefix: str, tag: str, content_hash: str, owner_email: str ) -> dict[str, bytes]: connection = self.connection_for_datasite_watcher() - files = connection.watcher_download_dataset_collection( - tag, content_hash, owner_email + files = connection.watcher_download_collection( + prefix, tag, content_hash, owner_email ) if self.peer_store and owner_email: files = { @@ -331,32 +330,6 @@ def watcher_download_dataset_collection( } return files - def owner_create_private_dataset_collection_folder( - self, tag: str, content_hash: str, owner_email: str - ) -> str: - connection = self.connection_for_send_message() - return connection.owner_create_private_dataset_collection_folder( - tag, content_hash, owner_email - ) - - def owner_upload_private_dataset_files( - self, tag: str, content_hash: str, files: dict[str, bytes] - ) -> None: - connection = self.connection_for_send_message() - connection.owner_upload_private_dataset_files(tag, content_hash, files) - - def owner_list_private_dataset_collections(self) -> list[FileCollection]: - connection = self.connection_for_send_message() - return connection.owner_list_private_dataset_collections() - - def owner_get_private_collection_file_metadatas( - self, tag: str, content_hash: str, owner_email: str - ) -> List[dict]: - connection = self.connection_for_datasite_watcher() - return connection.owner_get_private_collection_file_metadatas( - tag, content_hash, owner_email - ) - def connection_for_version_read( self, create_new: bool = False ) -> SyftboxPlatformConnection: @@ -387,17 +360,17 @@ def share_version_file_with_peer(self, peer_email: str) -> None: connection = self.connection_for_own_syftbox() connection.share_version_file_with_peer(peer_email) - def watcher_get_dataset_collection_file_metadatas( - self, tag: str, content_hash: str, owner_email: str + def watcher_get_collection_file_metadatas( + self, prefix: str, tag: str, content_hash: str, owner_email: str ) -> List[dict]: connection = self.connection_for_datasite_watcher() - return connection.watcher_get_dataset_collection_file_metadatas( - tag, content_hash, owner_email + return connection.watcher_get_collection_file_metadatas( + prefix, tag, content_hash, owner_email ) - def watcher_download_dataset_file(self, file_id: str, owner_email: str) -> bytes: + def watcher_download_collection_file(self, file_id: str, owner_email: str) -> bytes: connection = self.connection_for_datasite_watcher() - data = connection.watcher_download_dataset_file(file_id) + data = connection.watcher_download_collection_file(file_id) data = self.peer_store.decrypt_dataset_if_needed(owner_email, data) return data diff --git a/syft_client/sync/connections/drive/gdrive_transport.py b/syft_client/sync/connections/drive/gdrive_transport.py index 3cf50f2dd7c..f20832fc5a6 100644 --- a/syft_client/sync/connections/drive/gdrive_transport.py +++ b/syft_client/sync/connections/drive/gdrive_transport.py @@ -1,58 +1,53 @@ """Google Drive Files transport layer implementation""" -import logging import io import json -from pathlib import Path +import logging import pickle -from syft_client.sync.utils.syftbox_utils import check_env -from syft_client.version import SYFT_CLIENT_VERSION -from typing import Any, Dict, List, Optional, Tuple -from typing import TYPE_CHECKING -from pydantic import BaseModel +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +from google.oauth2.credentials import Credentials as GoogleCredentials from google_auth_httplib2 import AuthorizedHttp from googleapiclient.discovery import build from googleapiclient.http import MediaIoBaseDownload, MediaIoBaseUpload, build_http -from google.oauth2.credentials import Credentials as GoogleCredentials +from pydantic import BaseModel -from syft_client.sync.connections.drive.gdrive_utils import ( - gather_all_file_and_folder_ids_recursive, +from syft_client.sync.checkpoints.checkpoint import ( + CHECKPOINT_FILENAME_PREFIX, + INCREMENTAL_CHECKPOINT_PREFIX, + Checkpoint, + IncrementalCheckpoint, ) -from syft_client.sync.connections.drive.gdrive_retry import ( - execute_with_retries, - next_chunk_with_retries, - batch_execute_with_retries, +from syft_client.sync.checkpoints.rolling_state import ( + ROLLING_STATE_FILENAME_PREFIX, + RollingState, ) -from syft_client.sync.version.version_info import _parse_semver - from syft_client.sync.connections.base_connection import ( FileCollection, SyftboxPlatformConnection, ) -from syft_datasets.dataset_manager import ( - DATASET_COLLECTION_PREFIX, - PRIVATE_DATASET_COLLECTION_PREFIX, +from syft_client.sync.connections.drive.gdrive_retry import ( + batch_execute_with_retries, + execute_with_retries, + next_chunk_with_retries, ) +from syft_client.sync.connections.drive.gdrive_utils import ( + gather_all_file_and_folder_ids_recursive, +) +from syft_client.sync.environments.environment import Environment from syft_client.sync.events.file_change_event import ( - FileChangeEventsMessageFileName, FileChangeEventsMessage, + FileChangeEventsMessageFileName, ) from syft_client.sync.messages.proposed_filechange import ( - MessageFileName, FileNameParseError, + MessageFileName, ProposedFileChangesMessage, ) -from syft_client.sync.environments.environment import Environment -from syft_client.sync.checkpoints.checkpoint import ( - Checkpoint, - IncrementalCheckpoint, - CHECKPOINT_FILENAME_PREFIX, - INCREMENTAL_CHECKPOINT_PREFIX, -) -from syft_client.sync.checkpoints.rolling_state import ( - RollingState, - ROLLING_STATE_FILENAME_PREFIX, -) +from syft_client.sync.utils.syftbox_utils import check_env +from syft_client.sync.version.version_info import _parse_semver +from syft_client.version import SYFT_CLIENT_VERSION if TYPE_CHECKING: from syft_client.sync.connections.drive.grdrive_config import ( @@ -60,12 +55,19 @@ ) from syft_client.sync.version.version_info import VersionInfo +from syft_client.sync.connections.collection_prefixes import ( + DATASET_COLLECTION_PREFIX, + PRIVATE_DATASET_COLLECTION_PREFIX, +) + # Timeout for Google API requests (in seconds) GOOGLE_API_TIMEOUT = 120 # 2 minutes SYFTBOX_FOLDER = "SyftBox" GOOGLE_FOLDER_MIME_TYPE = "application/vnd.google-apps.folder" SCOPES = ["https://www.googleapis.com/auth/drive"] + + logging.getLogger("google_auth_httplib2").setLevel(logging.ERROR) @@ -80,8 +82,8 @@ def build_drive_service( http = build_http() http.timeout = timeout if environment == Environment.COLAB: - from google.colab import auth as colab_auth import google.auth + from google.colab import auth as colab_auth colab_auth.authenticate_user() creds, _ = google.auth.default() @@ -143,58 +145,42 @@ def as_string(self) -> str: return f"{SYFT_CLIENT_VERSION}#{self.email}" -class DatasetCollectionFolder(BaseModel): - """Represents a dataset collection folder with format: {prefix}_{tag}_{hash}""" - - tag: str - content_hash: str +class CollectionFolder(BaseModel): + """Naming value object for collection folders (domain-agnostic). - def as_string(self) -> str: - return f"{DATASET_COLLECTION_PREFIX}_{self.tag}_{self.content_hash}" - - @classmethod - def from_name(cls, name: str) -> "DatasetCollectionFolder": - """Parse folder name like 'syft_datasetcollection_mytag_abc123'""" - parts = name.split("_") - if len(parts) < 3: - raise ValueError(f"Invalid dataset collection folder name: {name}") - # prefix is parts[0:2] joined = "syft_datasetcollection" - # tag is parts[2:-1] joined (in case tag has underscores) - # hash is parts[-1] - tag = "_".join(parts[2:-1]) - content_hash = parts[-1] - return cls(tag=tag, content_hash=content_hash) - - @staticmethod - def compute_hash(files: dict[str, bytes]) -> str: - """Compute a hash from file contents.""" - from syft_client.sync.file_utils import compute_file_hashes - - return compute_file_hashes(files) - - -class PrivateDatasetCollectionFolder(BaseModel): - """Represents a private dataset collection folder with format: {prefix}_{tag}_{hash}""" + Collections (datasets, private datasets, or any future collection type) are + named ``{prefix}_{tag}_{content_hash}`` on the wire. This is the single + source of truth for building and parsing that name; it also computes the + content hash from the folder's file contents. + """ + prefix: str tag: str content_hash: str - def as_string(self) -> str: - return f"{PRIVATE_DATASET_COLLECTION_PREFIX}_{self.tag}_{self.content_hash}" + @property + def folder_name(self) -> str: + """The on-wire folder name. Doubles as the cache key (same string).""" + return f"{self.prefix}_{self.tag}_{self.content_hash}" @classmethod - def from_name(cls, name: str) -> "PrivateDatasetCollectionFolder": - """Parse folder name like 'syft_privatecollection_mytag_abc123'""" - parts = name.split("_") - if len(parts) < 3: - raise ValueError(f"Invalid private collection folder name: {name}") - tag = "_".join(parts[2:-1]) - content_hash = parts[-1] - return cls(tag=tag, content_hash=content_hash) + def from_name(cls, prefix: str, name: str) -> "CollectionFolder": + """Parse '{prefix}_{tag}_{hash}' into a CollectionFolder. + + Raises ValueError if the name does not carry the prefix, so callers can + skip non-matching folders. + """ + marker = f"{prefix}_" + if not name.startswith(marker): + raise ValueError(f"Invalid collection folder name: {name}") + tag, separator, content_hash = name[len(prefix) + 1 :].rpartition("_") + if not separator or not tag or not content_hash: + raise ValueError(f"Invalid collection folder name: {name}") + return cls(prefix=prefix, tag=tag, content_hash=content_hash) @staticmethod def compute_hash(files: dict[str, bytes]) -> str: - """Compute a hash from file contents.""" + """Compute a content hash from file contents.""" from syft_client.sync.file_utils import compute_file_hashes return compute_file_hashes(files) @@ -286,7 +272,7 @@ class Config: personal_syftbox_event_id_cache: Dict[str, str] = {} # tag -> dataset collection folder id - dataset_collection_folder_id_cache: Dict[str, str] = {} + collection_folder_id_cache: Dict[str, str] = {} # Rolling state caches for single-API-call optimization _rolling_state_folder_id: str | None = None @@ -404,9 +390,7 @@ def _copy_caches_to(self, other: "GDriveConnection") -> None: other.personal_syftbox_event_id_cache = dict( self.personal_syftbox_event_id_cache ) - other.dataset_collection_folder_id_cache = dict( - self.dataset_collection_folder_id_cache - ) + other.collection_folder_id_cache = dict(self.collection_folder_id_cache) @property def environment(self) -> Environment: @@ -1174,7 +1158,7 @@ def reset_caches(self): self.own_datasite_outbox_cache.clear() self.archive_folder_id_cache.clear() self.personal_syftbox_event_id_cache.clear() - self.dataset_collection_folder_id_cache.clear() + self.collection_folder_id_cache.clear() self._rolling_state_folder_id = None self._rolling_state_file_id = None self._encryption_bundles_folder_id = None @@ -1493,34 +1477,44 @@ def get_inbox_proposed_event_id_from_name( items = results.get("files", []) return items[0]["id"] if items else None - def owner_create_dataset_collection_folder( - self, tag: str, content_hash: str, owner_email: str + # ========================================================================= + # GENERIC COLLECTION METHODS (parameterized on `prefix`) + # + # Folder names are built and parsed via CollectionFolder, the single source + # of truth for the f"{prefix}_{tag}_{content_hash}" scheme. On-wire behavior + # is byte-identical to the previous inline construction. + # ========================================================================= + + def owner_create_collection_folder( + self, prefix: str, tag: str, content_hash: str, owner_email: str ) -> str: - """Create /SyftBox/{DATASET_COLLECTION_PREFIX}_{tag}_{hash} folder.""" - folder_obj = DatasetCollectionFolder(tag=tag, content_hash=content_hash) - folder_name = folder_obj.as_string() - cache_key = f"{tag}_{content_hash}" + """Create /SyftBox/{prefix}_{tag}_{hash} folder.""" + folder_name = CollectionFolder( + prefix=prefix, tag=tag, content_hash=content_hash + ).folder_name # Check cache - if cache_key in self.dataset_collection_folder_id_cache: - return self.dataset_collection_folder_id_cache[cache_key] + if folder_name in self.collection_folder_id_cache: + return self.collection_folder_id_cache[folder_name] syftbox_folder_id = self.get_syftbox_folder_id() # Check if exists folder_id = self._find_folder_by_name(folder_name, parent_id=syftbox_folder_id) if folder_id: - self.dataset_collection_folder_id_cache[cache_key] = folder_id + self.collection_folder_id_cache[folder_name] = folder_id return folder_id # Create new folder folder_id = self.create_folder(folder_name, syftbox_folder_id) - self.dataset_collection_folder_id_cache[cache_key] = folder_id + self.collection_folder_id_cache[folder_name] = folder_id return folder_id - def owner_tag_dataset_collection_as_any(self, tag: str, content_hash: str) -> None: - """Mark dataset collection as shared with 'any' via appProperties.""" - folder_id = self._get_dataset_collection_folder_id(tag, content_hash) + def owner_tag_collection_as_any( + self, prefix: str, tag: str, content_hash: str + ) -> None: + """Mark collection as shared with 'any' via appProperties.""" + folder_id = self._get_collection_folder_id(prefix, tag, content_hash) execute_with_retries( self.drive_service.files().update( fileId=folder_id, @@ -1528,13 +1522,13 @@ def owner_tag_dataset_collection_as_any(self, tag: str, content_hash: str) -> No ) ) - def owner_share_dataset_collection( - self, tag: str, content_hash: str, users: list[str] + def owner_share_collection( + self, prefix: str, tag: str, content_hash: str, users: list[str] ) -> None: - """Share dataset collection folder with specific users via batch API.""" + """Share collection folder with specific users via batch API.""" if not users: return - folder_id = self._get_dataset_collection_folder_id(tag, content_hash) + folder_id = self._get_collection_folder_id(prefix, tag, content_hash) self._batch_add_permissions(folder_id, users) def _batch_add_permissions(self, file_id: str, users: list[str]) -> None: @@ -1565,11 +1559,11 @@ def callback(request_id, response, exception): ) batch_execute_with_retries(batch) - def owner_upload_dataset_files( - self, tag: str, content_hash: str, files: dict[str, bytes] + def owner_upload_collection_files( + self, prefix: str, tag: str, content_hash: str, files: dict[str, bytes] ) -> None: - """Upload dataset files to collection folder.""" - folder_id = self._get_dataset_collection_folder_id(tag, content_hash) + """Upload files to a collection folder.""" + folder_id = self._get_collection_folder_id(prefix, tag, content_hash) for file_path, content in files.items(): file_payload, _ = self.create_file_payload(content) @@ -1582,11 +1576,11 @@ def owner_upload_dataset_files( ) ) - def owner_list_dataset_collections(self) -> list[str]: - """List collections created by DO (owned by me).""" + def owner_list_collections(self, prefix: str) -> list[str]: + """List collections created by DO (owned by me). Returns list of tags.""" syftbox_folder_id = self.get_syftbox_folder_id() query = ( - f"name contains '{DATASET_COLLECTION_PREFIX}_' and '{syftbox_folder_id}' in parents " + f"name contains '{prefix}_' and '{syftbox_folder_id}' in parents " f"and 'me' in owners and trashed=false and mimeType='{GOOGLE_FOLDER_MIME_TYPE}'" ) results = execute_with_retries( @@ -1597,19 +1591,20 @@ def owner_list_dataset_collections(self) -> list[str]: result = [] for folder in folders: try: - folder_obj = DatasetCollectionFolder.from_name(folder["name"]) - result.append(folder_obj.tag) + cf = CollectionFolder.from_name(prefix, folder["name"]) + result.append(cf.tag) except ValueError: continue return result - def owner_list_all_dataset_collections_with_permissions( + def owner_list_all_collections_with_permissions( self, + prefix: str, ) -> list[FileCollection]: - """List all DO's dataset collections with permissions info.""" + """List all DO's collections with permissions info.""" syftbox_folder_id = self.get_syftbox_folder_id() query = ( - f"name contains '{DATASET_COLLECTION_PREFIX}_' and '{syftbox_folder_id}' in parents " + f"name contains '{prefix}_' and '{syftbox_folder_id}' in parents " f"and 'me' in owners and trashed=false and mimeType='{GOOGLE_FOLDER_MIME_TYPE}'" ) results = execute_with_retries( @@ -1622,7 +1617,7 @@ def owner_list_all_dataset_collections_with_permissions( for folder in results.get("files", []): folder_id = folder["id"] try: - folder_obj = DatasetCollectionFolder.from_name(folder["name"]) + cf = CollectionFolder.from_name(prefix, folder["name"]) has_anyone = ( folder.get("appProperties", {}).get("syft_shared_with_any") == "true" @@ -1630,8 +1625,8 @@ def owner_list_all_dataset_collections_with_permissions( collections.append( FileCollection( folder_id=folder_id, - tag=folder_obj.tag, - content_hash=folder_obj.content_hash, + tag=cf.tag, + content_hash=cf.content_hash, has_any_permission=has_anyone, ) ) @@ -1640,22 +1635,24 @@ def owner_list_all_dataset_collections_with_permissions( return collections - def owner_delete_dataset_collection(self, tag: str) -> None: - """Delete all public dataset collection folders matching the given tag.""" - collections = self.owner_list_all_dataset_collections_with_permissions() + def owner_delete_collection(self, prefix: str, tag: str) -> None: + """Delete all collection folders (for prefix) matching the given tag.""" + collections = self.owner_list_all_collections_with_permissions(prefix) for c in collections: if c.tag == tag: self.delete_file_by_id(c.folder_id) - cache_key = f"{c.tag}_{c.content_hash}" - self.dataset_collection_folder_id_cache.pop(cache_key, None) + folder_name = CollectionFolder( + prefix=prefix, tag=c.tag, content_hash=c.content_hash + ).folder_name + self.collection_folder_id_cache.pop(folder_name, None) - def watcher_list_dataset_collections(self) -> list[dict]: + def watcher_list_collections(self, prefix: str) -> list[dict]: """List collections shared with DS (not owned by me). Returns list of dicts with keys: owner_email, tag, content_hash """ query = ( - f"name contains '{DATASET_COLLECTION_PREFIX}_' and not 'me' in owners " + f"name contains '{prefix}_' and not 'me' in owners " f"and trashed=false and mimeType='{GOOGLE_FOLDER_MIME_TYPE}'" ) results = execute_with_retries( @@ -1666,15 +1663,15 @@ def watcher_list_dataset_collections(self) -> list[dict]: result = [] for folder in folders: try: - folder_obj = DatasetCollectionFolder.from_name(folder["name"]) + cf = CollectionFolder.from_name(prefix, folder["name"]) owner_email = folder.get("owners", [{}])[0].get( "emailAddress", "unknown" ) result.append( { "owner_email": owner_email, - "tag": folder_obj.tag, - "content_hash": folder_obj.content_hash, + "tag": cf.tag, + "content_hash": cf.content_hash, } ) except ValueError: @@ -1682,12 +1679,13 @@ def watcher_list_dataset_collections(self) -> list[dict]: continue return result - def watcher_download_dataset_collection( - self, tag: str, content_hash: str, owner_email: str + def watcher_download_collection( + self, prefix: str, tag: str, content_hash: str, owner_email: str ) -> dict[str, bytes]: - """Download all files from a dataset collection.""" - folder_obj = DatasetCollectionFolder(tag=tag, content_hash=content_hash) - folder_name = folder_obj.as_string() + """Download all files from a collection.""" + folder_name = CollectionFolder( + prefix=prefix, tag=tag, content_hash=content_hash + ).folder_name # Try to find folder by name (could be owned by someone else) folder_id = self._find_folder_by_name(folder_name, owner_email=owner_email) @@ -1703,12 +1701,13 @@ def watcher_download_dataset_collection( return files - def watcher_get_dataset_collection_file_metadatas( - self, tag: str, content_hash: str, owner_email: str + def watcher_get_collection_file_metadatas( + self, prefix: str, tag: str, content_hash: str, owner_email: str ) -> List[Dict]: - """Get file metadata from a dataset collection without downloading.""" - folder_obj = DatasetCollectionFolder(tag=tag, content_hash=content_hash) - folder_name = folder_obj.as_string() + """Get file metadata from a collection without downloading.""" + folder_name = CollectionFolder( + prefix=prefix, tag=tag, content_hash=content_hash + ).folder_name folder_id = self._find_folder_by_name(folder_name, owner_email=owner_email) if not folder_id: @@ -1717,142 +1716,29 @@ def watcher_get_dataset_collection_file_metadatas( file_metadatas = self.get_file_metadatas_from_folder(folder_id) return [{"file_id": f["id"], "file_name": f["name"]} for f in file_metadatas] - def watcher_download_dataset_file(self, file_id: str) -> bytes: - """Download a single file from a dataset collection.""" + def watcher_download_collection_file(self, file_id: str) -> bytes: + """Download a single file from a collection.""" return self.download_file(file_id) - def _get_dataset_collection_folder_id(self, tag: str, content_hash: str) -> str: - """Get folder ID for dataset collection, with caching.""" - cache_key = f"{tag}_{content_hash}" - if cache_key in self.dataset_collection_folder_id_cache: - return self.dataset_collection_folder_id_cache[cache_key] - - folder_obj = DatasetCollectionFolder(tag=tag, content_hash=content_hash) - folder_name = folder_obj.as_string() - syftbox_folder_id = self.get_syftbox_folder_id() - folder_id = self._find_folder_by_name(folder_name, parent_id=syftbox_folder_id) - - if not folder_id: - raise ValueError( - f"Collection folder {tag} with hash {content_hash} not found" - ) - - self.dataset_collection_folder_id_cache[cache_key] = folder_id - return folder_id - - # ========================================================================= - # PRIVATE DATASET COLLECTION METHODS - # ========================================================================= - - def owner_create_private_dataset_collection_folder( - self, tag: str, content_hash: str, owner_email: str + def _get_collection_folder_id( + self, prefix: str, tag: str, content_hash: str ) -> str: - """Create /SyftBox/{PRIVATE_DATASET_COLLECTION_PREFIX}_{tag}_{hash} folder. - - No sharing is applied — only the owner can access this folder. - """ - folder_obj = PrivateDatasetCollectionFolder(tag=tag, content_hash=content_hash) - folder_name = folder_obj.as_string() - cache_key = f"private_{tag}_{content_hash}" - - if cache_key in self.dataset_collection_folder_id_cache: - return self.dataset_collection_folder_id_cache[cache_key] - - syftbox_folder_id = self.get_syftbox_folder_id() - folder_id = self._find_folder_by_name(folder_name, parent_id=syftbox_folder_id) - if folder_id: - self.dataset_collection_folder_id_cache[cache_key] = folder_id - return folder_id - - folder_id = self.create_folder(folder_name, syftbox_folder_id) - self.dataset_collection_folder_id_cache[cache_key] = folder_id - return folder_id - - def owner_upload_private_dataset_files( - self, tag: str, content_hash: str, files: dict[str, bytes] - ) -> None: - """Upload files to a private dataset collection folder.""" - folder_id = self._get_private_collection_folder_id(tag, content_hash) - for file_path, content in files.items(): - file_payload, _ = self.create_file_payload(content) - file_name = Path(file_path).name - file_metadata = {"name": file_name, "parents": [folder_id]} - execute_with_retries( - self.drive_service.files().create( - body=file_metadata, media_body=file_payload, fields="id" - ) - ) + """Get folder ID for a collection, with caching.""" + folder_name = CollectionFolder( + prefix=prefix, tag=tag, content_hash=content_hash + ).folder_name + if folder_name in self.collection_folder_id_cache: + return self.collection_folder_id_cache[folder_name] - def owner_list_private_dataset_collections(self) -> list[FileCollection]: - """List private collections owned by DO.""" - syftbox_folder_id = self.get_syftbox_folder_id() - query = ( - f"name contains '{PRIVATE_DATASET_COLLECTION_PREFIX}_' " - f"and '{syftbox_folder_id}' in parents " - f"and 'me' in owners and trashed=false " - f"and mimeType='{GOOGLE_FOLDER_MIME_TYPE}'" - ) - results = execute_with_retries( - self.drive_service.files().list(q=query, fields="files(id,name)") - ) - - collections = [] - for folder in results.get("files", []): - try: - folder_obj = PrivateDatasetCollectionFolder.from_name(folder["name"]) - collections.append( - FileCollection( - folder_id=folder["id"], - tag=folder_obj.tag, - content_hash=folder_obj.content_hash, - ) - ) - except ValueError: - continue - return collections - - def owner_delete_private_dataset_collection(self, tag: str) -> None: - """Delete all private dataset collection folders matching the given tag.""" - collections = self.owner_list_private_dataset_collections() - for c in collections: - if c.tag == tag: - self.delete_file_by_id(c.folder_id) - cache_key = f"private_{c.tag}_{c.content_hash}" - self.dataset_collection_folder_id_cache.pop(cache_key, None) - - def owner_get_private_collection_file_metadatas( - self, tag: str, content_hash: str, owner_email: str - ) -> List[Dict]: - """Get file metadata from a private dataset collection without downloading.""" - folder_obj = PrivateDatasetCollectionFolder(tag=tag, content_hash=content_hash) - folder_name = folder_obj.as_string() - folder_id = self._find_folder_by_name(folder_name, owner_email=owner_email) - - if not folder_id: - raise ValueError( - f"Private collection {tag} with hash {content_hash} not found" - ) - - file_metadatas = self.get_file_metadatas_from_folder(folder_id) - return [{"file_id": f["id"], "file_name": f["name"]} for f in file_metadatas] - - def _get_private_collection_folder_id(self, tag: str, content_hash: str) -> str: - """Get folder ID for private dataset collection, with caching.""" - cache_key = f"private_{tag}_{content_hash}" - if cache_key in self.dataset_collection_folder_id_cache: - return self.dataset_collection_folder_id_cache[cache_key] - - folder_obj = PrivateDatasetCollectionFolder(tag=tag, content_hash=content_hash) - folder_name = folder_obj.as_string() syftbox_folder_id = self.get_syftbox_folder_id() folder_id = self._find_folder_by_name(folder_name, parent_id=syftbox_folder_id) if not folder_id: raise ValueError( - f"Private collection folder {tag} with hash {content_hash} not found" + f"Collection folder {tag} with hash {content_hash} not found" ) - self.dataset_collection_folder_id_cache[cache_key] = folder_id + self.collection_folder_id_cache[folder_name] = folder_id return folder_id def _get_version_file_id(self) -> Optional[str]: diff --git a/syft_client/sync/syftbox_manager.py b/syft_client/sync/syftbox_manager.py index 4e9837d8a79..99ad9075700 100644 --- a/syft_client/sync/syftbox_manager.py +++ b/syft_client/sync/syftbox_manager.py @@ -1,74 +1,66 @@ -from pathlib import Path import fcntl -from syft_client.sync.peers.peer_store import PeerStore -from syft_client.sync.utils.path_filters import is_normal_syncable_path import logging +import os import shutil -from contextlib import contextmanager -from syft_client.sync.connections.drive.gdrive_transport import GDriveConnection -from syft_client.utils import resolve_path -from concurrent.futures import ThreadPoolExecutor import time -from pydantic import ConfigDict -from syft_job.client import BaseJobClient, JobClient -from syft_job.job import JobsList -from syft_job.job_runner import SyftJobRunner -from syft_job import SyftJobConfig -from syft_datasets.config import SyftBoxConfig -from syft_datasets.dataset_manager import SyftDatasetManager -from syft_client.sync.platforms.base_platform import BasePlatform -from pydantic import BaseModel, PrivateAttr +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from pathlib import Path from typing import List, Optional, cast -from syft_client.sync.sync.caches.datasite_watcher_cache import ( - DataSiteWatcherCacheConfig, -) -from syft_client.sync.sync.caches.datasite_owner_cache import ( - DataSiteOwnerEventCacheConfig, -) -from syft_client.sync.peers.peer_list import PeerList -from syft_client.sync.peers.peer import Peer + +from pydantic import BaseModel, ConfigDict, PrivateAttr + +from syft_client.sync.callback_mixin import BaseModelCallbackMixin from syft_client.sync.connections.base_connection import ( SyftboxPlatformConnection, ) +from syft_client.sync.connections.connection_router import ConnectionRouter +from syft_client.sync.connections.drive import mock_drive_service +from syft_client.sync.connections.drive.gdrive_transport import GDriveConnection +from syft_client.sync.connections.drive.grdrive_config import GdriveConnectionConfig from syft_client.sync.events.file_change_event import ( FileChangeEvent, FileChangeEventsMessage, ) -from syft_client.sync.utils.pre_submit_scan import run_pre_submit_check -from syft_client.sync.utils.syftbox_utils import ( - random_email, - random_syftbox_folder_for_testing, -) from syft_client.sync.file_writer import FileWriter - from syft_client.sync.job_file_change_handler import JobFileChangeHandler -from syft_client.sync.connections.connection_router import ConnectionRouter - -from syft_client.sync.connections.drive.grdrive_config import GdriveConnectionConfig -from syft_client.sync.connections.drive import mock_drive_service +from syft_client.sync.peers.peer import Peer +from syft_client.sync.peers.peer_list import PeerList +from syft_client.sync.peers.peer_store import PeerStore +from syft_client.sync.platforms.base_platform import BasePlatform +from syft_client.sync.sync.caches.datasite_owner_cache import ( + DataSiteOwnerEventCacheConfig, +) +from syft_client.sync.sync.caches.datasite_watcher_cache import ( + DataSiteWatcherCacheConfig, +) +from syft_client.sync.sync.collection_spec import CollectionSyncSpec from syft_client.sync.sync.datasite_owner_syncer import ( + MIN_MESSAGES_COMPACT, DatasiteOwnerSyncer, DatasiteOwnerSyncerConfig, - MIN_MESSAGES_COMPACT, ) from syft_client.sync.sync.datasite_watcher_syncer import ( DatasiteWatcherSyncer, DatasiteWatcherSyncerConfig, ) +from syft_client.sync.utils.path_filters import is_normal_syncable_path +from syft_client.sync.utils.syftbox_utils import ( + random_email, + random_syftbox_folder_for_testing, +) from syft_client.sync.version.peer_manager import ( - CompatAction, PeerManager, PeerManagerConfig, ) from syft_client.sync.version.version_info import VersionInfo +from syft_client.utils import resolve_path from syft_client.version import VERSION_FILE_NAME -import os logger = logging.getLogger(__name__) COLAB_DEFAULT_SYFTBOX_FOLDER = Path("/") JUPYTER_DEFAULT_SYFTBOX_FOLDER = Path.home() / "SyftBox" -COLLECTION_SUBPATH = Path("public/syft_datasets") # ANSI codes for highlighting important warnings in terminals / notebooks. _ANSI_RED = "\033[1;91m" @@ -83,6 +75,25 @@ def get_colab_default_syftbox_folder(email: str): return Path("/content") / f"SyftBox_{email}" +def default_collections_folder( + syftbox_folder: Path | str, + email: str, + collection_specs: list["CollectionSyncSpec"], +) -> Path | None: + """Local folder holding a datasite's collections. + + Derived from the first (public) collection spec by convention; returns + ``None`` when no specs are configured (the generic engine has no + collections of its own). + """ + + # get the first shareable spec + spec = next((s for s in collection_specs if not s.owner_only), None) + if spec is None: + return None + return Path(syftbox_folder) / email / spec.local_subpath + + class SyftboxManagerConfig(BaseModel): email: str syftbox_folder: Path @@ -95,8 +106,6 @@ class SyftboxManagerConfig(BaseModel): peer_manager_config: PeerManagerConfig datasite_watcher_syncer_config: DatasiteWatcherSyncerConfig - dataset_manager_config: SyftBoxConfig - job_client_config: SyftJobConfig @classmethod def for_colab( @@ -110,18 +119,27 @@ def for_colab( bool ] = None, # None: value is determined by the role force_ignore_peer_version: bool = False, + collection_specs: list["CollectionSyncSpec"] | None = None, ): if not has_ds_role and not has_do_role: raise ValueError("At least one of has_ds_role or has_do_role must be True") + # No collection specs by default: the generic sync engine knows nothing + # about custom collections (eg: datasets or jobs). + if collection_specs is None: + collection_specs = [] + syftbox_folder = get_colab_default_syftbox_folder(email) use_in_memory_cache = False - collections_folder = syftbox_folder / email / COLLECTION_SUBPATH + collections_folder = default_collections_folder( + syftbox_folder, email, collection_specs + ) connection_configs = [GdriveConnectionConfig(email=email, token_path=None)] datasite_owner_syncer_config = DatasiteOwnerSyncerConfig( email=email, syftbox_folder=syftbox_folder, collections_folder=collections_folder, + collection_specs=collection_specs, connection_configs=connection_configs, cache_config=DataSiteOwnerEventCacheConfig( email=email, @@ -138,19 +156,10 @@ def for_colab( email=email, use_in_memory_cache=use_in_memory_cache, syftbox_folder=syftbox_folder, - collection_subpath=COLLECTION_SUBPATH, + collection_specs=collection_specs, connection_configs=connection_configs, ), ) - job_client_config = SyftJobConfig( - syftbox_folder=syftbox_folder, - current_user_email=email, - has_do_role=has_do_role, - ) - dataset_manager_config = SyftBoxConfig( - syftbox_folder=syftbox_folder, - email=email, - ) peer_manager_config = PeerManagerConfig( syftbox_folder=syftbox_folder, email=email, @@ -171,8 +180,6 @@ def for_colab( use_in_memory_cache=False, datasite_owner_syncer_config=datasite_owner_syncer_config, datasite_watcher_syncer_config=datasite_watcher_syncer_config, - dataset_manager_config=dataset_manager_config, - job_client_config=job_client_config, peer_manager_config=peer_manager_config, ) @@ -189,12 +196,21 @@ def for_jupyter( bool ] = None, # None: value is determined by the role force_ignore_peer_version: bool = False, + collection_specs: list["CollectionSyncSpec"] | None = None, ): if not has_ds_role and not has_do_role: raise ValueError("At least one of has_ds_role or has_do_role must be True") + # No collection specs by default: the generic sync engine knows nothing + # about datasets. syft-rds registers its dataset spec at initialization + # (see syft_rds.config.DATASET_COLLECTION_SPECS). + if collection_specs is None: + collection_specs = [] + syftbox_folder = get_jupyter_default_syftbox_folder(email) - collections_folder = syftbox_folder / email / COLLECTION_SUBPATH + collections_folder = default_collections_folder( + syftbox_folder, email, collection_specs + ) connection_configs = [ GdriveConnectionConfig(email=email, token_path=token_path) @@ -203,6 +219,7 @@ def for_jupyter( email=email, syftbox_folder=syftbox_folder, collections_folder=collections_folder, + collection_specs=collection_specs, connection_configs=connection_configs, cache_config=DataSiteOwnerEventCacheConfig( email=email, @@ -220,19 +237,10 @@ def for_jupyter( email=email, use_in_memory_cache=False, syftbox_folder=syftbox_folder, - collection_subpath=COLLECTION_SUBPATH, + collection_specs=collection_specs, connection_configs=connection_configs, ), ) - dataset_manager_config = SyftBoxConfig( - syftbox_folder=syftbox_folder, - email=email, - ) - job_client_config = SyftJobConfig( - syftbox_folder=syftbox_folder, - current_user_email=email, - has_do_role=has_do_role, - ) peer_manager_config = PeerManagerConfig( syftbox_folder=syftbox_folder, email=email, @@ -252,8 +260,6 @@ def for_jupyter( use_in_memory_cache=False, datasite_owner_syncer_config=datasite_owner_syncer_config, datasite_watcher_syncer_config=datasite_watcher_syncer_config, - dataset_manager_config=dataset_manager_config, - job_client_config=job_client_config, peer_manager_config=peer_manager_config, ) @@ -267,15 +273,25 @@ def _base_config_for_testing( has_do_role: bool = False, use_in_memory_cache: bool = True, check_versions: bool = False, + collection_specs: list["CollectionSyncSpec"] | None = None, ): + # No collection specs by default: the generic sync engine knows nothing + # about datasets. syft-rds registers its dataset spec at initialization + # (see syft_rds.config.DATASET_COLLECTION_SPECS). + if collection_specs is None: + collection_specs = [] + syftbox_folder = syftbox_folder or random_syftbox_folder_for_testing() email = email or random_email() - collections_folder = syftbox_folder / email / COLLECTION_SUBPATH + collections_folder = default_collections_folder( + syftbox_folder, email, collection_specs + ) datasite_owner_syncer_config = DatasiteOwnerSyncerConfig( email=email, syftbox_folder=syftbox_folder, collections_folder=collections_folder, + collection_specs=collection_specs, write_files=write_files, cache_config=DataSiteOwnerEventCacheConfig( email=email, @@ -291,19 +307,10 @@ def _base_config_for_testing( email=email, use_in_memory_cache=use_in_memory_cache, syftbox_folder=syftbox_folder, - collection_subpath=COLLECTION_SUBPATH, + collection_specs=collection_specs, ), ) - dataset_manager_config = SyftBoxConfig( - syftbox_folder=syftbox_folder, - email=email, - ) - job_client_config = SyftJobConfig( - syftbox_folder=Path(syftbox_folder), - current_user_email=email, - has_do_role=has_do_role, - ) peer_manager_config = PeerManagerConfig( syftbox_folder=syftbox_folder, email=email, @@ -323,8 +330,6 @@ def _base_config_for_testing( use_in_memory_cache=use_in_memory_cache, datasite_owner_syncer_config=datasite_owner_syncer_config, datasite_watcher_syncer_config=datasite_watcher_syncer_config, - dataset_manager_config=dataset_manager_config, - job_client_config=job_client_config, peer_manager_config=peer_manager_config, ) @@ -339,10 +344,19 @@ def for_google_drive_testing_connection( has_do_role: bool = False, use_in_memory_cache: bool = True, check_versions: bool = False, + collection_specs: list["CollectionSyncSpec"] | None = None, ): + # No collection specs by default: the generic sync engine knows nothing + # about datasets. syft-rds registers its dataset spec at initialization + # (see syft_rds.config.DATASET_COLLECTION_SPECS). + if collection_specs is None: + collection_specs = [] + syftbox_folder = syftbox_folder or random_syftbox_folder_for_testing() email = email or random_email() - collections_folder = Path(syftbox_folder) / email / COLLECTION_SUBPATH + collections_folder = default_collections_folder( + syftbox_folder, email, collection_specs + ) connection_configs = [ GdriveConnectionConfig(email=email, token_path=token_path) ] @@ -350,6 +364,7 @@ def for_google_drive_testing_connection( email=email, syftbox_folder=syftbox_folder, collections_folder=collections_folder, + collection_specs=collection_specs, connection_configs=connection_configs, cache_config=DataSiteOwnerEventCacheConfig( email=email, @@ -366,20 +381,11 @@ def for_google_drive_testing_connection( email=email, use_in_memory_cache=use_in_memory_cache, syftbox_folder=syftbox_folder, - collection_subpath=COLLECTION_SUBPATH, + collection_specs=collection_specs, connection_configs=connection_configs, ), ) - dataset_manager_config = SyftBoxConfig( - syftbox_folder=syftbox_folder, - email=email, - ) - job_client_config = SyftJobConfig( - syftbox_folder=syftbox_folder, - current_user_email=email, - has_do_role=has_do_role, - ) peer_manager_config = PeerManagerConfig( syftbox_folder=syftbox_folder, email=email, @@ -397,13 +403,11 @@ def for_google_drive_testing_connection( has_ds_role=has_ds_role, has_do_role=has_do_role, use_in_memory_cache=False, - dataset_manager_config=dataset_manager_config, - job_client_config=job_client_config, peer_manager_config=peer_manager_config, ) -class SyftboxManager(BaseModel): +class SyftboxManager(BaseModelCallbackMixin): # needed for peers model_config = ConfigDict(arbitrary_types_allowed=True) @@ -415,9 +419,6 @@ class SyftboxManager(BaseModel): datasite_owner_syncer: DatasiteOwnerSyncer | None = None job_file_change_handler: JobFileChangeHandler | None = None - dataset_manager: SyftDatasetManager | None = None - job_client: BaseJobClient | None = None - job_runner: SyftJobRunner | None = None peer_manager: PeerManager | None = None has_do_role: bool = False has_ds_role: bool = False @@ -435,22 +436,13 @@ class SyftboxManager(BaseModel): "config", "has_do_role", "has_ds_role", - "dataset_manager", "peer_manager", "peers", - "jobs", - "datasets", "add_peer", "load_peers", "approve_peer_request", "reject_peer_request", "sync", - "create_dataset", - "delete_dataset", - "share_dataset", - "submit_bash_job", - "submit_python_job", - "process_approved_jobs", "create_checkpoint", "should_create_checkpoint", "try_create_checkpoint", @@ -509,10 +501,6 @@ def from_config(cls, config: SyftboxManagerConfig): datasite_owner_syncer = None job_file_change_handler = None datasite_watcher_syncer = None - job_runner = None - - dataset_manager = SyftDatasetManager.from_config(config.dataset_manager_config) - job_client = JobClient.from_config(config.job_client_config) if config.has_do_role: datasite_owner_syncer = DatasiteOwnerSyncer.from_config( @@ -520,7 +508,6 @@ def from_config(cls, config: SyftboxManagerConfig): ) job_file_change_handler = JobFileChangeHandler() - job_runner = SyftJobRunner.from_config(config.job_client_config) if config.has_ds_role: datasite_watcher_syncer = DatasiteWatcherSyncer.from_config( @@ -538,9 +525,6 @@ def from_config(cls, config: SyftboxManagerConfig): datasite_owner_syncer=datasite_owner_syncer, job_file_change_handler=job_file_change_handler, datasite_watcher_syncer=datasite_watcher_syncer, - dataset_manager=dataset_manager, - job_client=job_client, - job_runner=job_runner, peer_manager=peer_manager, has_do_role=config.has_do_role, has_ds_role=config.has_ds_role, @@ -641,6 +625,7 @@ def _pair_with_google_drive_testing_connection( use_in_memory_cache: bool = True, clear_caches: bool = True, check_versions: bool = False, + collection_specs: list["CollectionSyncSpec"] | None = None, ): receiver_config = SyftboxManagerConfig.for_google_drive_testing_connection( email=do_email, @@ -650,6 +635,7 @@ def _pair_with_google_drive_testing_connection( has_ds_role=False, has_do_role=True, check_versions=check_versions, + collection_specs=collection_specs, ) receiver_manager = cls.from_config(receiver_config) @@ -662,6 +648,7 @@ def _pair_with_google_drive_testing_connection( has_ds_role=True, has_do_role=False, check_versions=check_versions, + collection_specs=collection_specs, ) sender_manager = cls.from_config(sender_config) @@ -722,6 +709,7 @@ def pair_with_mock_drive_service_connection( use_in_memory_cache: bool = True, check_versions: bool = False, encryption: bool = False, + collection_specs: list["CollectionSyncSpec"] | None = None, ): """Create a pair of managers using mock Google Drive services for testing. @@ -750,6 +738,7 @@ def pair_with_mock_drive_service_connection( has_do_role=True, use_in_memory_cache=use_in_memory_cache, check_versions=check_versions, + collection_specs=collection_specs, ) ds_config = SyftboxManagerConfig._base_config_for_testing( @@ -759,6 +748,7 @@ def pair_with_mock_drive_service_connection( has_do_role=False, use_in_memory_cache=use_in_memory_cache, check_versions=check_versions, + collection_specs=collection_specs, ) # Create managers from configs @@ -822,84 +812,12 @@ def add_peer( ): """Add a peer. Delegates to PeerManager.""" self.peer_manager.add_peer(peer_email, force=force, verbose=verbose) - self._sync_peer_install_sources_to_job_client() + self._emit_peers_loaded() if self.has_do_role: self._post_approve_peer_do(peer_email) if sync: self.sync() - def submit_bash_job( - self, - user: str, - script: str, - job_name: str = "", - sync=True, - force_submission: bool = False, - ignore_peer_version: bool = False, - ): - # Check version compatibility before submission (uses cached versions) - if not force_submission: - result = self.peer_manager.get_peer_compatibility_status( - user, - action=CompatAction.SUBMIT, - ignore_peer_version=ignore_peer_version, - ) - result.raise_on_skip(operation="submit job") - result.maybe_warn() - job_dir = self.job_client.submit_bash_job(user, script, job_name=job_name) - self.push_job_files(job_dir) - - def submit_python_job( - self, - user: str, - code_path: str, - job_name: str | None = "", - dependencies: list[str] | None = None, - entrypoint: str | None = None, - sync=True, - force_submission: bool = False, - ignore_peer_version: bool = False, - ): - peer_emails = {p.email for p in self.peer_manager.syncable_peers} - if user not in peer_emails: - print(f"⚠️ {user} is not in your peer list.") - print(f" Add them first with: client.add_peer('{user}')") - return - - if not force_submission: - if not run_pre_submit_check(Path(code_path)): - print("Submission aborted.") - return - - print(f"📤 Submitting '{code_path}' to {user}...") - if job_name: - print(f" Job name : {job_name}") - if dependencies: - print(f" Dependencies : {', '.join(dependencies)}") - - # Check version compatibility before submission (uses cached versions) - if not force_submission: - result = self.peer_manager.get_peer_compatibility_status( - user, - action=CompatAction.SUBMIT, - ignore_peer_version=ignore_peer_version, - ) - result.raise_on_skip(operation="submit job") - result.maybe_warn() - job_dir = self.job_client.submit_python_job( - user, - code_path, - job_name=job_name, - dependencies=dependencies, - entrypoint=entrypoint, - ) - self.push_job_files(job_dir) - - print("\n✅ Job submitted successfully!") - print(" Status : inbox (waiting for DO to review)") - print(f"\n⏳ Next step: wait for {user} to approve and run it.") - print(" Check progress with: client.jobs") - def push_job_files(self, job_dir: Path): file_paths = [Path(p) for p in job_dir.rglob("*") if p.is_file()] relative_file_paths = [p.relative_to(self.syftbox_folder) for p in file_paths] @@ -920,8 +838,47 @@ def push_job_files(self, job_dir: Path): relative_file_path, process_now=last_file ) + # --- collection operations (used by the product layer) --- + + def create_collection_folder(self, prefix: str, tag: str, content_hash: str) -> str: + """Create the owner-side folder for a collection. Delegates to ConnectionRouter.""" + return self._connection_router.owner_create_collection_folder( + prefix, tag=tag, content_hash=content_hash, owner_email=self.email + ) + + def upload_collection_files( + self, + prefix: str, + tag: str, + content_hash: str, + files: dict[str, bytes], + recipient_email: str | None = None, + ) -> None: + """Upload a collection's files. Delegates to ConnectionRouter.""" + self._connection_router.owner_upload_collection_files( + prefix, tag, content_hash, files, recipient_email=recipient_email + ) + + def share_collection( + self, prefix: str, tag: str, content_hash: str, users: list[str] + ) -> None: + """Share a collection with specific users. Delegates to ConnectionRouter.""" + self._connection_router.owner_share_collection(prefix, tag, content_hash, users) + + def tag_collection_as_any(self, prefix: str, tag: str, content_hash: str) -> None: + """Mark a collection as shared with "any". Delegates to ConnectionRouter.""" + self._connection_router.owner_tag_collection_as_any(prefix, tag, content_hash) + + def delete_collection(self, prefix: str, tag: str) -> None: + """Delete a collection. Delegates to ConnectionRouter.""" + self._connection_router.owner_delete_collection(prefix, tag) + + def delete_file_by_id(self, file_id: str) -> None: + """Delete a backend file/folder by id. Delegates to ConnectionRouter.""" + self._connection_router.delete_file_by_id(file_id) + @contextmanager - def _sync_file_lock(self): + def sync_file_lock(self): lock_path = self.syftbox_folder / ".sync.lock" lock_path.parent.mkdir(parents=True, exist_ok=True) lock_path.touch(exist_ok=True) @@ -963,7 +920,7 @@ def sync( from background daemons (e.g. syft-bg) where a separate process may have updated the file. """ - with self._sync_file_lock(): + with self.sync_file_lock(): self.load_peers(force_download=force_download_peer_state) if self.has_do_role: peer_emails = [peer.email for peer in self.peer_manager.approved_peers] @@ -995,7 +952,7 @@ def load_peers(self, force_download: bool = False): instead of using the cached copy. """ cast(PeerManager, self.peer_manager).load_peers(force_download=force_download) - self._sync_peer_install_sources_to_job_client() + self._emit_peers_loaded() def _check_peer_request_exists(self, email: str) -> bool: """Check if a peer request exists. Delegates to PeerManager.""" @@ -1011,133 +968,22 @@ def approve_peer_request( self.peer_manager.approve_peer_request( email_or_peer, verbose=verbose, peer_must_exist=peer_must_exist ) - self._sync_peer_install_sources_to_job_client() self._post_approve_peer_do(email_or_peer) def _post_approve_peer_do(self, email_or_peer: str | Peer): - """Shared post-approval logic: job folder setup and dataset sharing.""" peer_email = ( email_or_peer if isinstance(email_or_peer, str) else email_or_peer.email ) + self._emit_peers_loaded() + self._emit("peer_approved", peer_email) - if self.has_do_role: - self.job_client.setup_ds_job_folder_as_do(peer_email) - self._share_any_datasets_with_peer(peer_email) - - def _sync_peer_install_sources_to_job_client(self) -> None: - """Copy each peer's advertised syft-client install source into job_client. - - Called after peer version exchanges so that, when the DS submits a job - to a DO, the run.sh references the DO's local install path rather than - the DS's local detection. - """ - if not self.job_client: - return - for peer in self.peer_manager.peer_store.syncable_peers: - source = peer.version.syft_client_install_source if peer.version else None - if source: - self.job_client.peer_install_sources[peer.email] = source - - def _ensure_local_peer_permissions(self) -> None: - """Recreate local permission files for all approved peers. - - After an upgrade that deletes local state, permission files - (syft.pub.yaml) are lost. This restores them so that incoming - proposals from approved peers are not silently rejected. - """ - if not self.has_do_role: - return - for peer in self.peer_manager.approved_peers: - self.job_client.setup_ds_job_folder_as_do(peer.email) - - def _share_any_datasets_with_peer(self, peer_email: str): - """Share all datasets that have 'any' permission with a specific peer. - - This is needed because Google Drive "anyone with link" files are not - discoverable via search. By adding explicit user sharing, the peer - can discover these datasets. - - Uses cache populated during pull_initial_state() in DatasiteOwnerSyncer. - """ - for tag, content_hash in self.datasite_owner_syncer._any_shared_datasets: - try: - self._connection_router.owner_share_dataset_collection( - tag, content_hash, [peer_email] - ) - except Exception: - # Ignore errors (e.g., already shared) - pass + def _emit_peers_loaded(self) -> None: + self._emit("peers_loaded") def reject_peer_request(self, email_or_peer: str | Peer): """Reject a pending peer request. Delegates to PeerManager.""" self.peer_manager.reject_peer_request(email_or_peer) - @property - def jobs(self) -> JobsList: - """ - Get the list of jobs. Automatically calls sync() before returning jobs - if PRE_SYNC environment variable is set to "true" (case-insensitive). - - PRE_SYNC defaults to "true", so auto-sync is enabled by default. - To disable auto-sync, set: PRE_SYNC=false - """ - if os.environ.get("PRE_SYNC", "true").lower() == "true": - self.sync() - return self.job_client.jobs - - def process_approved_jobs( - self, - stream_output: bool = True, - timeout: int | None = None, - force_execution: bool = False, - share_outputs_with_submitter: bool = False, - share_logs_with_submitter: bool = False, - ignore_peer_version: bool = False, - ) -> None: - """ - Process approved jobs. Automatically calls sync() after processing - - Args: - stream_output: If True (default), stream output in real-time. - If False, capture output at end. - timeout: Timeout in seconds per job. Defaults to 300 (5 minutes). - Can also be set via SYFT_DEFAULT_JOB_TIMEOUT_SECONDS env var. - force_execution: If True, process all approved jobs regardless of - version compatibility. If False (default), skip jobs - from peers with incompatible or unknown versions. - share_outputs_with_submitter: If True, grant read access on outputs to submitter. - share_logs_with_submitter: If True, grant read access on logs to submitter. - - PRE_SYNC defaults to "true", so auto-sync is enabled by default. - To disable auto-sync, set: PRE_SYNC=false - """ - skip_job_names = [] - - if not force_execution: - approved_jobs = [ - job for job in self.job_client.jobs if job.status == "approved" - ] - for job in approved_jobs: - result = self.peer_manager.get_peer_compatibility_status( - job.submitted_by, - action=CompatAction.EXECUTE, - ignore_peer_version=ignore_peer_version, - ) - result.maybe_warn() - if result.should_skip: - skip_job_names.append(job.name) - - self.job_runner.process_approved_jobs( - stream_output=stream_output, - timeout=timeout, - skip_job_names=skip_job_names if skip_job_names else None, - share_outputs_with_submitter=share_outputs_with_submitter, - share_logs_with_submitter=share_logs_with_submitter, - ) - - if os.environ.get("PRE_SYNC", "true").lower() == "true": - self.sync() - def _add_connection(self, connection: SyftboxPlatformConnection): if not ( isinstance(connection, GDriveConnection) @@ -1166,317 +1012,6 @@ def _send_file_change(self, path: str | Path, content: str): def _get_all_accepted_events_do(self) -> List[FileChangeEvent]: return self.datasite_owner_syncer.connection_router.owner_get_all_accepted_events_messages() - def create_dataset( - self, - name: str, - mock_path: str | Path, - private_path: str | Path, - summary: str | None = None, - readme_path: Path | None = None, - location: str | None = None, - tags: list[str] | None = None, - users: list[str] | str | None = None, - upload_private: bool = False, - sync=True, - ): - if self.dataset_manager is None: - raise ValueError("Dataset manager is not set") - - # Only DO can create datasets - if not self.has_do_role: - raise ValueError("Only dataset owners can create datasets") - - # Convert None to empty list - if users is None: - users = [] - - dataset_name = None - created_local = False - mock_folder_id = None - private_folder_id = None - - try: - # Create dataset locally - dataset = self.dataset_manager.create( - name=name, - mock_path=mock_path, - private_path=private_path, - summary=summary, - readme_path=readme_path, - location=location, - tags=tags, - users=users, - ) - created_local = True - dataset_name = dataset.name - - # Upload mock data to collection folder - mock_folder_id = self._upload_dataset_to_collection(dataset, users) - - # Upload private data to a separate owner-only collection - if upload_private: - private_folder_id = self._upload_private_dataset_to_collection(dataset) - - if sync: - self.sync() - - return dataset - - except Exception: - logger.error( - "Failed to create dataset%s, cleaning up", - f" '{dataset_name}'" if dataset_name else "", - ) - self._cleanup_failed_dataset_creation( - dataset_name, created_local, mock_folder_id, private_folder_id - ) - raise - - def _cleanup_failed_dataset_creation( - self, - dataset_name: str | None, - created_local: bool, - mock_folder_id: str | None, - private_folder_id: str | None, - ) -> None: - """Best-effort cleanup after a failed create_dataset, in reverse order.""" - if private_folder_id is not None: - try: - self._connection_router.delete_file_by_id(private_folder_id) - except Exception: - logger.warning( - "Cleanup: failed to delete private GDrive folder %s", - private_folder_id, - ) - - if mock_folder_id is not None: - try: - self._connection_router.delete_file_by_id(mock_folder_id) - except Exception: - logger.warning( - "Cleanup: failed to delete mock GDrive folder %s", - mock_folder_id, - ) - - if created_local and dataset_name is not None: - try: - self.dataset_manager.delete(dataset_name, require_confirmation=False) - except Exception: - logger.warning( - "Cleanup: failed to delete local dataset '%s'", - dataset_name, - ) - - def _upload_dataset_to_collection(self, dataset, users: list[str] | str) -> str: - """Upload dataset files to collection folder. Returns the folder ID.""" - from syft_client.sync.connections.drive.gdrive_transport import ( - DatasetCollectionFolder, - ) - - collection_tag = dataset.name - - # Prepare files to upload - files = {} - for mock_file in dataset.mock_files: - if mock_file.exists(): - files[mock_file.name] = mock_file.read_bytes() - - metadata_path = dataset.mock_dir / "dataset.yaml" - if metadata_path.exists(): - files["dataset.yaml"] = metadata_path.read_bytes() - - if dataset.readme_path and dataset.readme_path.exists(): - files[dataset.readme_path.name] = dataset.readme_path.read_bytes() - - # Compute content hash - content_hash = DatasetCollectionFolder.compute_hash(files) - - # Create collection folder with hash in name - folder_id = self._connection_router.owner_create_dataset_collection_folder( - tag=collection_tag, content_hash=content_hash, owner_email=self.email - ) - - # Upload files - self._connection_router.owner_upload_dataset_files( - collection_tag, content_hash, files - ) - - # Share with users - if users == "any": - self._connection_router.owner_tag_dataset_collection_as_any( - collection_tag, content_hash - ) - self.datasite_owner_syncer._any_shared_datasets.append( - (collection_tag, content_hash) - ) - # Share with all already-approved peers - peer_emails = [p.email for p in self.peer_manager.approved_peers] - if peer_emails: - self._connection_router.owner_share_dataset_collection( - collection_tag, content_hash, peer_emails - ) - else: - self._connection_router.owner_share_dataset_collection( - collection_tag, content_hash, users - ) - - return folder_id - - def _upload_private_dataset_to_collection(self, dataset) -> str | None: - """Upload private dataset files to a separate owner-only collection folder. - Returns the folder ID, or None if no files to upload.""" - from syft_client.sync.connections.drive.gdrive_transport import ( - PrivateDatasetCollectionFolder, - ) - - collection_tag = dataset.name - - # Collect all files in private dir (data, metadata, permissions) - files = {} - for f in dataset.private_dir.iterdir(): - if f.is_file(): - files[f.name] = f.read_bytes() - - if not files: - return None - - content_hash = PrivateDatasetCollectionFolder.compute_hash(files) - - # Create private collection folder (no sharing) - folder_id = ( - self._connection_router.owner_create_private_dataset_collection_folder( - tag=collection_tag, content_hash=content_hash, owner_email=self.email - ) - ) - - # Upload files - self._connection_router.owner_upload_private_dataset_files( - collection_tag, content_hash, files - ) - - return folder_id - - def delete_dataset( - self, - name: str, - datasite: str | None = None, - require_confirmation: bool = True, - sync=True, - ): - if self.dataset_manager is None: - raise ValueError("Dataset manager is not set") - self.dataset_manager.delete( - name=name, - datasite=datasite, - require_confirmation=require_confirmation, - ) - # Delete collection folders from Google Drive so DS peers - # pick up the deletion on their next sync. - try: - self._connection_router.owner_delete_dataset_collection(name) - except Exception: - logger.warning("Failed to delete dataset collection '%s' from Drive", name) - try: - self._connection_router.owner_delete_private_dataset_collection(name) - except Exception: - logger.warning( - "Failed to delete private dataset collection '%s' from Drive", - name, - ) - if sync: - self.sync() - - def share_dataset(self, tag: str, users: list[str] | str, sync=True): - """ - Share an existing dataset with additional users. - - Args: - tag: Dataset name - users: List of email addresses or "any" - sync: Whether to sync after sharing - """ - from syft_client.sync.connections.drive.gdrive_transport import ( - DatasetCollectionFolder, - ) - - if self.dataset_manager is None: - raise ValueError("Dataset manager is not set") - - if not self.has_do_role: - raise ValueError("Only dataset owners can share datasets") - - # Verify dataset exists - dataset = self.dataset_manager.get(name=tag, datasite=self.email) - if dataset is None: - raise ValueError(f"Dataset {tag} not found") - - # Compute current content hash from local files - files = {} - for mock_file in dataset.mock_files: - if mock_file.exists(): - files[mock_file.name] = mock_file.read_bytes() - metadata_path = dataset.mock_dir / "dataset.yaml" - if metadata_path.exists(): - files["dataset.yaml"] = metadata_path.read_bytes() - if dataset.readme_path and dataset.readme_path.exists(): - files[dataset.readme_path.name] = dataset.readme_path.read_bytes() - - content_hash = DatasetCollectionFolder.compute_hash(files) - - # Share collection - if users == "any": - self._connection_router.owner_tag_dataset_collection_as_any( - tag, content_hash - ) - self.datasite_owner_syncer._any_shared_datasets.append((tag, content_hash)) - peer_emails = [p.email for p in self.peer_manager.approved_peers] - if peer_emails: - self._connection_router.owner_share_dataset_collection( - tag, content_hash, peer_emails - ) - else: - if isinstance(users, str): - users = [users] - self._connection_router.owner_share_dataset_collection( - tag, content_hash, users - ) - - if sync: - self.sync() - - def share_private_dataset(self, tag: str, enclave_email: str): - """Share private dataset files with an enclave via outbox events.""" - if not self.has_do_role: - raise ValueError("Only data owners can share private datasets") - - with self._sync_file_lock(): - files = self.dataset_manager.get_private_dataset_files(tag) - events_message = ( - self.datasite_owner_syncer.event_cache.create_events_for_files(files) - ) - self.datasite_owner_syncer.queue_event_for_syftbox( - recipients=[enclave_email], - file_change_events_message=events_message, - ) - self.datasite_owner_syncer.process_syftbox_events_queue() - - @property - def datasets(self) -> SyftDatasetManager: - """ - Get the dataset manager. Automatically calls sync() before returning datasets - if PRE_SYNC environment variable is set to "true" (case-insensitive). - - PRE_SYNC defaults to "true", so auto-sync is enabled by default. - To disable auto-sync, set: PRE_SYNC=false - """ - if self.dataset_manager is None: - raise ValueError("Dataset manager is not set") - - if os.environ.get("PRE_SYNC", "true").lower() == "true": - self.sync() - - return self.dataset_manager - @property def _connection_router(self) -> ConnectionRouter: # for DOs we have a syncer, for DSs we have a watcher syncer @@ -1510,6 +1045,7 @@ def _broadcast_delete_events( ): """Broadcast is_deleted=True events for all tracked files to each peer's outbox.""" from uuid import uuid4 + from syft_client.sync.utils.syftbox_utils import create_event_timestamp timestamp = create_event_timestamp() @@ -1643,7 +1179,7 @@ def create_checkpoint(self): """ if not self.has_do_role: raise ValueError("Checkpoints can only be created by Data Owners") - with self._sync_file_lock(): + with self.sync_file_lock(): return self.datasite_owner_syncer.create_checkpoint() def compact_outboxes_if_needed( @@ -1660,7 +1196,7 @@ def compact_outboxes_if_needed( """ if not self.has_do_role: return {} - with self._sync_file_lock(): + with self.sync_file_lock(): return { peer.email: self.datasite_owner_syncer.compact_outbox_if_needed( peer.email, min_messages=min_messages @@ -1707,13 +1243,6 @@ def _get_all_peer_platforms(self) -> List[BasePlatform]: def _resolve_path(self, path: str | Path) -> Path: return resolve_path(path, syftbox_folder=self.syftbox_folder) - def _resolve_dataset_owners_for_name(self, dataset_name: str) -> str | None: - matches = [] - for dataset in self.dataset_manager.get_all(): - if dataset.name == dataset_name: - matches.append(dataset.owner) - return matches - def _copy(self): from copy import deepcopy @@ -1730,19 +1259,3 @@ def _copy(self): new_do_connection = GDriveConnection.from_service(self.email, drive_service) new_manager._add_connection(new_do_connection) return new_manager - - # def resolve_dataset_path( - # self, dataset_name: str, owner_email: str | None = None - # ) -> Path: - # if owner_email is None: - # owner_emails = self._resolve_dataset_owners_for_name(dataset_name) - # if len(owner_emails) == 1: - # owner_email = owner_emails[0] - # else: - # raise ValueError( - # f"Dataset {dataset_name} has 0 or multiple owners: {owner_emails}, please specify the owner_email" - # ) - - # return resolve_dataset_path( - # dataset_name, syftbox_folder=self.syftbox_folder, owner_email=owner_email - # ) diff --git a/syft_client/sync/sync/caches/datasite_owner_cache.py b/syft_client/sync/sync/caches/datasite_owner_cache.py index eb54cd55962..c29f5754ff1 100644 --- a/syft_client/sync/sync/caches/datasite_owner_cache.py +++ b/syft_client/sync/sync/caches/datasite_owner_cache.py @@ -90,10 +90,6 @@ def from_config(cls, config: DataSiteOwnerEventCacheConfig): raise ValueError("syftbox_folder is required for non-in-memory cache") if config.email is None: raise ValueError("email is required for non-in-memory cache") - if config.collections_folder is None: - raise ValueError( - "collections_folder is required for non-in-memory cache" - ) syftbox_folder_name = Path(config.syftbox_folder).name my_datasite_folder = config.syftbox_folder / config.email syftbox_parent = Path(config.syftbox_folder).parent @@ -168,8 +164,14 @@ def latest_cached_timestamp(self) -> float | None: return None return max(m.timestamp for m in cached_messages) - def collections_relative_path(self) -> Path: - """Return the collections folder path relative to the datasite root.""" + def collections_relative_path(self) -> Path | None: + """Return the collections folder path relative to the datasite root. + + Returns None when no collections folder is configured (no collection + specs registered); + """ + if self.collections_folder is None: + return None return self.collections_folder.relative_to(self.syftbox_folder / self.email) def get_syncable_paths(self) -> dict[Path, bytes]: diff --git a/syft_client/sync/sync/caches/datasite_watcher_cache.py b/syft_client/sync/sync/caches/datasite_watcher_cache.py index 5263b0fcf36..3fca8babb7c 100644 --- a/syft_client/sync/sync/caches/datasite_watcher_cache.py +++ b/syft_client/sync/sync/caches/datasite_watcher_cache.py @@ -14,6 +14,7 @@ CacheFileConnection, InMemoryCacheFileConnection, ) +from syft_client.sync.sync.collection_spec import CollectionSyncSpec SECONDS_BEFORE_SYNCING_DOWN = 0 @@ -24,8 +25,8 @@ class DataSiteWatcherCacheConfig(BaseModel): syftbox_folder: Path | None = None events_base_path: Path | None = None connection_configs: List[ConnectionConfig] = [] - # Subpath from owner_email to collections folder (e.g., "public/syft_datasets") - collection_subpath: Path | None = None + # Collections to sync down (prefix + local subpath per collection type) + collection_specs: list[CollectionSyncSpec] = [] class DataSiteWatcherCache(BaseModel): @@ -46,10 +47,10 @@ class DataSiteWatcherCache(BaseModel): last_event_timestamp_per_peer: Dict[str, float] = {} # Base syftbox folder syftbox_folder: Path | None = None - # Subpath from owner_email to collections folder (e.g., "public/syft_datasets") - collection_subpath: Path | None = None - # Cache of dataset collection hashes: path -> content_hash - dataset_collection_hashes: Dict[Path, str] = {} + # Collections to sync down (prefix + local subpath per collection type) + collection_specs: list[CollectionSyncSpec] = [] + # Cache of collection hashes: path -> content_hash + collection_hashes: Dict[Path, str] = {} # Optional pre-write filter: (path_in_syftbox, is_delete) -> allow? # Return True to allow the write, False to deny it. pre_write_filter: Callable[[str, bool], bool] | None = None @@ -65,16 +66,12 @@ def from_config(cls, config: DataSiteWatcherCacheConfig): connection_configs=config.connection_configs, ), syftbox_folder=config.syftbox_folder, - collection_subpath=config.collection_subpath, + collection_specs=config.collection_specs, ) return res else: if config.syftbox_folder is None: raise ValueError("syftbox_folder is required for non-in-memory cache") - if config.collection_subpath is None: - raise ValueError( - "collection_subpath is required for non-in-memory cache" - ) syftbox_folder_name = Path(config.syftbox_folder).name syftbox_parent = Path(config.syftbox_folder).parent @@ -90,15 +87,15 @@ def from_config(cls, config: DataSiteWatcherCacheConfig): connection_configs=config.connection_configs, ), syftbox_folder=config.syftbox_folder, - collection_subpath=config.collection_subpath, + collection_specs=config.collection_specs, ) cache._load_cached_state() return cache def _load_cached_state(self): - """Load cached state from disk: file hashes, timestamps, and dataset hashes.""" + """Load cached state from disk: file hashes, timestamps, and collection hashes.""" self._load_file_hashes_from_events() - self._load_dataset_hashes_from_disk() + self._load_collection_hashes_from_disk() def _load_file_hashes_from_events(self): """Load file hashes and timestamps from cached events.""" @@ -130,42 +127,47 @@ def _load_file_hashes_from_events(self): else: self.file_hashes[path_key] = event.new_hash - def _load_dataset_hashes_from_disk(self): - """Scan local dataset directories and compute hashes to populate dataset_collection_hashes.""" - for collection_path in self._get_local_dataset_folders(): - content_hash = self._compute_local_dataset_hash(collection_path) + def _load_collection_hashes_from_disk(self): + """Scan local collection directories and compute hashes to populate collection_hashes.""" + for collection_path in self._get_local_collection_folders(): + content_hash = self._compute_local_collection_hash(collection_path) if content_hash: - self.dataset_collection_hashes[collection_path] = content_hash + self.collection_hashes[collection_path] = content_hash def get_collection_owner_email(self, collection_path: Path) -> str: """Extract the owner email from a collection path.""" return collection_path.relative_to(self.syftbox_folder).parts[0] - def get_collection_path(self, owner_email: str, tag: str) -> Path | None: - """Get the full path to a collection for a given owner and tag.""" - if self.syftbox_folder is None or self.collection_subpath is None: + def get_collection_path( + self, owner_email: str, tag: str, local_subpath: Path + ) -> Path | None: + """Get the full path to a collection for a given owner, tag and subpath.""" + if self.syftbox_folder is None: return None - return self.syftbox_folder / owner_email / self.collection_subpath / tag + return self.syftbox_folder / owner_email / local_subpath / tag - def _get_local_dataset_folders(self): - """Yield paths to all local dataset folders.""" + def _get_local_collection_folders(self): + """Yield paths to all local collection folders across all specs.""" if self.syftbox_folder is None or not self.syftbox_folder.exists(): return - if self.collection_subpath is None: - return - for email_dir in self.syftbox_folder.iterdir(): - if not email_dir.is_dir() or "@" not in email_dir.name: - continue - datasets_dir = email_dir / self.collection_subpath - if not datasets_dir.exists(): + for spec in self.collection_specs: + if spec.owner_only: + # Owner-only collections (e.g. private data) are never pulled from + # peers, so the peer-facing watcher does not track them locally. continue - for tag_dir in datasets_dir.iterdir(): - if tag_dir.is_dir(): - yield tag_dir + for email_dir in self.syftbox_folder.iterdir(): + if not email_dir.is_dir() or "@" not in email_dir.name: + continue + collections_dir = email_dir / spec.local_subpath + if not collections_dir.exists(): + continue + for tag_dir in collections_dir.iterdir(): + if tag_dir.is_dir(): + yield tag_dir - def _compute_local_dataset_hash(self, collection_path: Path) -> str | None: - """Compute content hash from local dataset files on disk.""" + def _compute_local_collection_hash(self, collection_path: Path) -> str | None: + """Compute content hash from local collection files on disk.""" from syft_client.sync.file_utils import compute_directory_hash return compute_directory_hash(collection_path) @@ -178,7 +180,7 @@ def clear_cache(self): self.peers = [] self.current_check_point = None self.last_event_timestamp_per_peer = {} - self.dataset_collection_hashes = {} + self.collection_hashes = {} @property def last_event_timestamp(self) -> float | None: @@ -280,19 +282,28 @@ def current_hash_for_file(self, path: str) -> int | None: self.sync_down_if_needed(peer) return self.file_hashes.get(path, None) - def _cleanup_stale_dataset_collections( - self, peer_email: str, remote_collections: list[dict] + def _cleanup_stale_collections( + self, peer_email: str, remote_collections: list[dict], local_subpath: Path ): - """Remove locally cached dataset collections that no longer exist remotely.""" + """Remove locally cached collections that no longer exist remotely. + + Only considers collections under the given local_subpath so that + collections from other specs are not treated as stale. + """ remote_tags = {c["tag"] for c in remote_collections} - for local_collection_path in list(self.dataset_collection_hashes.keys()): + for local_collection_path in list(self.collection_hashes.keys()): owner_email = self.get_collection_owner_email(local_collection_path) if owner_email != peer_email: continue + # Only consider collections that live under this spec's subpath. + if self.syftbox_folder is not None: + expected_parent = self.syftbox_folder / owner_email / local_subpath + if local_collection_path.parent != expected_parent: + continue if local_collection_path.name in remote_tags: continue - del self.dataset_collection_hashes[local_collection_path] + del self.collection_hashes[local_collection_path] if self.syftbox_folder is not None: try: rel_path = local_collection_path.relative_to(self.syftbox_folder) @@ -300,112 +311,133 @@ def _cleanup_stale_dataset_collections( except ValueError: pass - def sync_down_datasets(self, peer_email: str): + def sync_down_collections(self, peer_email: str): """ - Sync dataset collections from peer. + Sync collections from peer across all configured specs. Separate from message sync. Uses hash to skip unchanged collections. """ - # Get list of collections shared with us (now returns list of dicts) - collections = self.connection_router.watcher_list_dataset_collections() + for spec in self.collection_specs: + if spec.owner_only: + # Never pull owner-only collections (e.g. private data) from a peer. + continue + # Get list of collections shared with us (returns list of dicts) + collections = self.connection_router.watcher_list_collections(spec.prefix) - # Filter by peer - peer_collections = [c for c in collections if c["owner_email"] == peer_email] + # Filter by peer + peer_collections = [ + c for c in collections if c["owner_email"] == peer_email + ] - self._cleanup_stale_dataset_collections(peer_email, peer_collections) + self._cleanup_stale_collections( + peer_email, peer_collections, spec.local_subpath + ) - for collection in peer_collections: - owner_email = collection["owner_email"] - tag = collection["tag"] - content_hash = collection["content_hash"] + for collection in peer_collections: + owner_email = collection["owner_email"] + tag = collection["tag"] + content_hash = collection["content_hash"] - # Check if hash changed - skip download if unchanged - collection_path = self.get_collection_path(owner_email, tag) - if collection_path is None: - continue - cached_hash = self.dataset_collection_hashes.get(collection_path) - if cached_hash == content_hash: - continue + # Check if hash changed - skip download if unchanged + collection_path = self.get_collection_path( + owner_email, tag, spec.local_subpath + ) + if collection_path is None: + continue + cached_hash = self.collection_hashes.get(collection_path) + if cached_hash == content_hash: + continue - # Download collection files - files = self.connection_router.watcher_download_dataset_collection( - tag, content_hash, owner_email - ) + # Download collection files + files = self.connection_router.watcher_download_collection( + spec.prefix, tag, content_hash, owner_email + ) - # Write files to local cache (path relative to syftbox_folder) - for file_name, content in files.items(): - rel_path = f"{owner_email}/{self.collection_subpath}/{tag}/{file_name}" - self.file_connection.write_file(rel_path, content) + # Write files to local cache (path relative to syftbox_folder) + for file_name, content in files.items(): + rel_path = f"{owner_email}/{spec.local_subpath}/{tag}/{file_name}" + self.file_connection.write_file(rel_path, content) - # Update hash cache - self.dataset_collection_hashes[collection_path] = content_hash + # Update hash cache + self.collection_hashes[collection_path] = content_hash - def sync_down_datasets_parallel( + def sync_down_collections_parallel( self, peer_email: str, executor: ThreadPoolExecutor, download_fn: Callable[[str], bytes], ): """ - Sync dataset collections from peer with parallel file downloads. - Downloads all files from all collections in a single parallel batch. + Sync collections from peer with parallel file downloads, across all specs. + For each spec, downloads all files from all collections in a single + parallel batch. """ - collections = self.connection_router.watcher_list_dataset_collections() - peer_collections = [c for c in collections if c["owner_email"] == peer_email] + for spec in self.collection_specs: + if spec.owner_only: + # Never pull owner-only collections (e.g. private data) from a peer. + continue + collections = self.connection_router.watcher_list_collections(spec.prefix) + peer_collections = [ + c for c in collections if c["owner_email"] == peer_email + ] - self._cleanup_stale_dataset_collections(peer_email, peer_collections) + self._cleanup_stale_collections( + peer_email, peer_collections, spec.local_subpath + ) - # Gather all files to download across all collections - all_downloads = [] # List of (collection_info, file_metadata) - collections_to_update = [] + # Gather all files to download across all collections for this spec + all_downloads = [] # List of (collection_info, file_metadata) + collections_to_update = [] - for collection in peer_collections: - owner_email = collection["owner_email"] - tag = collection["tag"] - content_hash = collection["content_hash"] + for collection in peer_collections: + owner_email = collection["owner_email"] + tag = collection["tag"] + content_hash = collection["content_hash"] - # Check if hash changed - skip download if unchanged - collection_path = self.get_collection_path(owner_email, tag) - if collection_path is None: - continue - cached_hash = self.dataset_collection_hashes.get(collection_path) - if cached_hash == content_hash: - continue + # Check if hash changed - skip download if unchanged + collection_path = self.get_collection_path( + owner_email, tag, spec.local_subpath + ) + if collection_path is None: + continue + cached_hash = self.collection_hashes.get(collection_path) + if cached_hash == content_hash: + continue - # Get file metadata (no download yet) - file_metadatas = ( - self.connection_router.watcher_get_dataset_collection_file_metadatas( - tag, content_hash, owner_email + # Get file metadata (no download yet) + file_metadatas = ( + self.connection_router.watcher_get_collection_file_metadatas( + spec.prefix, tag, content_hash, owner_email + ) ) - ) - if not file_metadatas: + if not file_metadatas: + continue + + collections_to_update.append(collection) + for metadata in file_metadatas: + all_downloads.append((collection, metadata)) + + if not all_downloads: continue - collections_to_update.append(collection) - for metadata in file_metadatas: - all_downloads.append((collection, metadata)) + # Download all files from all collections in parallel + file_ids = [metadata["file_id"] for _, metadata in all_downloads] + downloaded_contents = list(executor.map(download_fn, file_ids)) - if not all_downloads: - return + # Write files to local cache (path relative to syftbox_folder) + for (collection, metadata), content in zip( + all_downloads, downloaded_contents + ): + owner_email = collection["owner_email"] + tag = collection["tag"] + file_name = metadata["file_name"] + rel_path = f"{owner_email}/{spec.local_subpath}/{tag}/{file_name}" + self.file_connection.write_file(rel_path, content) - # Download all files from all collections in parallel - file_ids = [metadata["file_id"] for _, metadata in all_downloads] - downloaded_contents = list(executor.map(download_fn, file_ids)) - - # Write files to local cache (path relative to syftbox_folder) - for (collection, metadata), content in zip(all_downloads, downloaded_contents): - owner_email = collection["owner_email"] - tag = collection["tag"] - file_name = metadata["file_name"] - rel_path = f"{owner_email}/{self.collection_subpath}/{tag}/{file_name}" - self.file_connection.write_file(rel_path, content) - - # Update hash cache for all collections - for collection in collections_to_update: - collection_path = self.get_collection_path( - collection["owner_email"], collection["tag"] - ) - if collection_path is not None: - self.dataset_collection_hashes[collection_path] = collection[ - "content_hash" - ] + # Update hash cache for all collections in this spec + for collection in collections_to_update: + collection_path = self.get_collection_path( + collection["owner_email"], collection["tag"], spec.local_subpath + ) + if collection_path is not None: + self.collection_hashes[collection_path] = collection["content_hash"] diff --git a/syft_client/sync/sync/collection_spec.py b/syft_client/sync/sync/collection_spec.py new file mode 100644 index 00000000000..1d17f67949b --- /dev/null +++ b/syft_client/sync/sync/collection_spec.py @@ -0,0 +1,56 @@ +"""The collection-sync primitive. + +``CollectionSyncSpec`` is the domain-free contract that tells the generic sync +engine HOW to sync a "collection" (a named, content-hashed folder like a dataset's +mock or private data) — WITHOUT the engine knowing what a dataset is. The concrete +specs are supplied by the domain layer (e.g. ``syft_rds.config.DATASET_COLLECTION_SPECS``). +""" + +from pathlib import Path + +from pydantic import BaseModel + + +class CollectionSyncSpec(BaseModel): + prefix: str + # Subpath from owner_email to the collection folder, + # e.g. Path("public/syft_datasets") + local_subpath: Path + # Download authority. False = mirror: re-download when the remote content-hash + # changes (the remote is the source of truth, e.g. published mock data). + # True = restore-only: download only when absent locally and never overwrite the + # local copy (the local copy is the source of truth, e.g. the owner's real data). + immutable: bool = False + # Sharing policy. False = shareable: a peer's watcher may pull it (e.g. mock data). + # True = owner-only: never shared with peers; the owner restores it for itself and + # peer-facing watchers skip it entirely (e.g. the owner's private data backup). + owner_only: bool = False + + @classmethod + def public(cls, prefix: str, local_subpath: "Path") -> "CollectionSyncSpec": + """A shareable, mirrored collection (e.g. a dataset's mock data). + + Peers' watchers pull it, and it re-downloads whenever the owner + republishes changed content. + """ + return cls( + prefix=prefix, + local_subpath=local_subpath, + immutable=False, + owner_only=False, + ) + + @classmethod + def private(cls, prefix: str, local_subpath: "Path") -> "CollectionSyncSpec": + """An owner-only, restore-only collection (e.g. a dataset's real data). + + Never shared with peers, peer-facing watchers skip it, and it is only + restored to the owner when absent locally (the local copy is authoritative + and is never overwritten by the backup). + """ + return cls( + prefix=prefix, + local_subpath=local_subpath, + immutable=True, + owner_only=True, + ) diff --git a/syft_client/sync/sync/datasite_owner_syncer.py b/syft_client/sync/sync/datasite_owner_syncer.py index 81e6b25241b..c16cfcdc489 100644 --- a/syft_client/sync/sync/datasite_owner_syncer.py +++ b/syft_client/sync/sync/datasite_owner_syncer.py @@ -1,6 +1,7 @@ import logging from pathlib import Path from uuid import uuid4 +from functools import partial from pydantic import ConfigDict, Field, BaseModel, PrivateAttr from concurrent.futures import ThreadPoolExecutor @@ -20,6 +21,7 @@ ) from syft_client.sync.connections.connection_router import ConnectionRouter from syft_client.sync.sync.caches.datasite_owner_cache import DataSiteOwnerEventCache +from syft_client.sync.sync.collection_spec import CollectionSyncSpec from syft_client.sync.callback_mixin import BaseModelCallbackMixin from syft_client.sync.messages.proposed_filechange import ProposedFileChangesMessage from syft_client.sync.utils.path_filters import is_normal_syncable_path @@ -53,6 +55,9 @@ class DatasiteOwnerSyncerConfig(BaseModel): write_files: bool = True # Full path to collections folder - must be provided explicitly collections_folder: Path | None = None + # Collection sync specs (public prefix + local subpath). Empty for a bare + # sync engine; the domain layer (e.g. syft-rds) supplies the concrete specs. + collection_specs: List[CollectionSyncSpec] = [] cache_config: DataSiteOwnerEventCacheConfig = Field( default_factory=DataSiteOwnerEventCacheConfig ) @@ -74,6 +79,8 @@ class DatasiteOwnerSyncer(BaseModelCallbackMixin): perm_context: SyftPermContext # Full path to collections folder collections_folder: Path | None = None + # Collection sync specs (public prefix + local subpath), supplied by rds. + collection_specs: List[CollectionSyncSpec] = [] syftbox_events_queue: Queue[FileChangeEventsMessage] = Field(default_factory=Queue) outbox_queue: Queue[Tuple[str, FileChangeEventsMessage]] = Field( @@ -83,8 +90,8 @@ class DatasiteOwnerSyncer(BaseModelCallbackMixin): _executor: ThreadPoolExecutor = PrivateAttr( default_factory=lambda: ThreadPoolExecutor(max_workers=10) ) - # Cache of datasets shared with "any" - list of (tag, content_hash) tuples - _any_shared_datasets: List[tuple] = PrivateAttr(default_factory=list) + # Cache of collections shared with "any" - list of (tag, content_hash) tuples + _any_shared_collections: List[tuple] = PrivateAttr(default_factory=list) # Cache of read permissions per file path → frozenset of peer emails _read_perm_cache: dict[str, frozenset[str]] = PrivateAttr(default_factory=dict) @@ -112,6 +119,7 @@ def from_config(cls, config: DatasiteOwnerSyncerConfig): syftbox_folder=config.syftbox_folder, perm_context=SyftPermContext(datasite=datasite), collections_folder=config.collections_folder, + collection_specs=config.collection_specs, ) @property @@ -289,11 +297,10 @@ def pull_initial_state(self): if restored_events and not self.event_cache.get_cached_events(): self._write_events_to_messages_cache(restored_events) - # Load datasets from connection and populate _any_shared_datasets cache - self._pull_datasets_for_initial_sync() - - # Restore private datasets from GDrive (owner-only collections) - self._pull_private_datasets_for_initial_sync() + # Restore ALL registered collections (public + private) generically. Each + # spec's flags decide the behaviour: `immutable` picks mirror vs restore-only; + # the local destination comes from the spec's local_subpath. + self._pull_collections_for_initial_sync() self.initial_sync_done = True @@ -316,43 +323,79 @@ def _apply_incremental_checkpoint_to_cache( str(event.path_in_datasite), event.content ) - def _pull_datasets_for_initial_sync(self): - """Load datasets from GDrive when DO connects. + def _collection_local_dir(self, spec: CollectionSyncSpec, tag: str) -> Path: + """Local directory a collection with the given tag restores to.""" + return self.syftbox_folder / self.email / spec.local_subpath / tag + + def _pull_collections_for_initial_sync(self): + """Restore the owner's own collections from the sync backend on connect. - Restores datasets to local filesystem and populates the _any_shared_datasets cache. + Iterates every registered spec (public, private, or any future kind). For + each it lists the owner's collections, refreshes the _any_shared_collections + cache, filters by the spec's download policy, and restores to disk. """ - collections = ( - self.connection_router.owner_list_all_dataset_collections_with_permissions() - ) + for spec in self.collection_specs: + collections = ( + self.connection_router.owner_list_all_collections_with_permissions( + spec.prefix + ) + ) + if not collections: + continue + + self._update_any_shared_collections_cache(collections) + + collections_to_download = self._filter_collections_needing_download( + collections, spec + ) + self._download_collections_parallel(collections_to_download, spec) + + @property + def any_shared_collections(self) -> List[tuple]: + """Collections shared with "any" as (tag, content_hash) pairs. - self._update_any_shared_datasets_cache(collections) + Read-only view: mutate via ``register_any_shared_collection``. + """ + return list(self._any_shared_collections) - collections_to_download = self._filter_collections_needing_download(collections) - self._download_dataset_collections_parallel(collections_to_download) + def register_any_shared_collection(self, tag: str, content_hash: str) -> None: + """Record a collection as shared-with-"any" (deduplicated).""" + entry = (tag, content_hash) + if entry not in self._any_shared_collections: + self._any_shared_collections.append(entry) - def _update_any_shared_datasets_cache(self, collections: list[FileCollection]): - """Populate _any_shared_datasets cache from collections with 'any' permission.""" + def _update_any_shared_collections_cache(self, collections: list[FileCollection]): + """Populate the any-shared cache from collections with 'any' permission.""" for collection in collections: if collection.has_any_permission: - entry = (collection.tag, collection.content_hash) - if entry not in self._any_shared_datasets: - self._any_shared_datasets.append(entry) + self.register_any_shared_collection( + collection.tag, collection.content_hash + ) def _filter_collections_needing_download( - self, collections: list[FileCollection] + self, collections: list[FileCollection], spec: CollectionSyncSpec ) -> list[FileCollection]: - """Return collections that don't exist locally or have different content hash.""" + """Return the collections that need downloading, per the spec's policy. + + * immutable (restore-only): download only when absent locally — the local + copy is authoritative and must never be overwritten by the backup. + * mutable (mirror): download when the remote content-hash differs from ours. + """ from syft_client.sync.file_utils import compute_directory_hash result = [] for collection in collections: - # Use cached hash from event_cache first + local_dir = self._collection_local_dir(spec, collection.tag) + + if spec.immutable: + if not local_dir.exists() or not any(local_dir.iterdir()): + result.append(collection) + continue + + # Mirror: use the cached hash first, else compute it from local files. cached_hash = self.event_cache.get_collection_hash(collection.tag) - if cached_hash is None and self.collections_folder is not None: - # Fallback: compute hash from local filesystem (for locally created datasets) - local_dataset_dir = self.collections_folder / collection.tag - cached_hash = compute_directory_hash(local_dataset_dir) - # Update cache if we computed a hash + if cached_hash is None and local_dir.exists(): + cached_hash = compute_directory_hash(local_dir) if cached_hash is not None: self.event_cache.set_collection_hash(collection.tag, cached_hash) @@ -360,7 +403,9 @@ def _filter_collections_needing_download( result.append(collection) return result - def _download_dataset_collections_parallel(self, collections: list[FileCollection]): + def _download_collections_parallel( + self, collections: list[FileCollection], spec: CollectionSyncSpec + ): """Download all files from collections in parallel and write to disk.""" if not collections: return @@ -368,7 +413,10 @@ def _download_dataset_collections_parallel(self, collections: list[FileCollectio # Fetch file metadatas for all collections in parallel all_file_metadatas = list( self._executor.map( - self._get_file_metadatas_with_new_connection, collections + partial( + self._get_file_metadatas_with_new_connection, prefix=spec.prefix + ), + collections, ) ) @@ -388,24 +436,36 @@ def _download_dataset_collections_parallel(self, collections: list[FileCollectio self._executor.map(self._download_file_with_new_connection, file_ids) ) - # Write all files to disk + # Write all files to disk under the spec's local subpath. for (collection, metadata), content in zip(all_downloads, downloaded_contents): - local_dataset_dir = self.collections_folder / collection.tag - local_dataset_dir.mkdir(parents=True, exist_ok=True) - (local_dataset_dir / metadata["file_name"]).write_bytes(content) + local_dir = self._collection_local_dir(spec, collection.tag) + local_dir.mkdir(parents=True, exist_ok=True) + (local_dir / metadata["file_name"]).write_bytes(content) + + # Update cached hashes ONLY for mutable specs + if not spec.immutable: + for collection in collections: + self.event_cache.set_collection_hash( + collection.tag, collection.content_hash + ) - # Update cached hashes for downloaded collections + # Notify the domain layer (e.g. syft-rds) that these collections were restored, + # so it can run any collection-specific post-processing for collection in collections: - self.event_cache.set_collection_hash( - collection.tag, collection.content_hash + self._emit( + "collection_restored", + spec.prefix, + collection.tag, + self._collection_local_dir(spec, collection.tag), ) def _get_file_metadatas_with_new_connection( - self, collection: FileCollection + self, collection: FileCollection, prefix: str ) -> list: """Get file metadatas for a collection using a new connection for thread safety.""" connection = self.connection_router.connection_for_parallel_download() - return connection.watcher_get_dataset_collection_file_metadatas( + return connection.watcher_get_collection_file_metadatas( + prefix, tag=collection.tag, content_hash=collection.content_hash, owner_email=self.email, @@ -414,100 +474,7 @@ def _get_file_metadatas_with_new_connection( def _download_file_with_new_connection(self, file_id: str) -> bytes: """Download a file using a new connection for thread safety.""" connection = self.connection_router.connection_for_parallel_download() - return connection.watcher_download_dataset_file(file_id) - - # ========================================================================= - # PRIVATE DATASET RESTORE METHODS - # ========================================================================= - - def _pull_private_datasets_for_initial_sync(self): - """Restore private datasets from GDrive when DO reconnects. - - Downloads private data from owner-only collection folders - to {syftbox_folder}/private/syft_datasets/{tag}/. - """ - collections = self.connection_router.owner_list_private_dataset_collections() - if not collections: - return - - collections_to_download = self._filter_private_collections_needing_download( - collections - ) - self._download_private_collections_parallel(collections_to_download) - - def _private_dataset_local_dir(self, tag: str) -> Path: - return self.syftbox_folder / self.email / "private" / "syft_datasets" / tag - - def _filter_private_collections_needing_download( - self, collections: list[FileCollection] - ) -> list[FileCollection]: - """Return private collections that don't exist locally yet.""" - result = [] - for collection in collections: - local_dir = self._private_dataset_local_dir(collection.tag) - if not local_dir.exists() or not any(local_dir.iterdir()): - result.append(collection) - return result - - def _download_private_collections_parallel(self, collections: list[FileCollection]): - """Download private collection files in parallel and write to disk.""" - if not collections: - return - - all_file_metadatas = list( - self._executor.map( - self._get_private_file_metadatas_with_new_connection, collections - ) - ) - - all_downloads = [ - (collection, metadata) - for collection, file_metadatas in zip(collections, all_file_metadatas) - for metadata in file_metadatas - ] - - if not all_downloads: - return - - file_ids = [metadata["file_id"] for _, metadata in all_downloads] - downloaded_contents = list( - self._executor.map(self._download_file_with_new_connection, file_ids) - ) - - for (collection, metadata), content in zip(all_downloads, downloaded_contents): - local_dir = self._private_dataset_local_dir(collection.tag) - local_dir.mkdir(parents=True, exist_ok=True) - (local_dir / metadata["file_name"]).write_bytes(content) - - # Fix data_dir in private_metadata.yaml to point to current local path - for collection in collections: - self._fix_private_metadata_data_dir(collection.tag) - - def _get_private_file_metadatas_with_new_connection( - self, collection: FileCollection - ) -> list: - """Get file metadatas for a private collection using a new connection.""" - connection = self.connection_router.connection_for_parallel_download() - return connection.owner_get_private_collection_file_metadatas( - tag=collection.tag, - content_hash=collection.content_hash, - owner_email=self.email, - ) - - def _fix_private_metadata_data_dir(self, dataset_tag: str): - """Update data_dir in private_metadata.yaml to match the current syftbox path.""" - local_dir = self._private_dataset_local_dir(dataset_tag) - metadata_path = local_dir / "private_metadata.yaml" - if not metadata_path.exists(): - return - - import yaml - - data = yaml.safe_load(metadata_path.read_text()) - expected_dir = str(local_dir) - if data.get("data_dir") != expected_dir: - data["data_dir"] = expected_dir - metadata_path.write_text(yaml.safe_dump(data, indent=2, sort_keys=False)) + return connection.watcher_download_collection_file(file_id) def _get_readers(self, path: str, recipients: list[str]) -> frozenset[str]: """Return the set of recipients that have read access to the given path.""" diff --git a/syft_client/sync/sync/datasite_watcher_syncer.py b/syft_client/sync/sync/datasite_watcher_syncer.py index 85e9f279081..a26c064310d 100644 --- a/syft_client/sync/sync/datasite_watcher_syncer.py +++ b/syft_client/sync/sync/datasite_watcher_syncer.py @@ -135,12 +135,12 @@ def download_events_message_with_new_connection( ) return FileChangeEventsMessage.from_compressed_data(raw) - def download_dataset_file_with_new_connection( + def download_collection_file_with_new_connection( self, file_id: str, owner_email: str ) -> bytes: - """Download dataset file using a new connection (thread-safe).""" + """Download collection file using a new connection (thread-safe).""" connection = self.connection_router.connection_for_parallel_download() - data = connection.watcher_download_dataset_file(file_id) + data = connection.watcher_download_collection_file(file_id) data = self.connection_router.peer_store.decrypt_dataset_if_needed( owner_email, data ) @@ -159,10 +159,12 @@ def sync_down(self, peer_emails: list[str]): ) if event_count: print(f"Pulled {event_count} inbound event(s) from {peer_email}") - # Sync datasets with parallel download - self.datasite_watcher_cache.sync_down_datasets_parallel( + # Sync collections with parallel download + self.datasite_watcher_cache.sync_down_collections_parallel( peer_email, self._executor, lambda fid, - pe=peer_email: self.download_dataset_file_with_new_connection(fid, pe), + pe=peer_email: self.download_collection_file_with_new_connection( + fid, pe + ), ) diff --git a/tests/integration/with_unit_coverage/test_sync_manager.py b/tests/integration/with_unit_coverage/test_sync_manager.py index 9e509060e32..520b9614a1f 100644 --- a/tests/integration/with_unit_coverage/test_sync_manager.py +++ b/tests/integration/with_unit_coverage/test_sync_manager.py @@ -14,14 +14,19 @@ """ from syft_datasets import Dataset +from syft_datasets.dataset_manager import DATASET_COLLECTION_PREFIX import os from pathlib import Path import time import json from time import sleep import pytest -from tests.unit.utils import create_tmp_dataset_files -from tests.integration.with_unit_coverage.utils import create_test_pair, is_mock_mode +from tests.unit.utils import create_tmp_dataset_files, grant_job_inbox_access +from tests.integration.with_unit_coverage.utils import ( + create_rds_test_pair, + create_test_pair, + is_mock_mode, +) def path_for_job(do_email: str, ds_email: str, filename: str = "test.job") -> str: @@ -52,6 +57,8 @@ def test_google_drive_connection_syncing(): ds_token_path=token_path_ds, ) + grant_job_inbox_access(manager_do, EMAIL_DS) + # this calls connection.send_propose_file_change_message via callbacks if not is_mock_mode(): sleep(1) @@ -131,7 +138,7 @@ def test_google_drive_files(): @pytest.mark.flaky(reruns=3, reruns_delay=2) @pytest.mark.usefixtures("setup_delete_syftboxes") def test_datasets(): - ds_manager, do_manager = create_test_pair( + ds_manager, do_manager = create_rds_test_pair( do_email=EMAIL_DO, ds_email=EMAIL_DS, do_token_path=token_path_do, @@ -181,7 +188,7 @@ def has_file(root_dir, filename): def test_datasets_shared_with_any(): """Test that datasets shared with 'any' become discoverable after peer approval.""" # Create managers WITHOUT auto peer setup - ds_manager, do_manager = create_test_pair( + ds_manager, do_manager = create_rds_test_pair( do_email=EMAIL_DO, ds_email=EMAIL_DS, do_token_path=token_path_do, @@ -204,7 +211,9 @@ def test_datasets_shared_with_any(): ) # DS should NOT see the dataset yet (not approved) - ds_collections = ds_manager._connection_router.watcher_list_dataset_collections() + ds_collections = ds_manager.sync_engine._connection_router.watcher_list_collections( + DATASET_COLLECTION_PREFIX + ) assert not any(c["tag"] == "any dataset" for c in ds_collections) # DS adds peer, DO approves (this should share 'any' datasets) @@ -215,14 +224,16 @@ def test_datasets_shared_with_any(): # DS should now see the dataset # Google Drive search index takes time to propagate permission changes time.sleep(5) - ds_collections = ds_manager._connection_router.watcher_list_dataset_collections() + ds_collections = ds_manager.sync_engine._connection_router.watcher_list_collections( + DATASET_COLLECTION_PREFIX + ) assert any(c["tag"] == "any dataset" for c in ds_collections) @pytest.mark.flaky(reruns=3, reruns_delay=2) @pytest.mark.usefixtures("setup_delete_syftboxes") def test_jobs(): - ds_manager, do_manager = create_test_pair( + ds_manager, do_manager = create_rds_test_pair( do_email=EMAIL_DO, ds_email=EMAIL_DS, do_token_path=token_path_do, @@ -341,7 +352,7 @@ def test_google_drive_connection_load_state(): # create the state # load the clients and add the peers - manager_ds1, manager_do1 = create_test_pair( + manager_ds1, manager_do1 = create_rds_test_pair( do_email=EMAIL_DO, ds_email=EMAIL_DS, do_token_path=token_path_do, @@ -350,11 +361,13 @@ def test_google_drive_connection_load_state(): load_peers=False, ) + grant_job_inbox_access(manager_do1, EMAIL_DS) + # make some changes - manager_ds1._send_file_change( + manager_ds1.sync_engine._send_file_change( path_for_job(EMAIL_DO, EMAIL_DS, "my.job"), "Hello, world!" ) - manager_ds1._send_file_change( + manager_ds1.sync_engine._send_file_change( path_for_job(EMAIL_DO, EMAIL_DS, "my_second.job"), "Hello, world!" ) @@ -372,10 +385,12 @@ def test_google_drive_connection_load_state(): # verify dataset was created and cache was populated assert len(manager_do1.datasets.get_all()) == 1 - assert len(manager_do1.datasite_owner_syncer._any_shared_datasets) == 1 + assert ( + len(manager_do1.sync_engine.datasite_owner_syncer.any_shared_collections) == 1 + ) # test loading the peers and loading the inbox - manager_ds2, manager_do2 = create_test_pair( + manager_ds2, manager_do2 = create_rds_test_pair( do_email=EMAIL_DO, ds_email=EMAIL_DS, do_token_path=token_path_do, @@ -393,11 +408,16 @@ def test_google_drive_connection_load_state(): # sync so we have something in the syftbox and do outbox manager_do2.sync() - assert len(manager_do2.datasite_owner_syncer.event_cache.get_cached_events()) == 2 + assert ( + len( + manager_do2.sync_engine.datasite_owner_syncer.event_cache.get_cached_events() + ) + == 2 + ) # we have created some state now, so now we can log in again and load the state # use a fresh syftbox folder to simulate clean filesystem - manager_ds3, manager_do3 = create_test_pair( + manager_ds3, manager_do3 = create_rds_test_pair( do_email=EMAIL_DO, ds_email=EMAIL_DS, do_token_path=token_path_do, @@ -409,12 +429,12 @@ def test_google_drive_connection_load_state(): manager_do3.sync() manager_ds3.sync() - loaded_events_do = manager_do3.datasite_owner_syncer.event_cache.get_cached_events() + loaded_events_do = ( + manager_do3.sync_engine.datasite_owner_syncer.event_cache.get_cached_events() + ) assert len(loaded_events_do) == 2 - loaded_events_ds = ( - manager_ds3.datasite_watcher_syncer.datasite_watcher_cache.get_cached_events() - ) + loaded_events_ds = manager_ds3.sync_engine.datasite_watcher_syncer.datasite_watcher_cache.get_cached_events() assert len(loaded_events_ds) == 2 # verify datasets were loaded from GDrive @@ -422,10 +442,12 @@ def test_google_drive_connection_load_state(): assert len(loaded_datasets) == 1 assert loaded_datasets[0].name == "load_state_dataset" - # verify _any_shared_datasets cache was populated during pull_initial_state - assert len(manager_do3.datasite_owner_syncer._any_shared_datasets) == 1 + # verify any_shared_collections cache was populated during pull_initial_state + assert ( + len(manager_do3.sync_engine.datasite_owner_syncer.any_shared_collections) == 1 + ) assert ( - manager_do3.datasite_owner_syncer._any_shared_datasets[0][0] + manager_do3.sync_engine.datasite_owner_syncer.any_shared_collections[0][0] == "load_state_dataset" ) diff --git a/tests/integration/with_unit_coverage/utils.py b/tests/integration/with_unit_coverage/utils.py index 8da03dca28c..82c12f7689f 100644 --- a/tests/integration/with_unit_coverage/utils.py +++ b/tests/integration/with_unit_coverage/utils.py @@ -2,6 +2,7 @@ from pathlib import Path from syft_client.sync.syftbox_manager import SyftboxManager +from syft_rds import SyftRDSClient def is_mock_mode() -> bool: @@ -38,3 +39,36 @@ def create_test_pair( clear_caches=clear_caches, check_versions=check_versions, ) + + +def create_rds_test_pair( + do_email: str, + ds_email: str, + do_token_path: Path | None = None, + ds_token_path: Path | None = None, + add_peers: bool = True, + load_peers: bool = False, + use_in_memory_cache: bool = True, + clear_caches: bool = True, + check_versions: bool = False, +): + """create_test_pair for tests that exercise datasets or jobs.""" + if is_mock_mode(): + return SyftRDSClient.pair_with_mock_drive_service_connection( + email1=do_email, + email2=ds_email, + add_peers=add_peers, + use_in_memory_cache=use_in_memory_cache, + check_versions=check_versions, + ) + return SyftRDSClient._pair_with_google_drive_testing_connection( + do_email=do_email, + ds_email=ds_email, + do_token_path=do_token_path, + ds_token_path=ds_token_path, + add_peers=add_peers, + load_peers=load_peers, + use_in_memory_cache=use_in_memory_cache, + clear_caches=clear_caches, + check_versions=check_versions, + ) diff --git a/tests/integration/without_unit_coverage/test_login.py b/tests/integration/without_unit_coverage/test_login.py index 5f4d82b16cd..dc410c0b0dc 100644 --- a/tests/integration/without_unit_coverage/test_login.py +++ b/tests/integration/without_unit_coverage/test_login.py @@ -11,6 +11,7 @@ import pytest import syft_client as sc +import syft_rds as rds from syft_client.sync.syftbox_manager import SyftboxManager @@ -94,9 +95,10 @@ def test_login_ds_with_sync_and_load_peers(): assert client is not None assert isinstance(client, SyftboxManager) assert client.email == EMAIL_DS - # Verify client has expected attributes - assert hasattr(client, "datasets") - assert hasattr(client, "job_client") + # syft-client's login returns the domain-free sync engine + assert hasattr(client, "peers") + assert not hasattr(client, "datasets") + assert not hasattr(client, "job_client") @pytest.mark.flaky(reruns=3, reruns_delay=2) @@ -113,9 +115,45 @@ def test_login_do_with_sync_and_load_peers(): assert client is not None assert isinstance(client, SyftboxManager) assert client.email == EMAIL_DO - # Verify client has expected attributes + # syft-client's login returns the domain-free sync engine + assert hasattr(client, "peers") + assert not hasattr(client, "datasets") + assert not hasattr(client, "job_client") + + +@pytest.mark.flaky(reruns=3, reruns_delay=2) +@pytest.mark.usefixtures("check_credentials") +def test_rds_login_ds_returns_product_client(): + """syft_rds.login_ds returns an RDS client carrying the dataset/job surface.""" + client = rds.login_ds( + email=EMAIL_DS, + token_path=token_path_ds, + sync=True, + load_peers=True, + ) + + assert isinstance(client, rds.SyftRDSClient) + assert client.email == EMAIL_DS + assert hasattr(client, "datasets") + assert hasattr(client, "job_client") + + +@pytest.mark.flaky(reruns=3, reruns_delay=2) +@pytest.mark.usefixtures("check_credentials") +def test_rds_login_do_returns_product_client(): + """syft_rds.login_do returns an RDS client carrying the dataset/job surface.""" + client = rds.login_do( + email=EMAIL_DO, + token_path=token_path_do, + sync=True, + load_peers=True, + ) + + assert isinstance(client, rds.SyftRDSClient) + assert client.email == EMAIL_DO assert hasattr(client, "datasets") assert hasattr(client, "job_client") + assert client.job_runner is not None @pytest.mark.flaky(reruns=3, reruns_delay=2) diff --git a/tests/unit/syft_bg/test_email_approval_flow.py b/tests/unit/syft_bg/test_email_approval_flow.py index f1a6240d2ca..7fecde67bc8 100644 --- a/tests/unit/syft_bg/test_email_approval_flow.py +++ b/tests/unit/syft_bg/test_email_approval_flow.py @@ -17,7 +17,7 @@ from syft_bg.notify.handlers.job import JobHandler from syft_bg.notify.monitors.job import JobMonitor from syft_bg.notify.orchestrator import NotificationOrchestrator -from syft_client.sync.syftbox_manager import SyftboxManager +from syft_rds import SyftRDSClient from tests.unit.utils import create_test_project_folder, create_tmp_dataset_files @@ -25,7 +25,7 @@ def _make_notify_orchestrator( - do_manager: SyftboxManager, + do_manager: SyftRDSClient, tmp: Path, ) -> tuple[NotificationOrchestrator, JsonStateManager, MagicMock]: """Create a NotificationOrchestrator with a mocked GmailSender. @@ -64,7 +64,7 @@ def _make_notify_orchestrator( def _make_email_approve_orchestrator( - do_manager: SyftboxManager, + do_manager: SyftRDSClient, notify_state: JsonStateManager, tmp: Path, ) -> tuple[EmailApproveOrchestrator, EmailApproveMonitor, JsonStateManager, MagicMock]: @@ -132,7 +132,7 @@ def test_email_approval_e2e(): """Full flow: DS submits job -> notify detects it -> email approval -> DS gets results.""" # -- Step 1: Create DO/DS clients -- - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) diff --git a/tests/unit/syft_bg/test_email_auto_approve_flow.py b/tests/unit/syft_bg/test_email_auto_approve_flow.py index 0fb0fed921c..6593b774320 100644 --- a/tests/unit/syft_bg/test_email_auto_approve_flow.py +++ b/tests/unit/syft_bg/test_email_auto_approve_flow.py @@ -20,7 +20,7 @@ from syft_bg.notify.handlers.job import JobHandler from syft_bg.notify.monitors.job import JobMonitor from syft_bg.notify.orchestrator import NotificationOrchestrator -from syft_client.sync.syftbox_manager import SyftboxManager +from syft_rds import SyftRDSClient FAKE_THREAD_ID = "thread_auto_approve_123" @@ -44,7 +44,7 @@ def _temp_config_paths(): def _make_notify_orchestrator( - do_manager: SyftboxManager, + do_manager: SyftRDSClient, tmp: Path, ) -> tuple[NotificationOrchestrator, JsonStateManager, MagicMock]: """Create a NotificationOrchestrator with a mocked GmailSender.""" @@ -81,7 +81,7 @@ def _make_notify_orchestrator( def _make_email_approve_orchestrator( - do_manager: SyftboxManager, + do_manager: SyftRDSClient, notify_state: JsonStateManager, tmp: Path, reply_text: str = "auto-approve", @@ -146,7 +146,7 @@ def _create_project_code_files_with_json_contents(params: dict) -> Path: def test_email_auto_approve_creates_object_and_approves_future_jobs(): """Full flow: auto-approve reply creates approval object, second job is auto-approved.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( use_in_memory_cache=False, sync_automatically=False, ) diff --git a/tests/unit/test_cache_aware_sync.py b/tests/unit/test_cache_aware_sync.py index 2d3dbb1ffa7..84f8877a429 100644 --- a/tests/unit/test_cache_aware_sync.py +++ b/tests/unit/test_cache_aware_sync.py @@ -7,7 +7,13 @@ from pathlib import Path from syft_client.sync.syftbox_manager import SyftboxManager -from tests.unit.utils import get_mock_events_messages +from syft_client.sync.sync.collection_spec import CollectionSyncSpec + +from tests.unit.utils import ( + TEST_COLLECTION_PREFIX, + TEST_COLLECTION_SUBPATH, + get_mock_events_messages, +) def test_do_incremental_sync_downloads_only_new_events(): @@ -173,9 +179,7 @@ def test_do_cache_handles_deletions_correctly(): DataSiteOwnerEventCacheConfig, ) - from syft_client.sync.syftbox_manager import COLLECTION_SUBPATH - - collections_folder = syftbox_folder / do_manager.email / COLLECTION_SUBPATH + collections_folder = syftbox_folder / do_manager.email / TEST_COLLECTION_SUBPATH config = DataSiteOwnerEventCacheConfig( use_in_memory_cache=False, syftbox_folder=syftbox_folder, @@ -233,13 +237,13 @@ def test_ds_cache_handles_deletions_correctly(): DataSiteWatcherCacheConfig, ) - from syft_client.sync.syftbox_manager import COLLECTION_SUBPATH - config = DataSiteWatcherCacheConfig( email=do_manager.email, use_in_memory_cache=False, syftbox_folder=syftbox_folder, - collection_subpath=COLLECTION_SUBPATH, + collection_specs=[ + CollectionSyncSpec.public(TEST_COLLECTION_PREFIX, TEST_COLLECTION_SUBPATH) + ], connection_configs=[], ) new_cache = DataSiteWatcherCache.from_config(config) diff --git a/tests/unit/test_delete_syftbox.py b/tests/unit/test_delete_syftbox.py index 5caf0ec227e..fffa08e8439 100644 --- a/tests/unit/test_delete_syftbox.py +++ b/tests/unit/test_delete_syftbox.py @@ -11,11 +11,6 @@ from syft_client.sync.login_utils import handle_potential_version_mismatches_on_login from syft_client.sync.syftbox_manager import SyftboxManager from syft_client.sync.version.version_info import VersionInfo -from syft_datasets.dataset_manager import ( - DATASET_COLLECTION_PREFIX, - PRIVATE_DATASET_COLLECTION_PREFIX, -) -from tests.unit.utils import create_tmp_dataset_files EMAIL = "test@example.com" @@ -105,18 +100,6 @@ def test_delete_unversioned_state_removes_correct_folders(): sync_automatically=False, encryption=True, ) - - # Create dataset so collection folders exist - mock_path, private_path, readme_path = create_tmp_dataset_files() - do_manager.create_dataset( - name="my dataset", - mock_path=mock_path, - private_path=private_path, - summary="Test", - readme_path=readme_path, - users=[ds_manager.email], - upload_private=True, - ) do_manager.sync() do_conn = do_manager.peer_manager.connection_router.connections[0] @@ -125,8 +108,6 @@ def test_delete_unversioned_state_removes_correct_folders(): # Assert artifacts exist before deletion do_enc_bundles = f"syft_encryption_bundles#{do_email}" assert len(_query_files(do_conn, do_enc_bundles)) > 0 - assert len(_query_files(do_conn, DATASET_COLLECTION_PREFIX)) > 0 - assert len(_query_files(do_conn, PRIVATE_DATASET_COLLECTION_PREFIX)) > 0 assert len(_query_files(do_conn, SYFT_PEERS_FILE)) > 0 assert len(_query_files(do_conn, SYFT_VERSION_FILE)) > 0 @@ -139,8 +120,6 @@ def test_delete_unversioned_state_removes_correct_folders(): # Assert unversioned artifacts are gone assert len(_query_files(do_conn, do_enc_bundles)) == 0 - assert len(_query_files(do_conn, DATASET_COLLECTION_PREFIX)) == 0 - assert len(_query_files(do_conn, PRIVATE_DATASET_COLLECTION_PREFIX)) == 0 # peers/version files: DO's are gone, DS's may still exist do_peers = [ f diff --git a/tests/unit/test_dir_autocomplete.py b/tests/unit/test_dir_autocomplete.py index 22f7d80ec0f..60377617717 100644 --- a/tests/unit/test_dir_autocomplete.py +++ b/tests/unit/test_dir_autocomplete.py @@ -10,11 +10,17 @@ def test_dir_returns_only_public_api(): assert "email" in public_names assert "sync" in public_names assert "peers" in public_names - assert "jobs" in public_names - assert "datasets" in public_names assert "add_peer" in public_names - assert "create_dataset" in public_names - assert "submit_python_job" in public_names + assert "approve_peer_request" in public_names + assert "delete_syftbox" in public_names + + # Domain-free: no dataset/job surface + assert "jobs" not in public_names + assert "datasets" not in public_names + assert "dataset_manager" not in public_names + assert "create_dataset" not in public_names + assert "submit_python_job" not in public_names + assert "process_approved_jobs" not in public_names # Hides Pydantic internals assert "model_dump" not in public_names diff --git a/tests/unit/test_encryption.py b/tests/unit/test_encryption.py index eaaa4547c94..1e1045603bf 100644 --- a/tests/unit/test_encryption.py +++ b/tests/unit/test_encryption.py @@ -7,7 +7,7 @@ from syft_client.sync.peers.peer_store import PeerStore from syft_client.sync.syftbox_manager import SyftboxManager from tests.unit.test_sync_manager import path_for_job -from tests.unit.utils import create_tmp_dataset_files +from tests.unit.utils import grant_job_inbox_access # ========================================================================= @@ -148,6 +148,8 @@ def test_encrypted_message_flow(): encryption=True, ) + grant_job_inbox_access(do_manager, ds_manager.email) + # Verify peer stores are set with encryption enabled assert ds_manager._peer_store is not None assert do_manager._peer_store is not None @@ -182,6 +184,8 @@ def test_encrypted_sync_down_ds(): encryption=True, ) + grant_job_inbox_access(do_manager, ds_manager.email) + # DS needs DO's bundle to decrypt ds_manager.load_peers() @@ -208,6 +212,8 @@ def test_no_encryption_backward_compat(): encryption=False, ) + grant_job_inbox_access(do_manager, ds_manager.email) + assert ds_manager._peer_store is None assert do_manager._peer_store is None @@ -308,6 +314,8 @@ def test_encrypted_events_at_rest(): ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( encryption=True, ) + + grant_job_inbox_access(do_manager, ds_manager.email) ds_manager.load_peers() # DS sends a file change @@ -328,6 +336,8 @@ def test_encrypted_checkpoint_roundtrip(): ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( encryption=True, ) + + grant_job_inbox_access(do_manager, ds_manager.email) ds_manager.load_peers() # Create some data and a checkpoint @@ -387,6 +397,8 @@ def test_at_rest_no_encryption_backward_compat(): encryption=False, ) + grant_job_inbox_access(do_manager, ds_manager.email) + file_path = path_for_job(do_manager.email, ds_manager.email, "my.job") ds_manager._send_file_change(file_path, "unencrypted data") do_manager.sync() @@ -483,6 +495,8 @@ def test_verified_encrypted_message_flow(): ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( encryption=True, ) + + grant_job_inbox_access(do_manager, ds_manager.email) ds_manager.load_peers() # DS sends a file change (encrypted + signed) @@ -525,36 +539,3 @@ def test_tampered_inbox_message_raises(): # DO sync should fail due to signature verification with pytest.raises(Exception): do_manager.sync() - - -def test_encrypted_dataset_collection_syncs(): - """Dataset-collection sync-down works under encryption.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - encryption=True, - ) - - mock_path, private_path, readme_path = create_tmp_dataset_files() - do_manager.create_dataset( - name="demo", - mock_path=mock_path, - private_path=private_path, - summary="demo dataset", - readme_path=readme_path, - users=[ds_manager.email], - upload_private=True, - sync=False, - ) - do_manager.sync() - - ds_manager.sync() - - cr = ds_manager.peer_manager.connection_router - collections = cr.watcher_list_dataset_collections() - do_collections = [c for c in collections if c["owner_email"] == do_manager.email] - assert do_collections, "DS does not see the DO's dataset collection" - - c = do_collections[0] - files = cr.watcher_download_dataset_collection( - c["tag"], c["content_hash"], do_manager.email - ) - assert files, "DS could not download the dataset collection files" diff --git a/tests/unit/test_exclude_hidden_files.py b/tests/unit/test_exclude_hidden_files.py index e9e44fd0026..c5282964304 100644 --- a/tests/unit/test_exclude_hidden_files.py +++ b/tests/unit/test_exclude_hidden_files.py @@ -95,6 +95,10 @@ def test_push_job_files_pushes_all_files_and_warns(caplog): """push_job_files should push every file but log a warning for non-syncable ones.""" ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection() + do_manager.datasite_owner_syncer.perm_context.open(".").grant_write_access( + ds_manager.email + ) + with tempfile.TemporaryDirectory() as tmp: job_dir = _create_job_dir_with_hidden_files(Path(tmp)) diff --git a/tests/unit/test_install_source_advertise.py b/tests/unit/test_install_source_advertise.py index 51bef9c589e..d6cf765d519 100644 --- a/tests/unit/test_install_source_advertise.py +++ b/tests/unit/test_install_source_advertise.py @@ -1,23 +1,12 @@ -"""Tests for DO advertising syft-client install source via VersionInfo. - -Covers: -- VersionInfo carries the new field and round-trips through JSON -- Backward compat: JSON missing the new field parses with field=None -- End-to-end: after pair setup, DS's JobClient knows DO's install source -- submit_python_job bakes the DO's source into run.sh -- Fallback path: DS warns and falls back to local detection when DO did not advertise -""" +"""VersionInfo carries the syft-client install source and round-trips through JSON.""" from __future__ import annotations import json -import warnings -from pathlib import Path from unittest.mock import patch import pytest -from syft_client.sync.syftbox_manager import SyftboxManager from syft_client.sync.version.version_info import VersionInfo @@ -62,66 +51,6 @@ def test_missing_field_parses_as_none_backward_compat(self): assert restored.syft_client_version == base.syft_client_version -class TestEndToEndPeerInstallSourcePropagation: - def test_ds_job_client_learns_do_install_source(self, monkeypatch): - monkeypatch.setenv("SYFT_CLIENT_INSTALL_SOURCE", "/do/local/syft-client") - - ds, do = SyftboxManager.pair_with_mock_drive_service_connection() - - assert do.email in ds.job_client.peer_install_sources - assert ds.job_client.peer_install_sources[do.email] == "/do/local/syft-client" - - def test_submit_python_job_bakes_do_source_into_run_sh(self, monkeypatch, tmp_path): - monkeypatch.setenv("SYFT_CLIENT_INSTALL_SOURCE", "/do/local/syft-client") - - ds, do = SyftboxManager.pair_with_mock_drive_service_connection() - - # Minimal job: a single Python file - code = tmp_path / "main.py" - code.write_text("print('hello')\n") - - job_dir = ds.job_client.submit_python_job( - user=do.email, code_path=str(code), job_name="my-test-job" - ) - - run_sh = (Path(job_dir) / "run.sh").read_text() - # The DO's advertised source must appear in the uv pip install line, - # and the DS's local source must NOT be substituted instead. - assert "/do/local/syft-client" in run_sh - assert "uv pip install" in run_sh - - -class TestFallbackWhenDoDidNotAdvertise: - def test_falls_back_and_warns_when_peer_has_no_source(self, monkeypatch, tmp_path): - monkeypatch.setenv("SYFT_CLIENT_INSTALL_SOURCE", "/ds/local/syft-client") - - ds, do = SyftboxManager.pair_with_mock_drive_service_connection() - - # Simulate an older DO: clear the advertised source from DS's JobClient. - ds.job_client.peer_install_sources.pop(do.email, None) - - code = tmp_path / "main.py" - code.write_text("print('hello')\n") - - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - job_dir = ds.job_client.submit_python_job( - user=do.email, - code_path=str(code), - job_name="fallback-job", - ) - - # A prominent warning must have been emitted. - messages = [str(w.message) for w in caught] - assert any("No syft-client install source advertised" in m for m in messages), ( - f"Expected fallback warning, got: {messages}" - ) - - # The DS's local detection result should be used as the fallback. - run_sh = (Path(job_dir) / "run.sh").read_text() - assert "/ds/local/syft-client" in run_sh - - class TestVersionInfoCurrentNeverRaises: def test_current_returns_versioninfo_even_if_detection_fails(self): # Force the install-source helper to blow up; current() must still return diff --git a/tests/unit/test_mock_drive_service.py b/tests/unit/test_mock_drive_service.py index ba03c0608b2..82c1ebce3bc 100644 --- a/tests/unit/test_mock_drive_service.py +++ b/tests/unit/test_mock_drive_service.py @@ -14,7 +14,6 @@ GOOGLE_FOLDER_MIME_TYPE, ) from syft_client.sync.connections.drive.gdrive_transport import GDriveConnection -from tests.unit.utils import create_tmp_dataset_files class TestMockDriveBackingStore: @@ -498,9 +497,13 @@ def test_ds_to_do_file_sync(self): sync_automatically=False, ) - # DS sends a file change to DO's job folder (where DS has write access) + # DS sends a file change to DO's job folder from tests.unit.test_sync_manager import path_for_job + do_manager.datasite_owner_syncer.perm_context.open(".").grant_write_access( + ds_manager.email + ) + file_path = path_for_job(do_manager.email, ds_manager.email) ds_manager._send_file_change(file_path, "Hello from DS!") @@ -510,39 +513,3 @@ def test_ds_to_do_file_sync(self): # Check that DO received the event events = do_manager._get_all_accepted_events_do() assert len(events) > 0 - - def test_dataset_creation_and_sync(self): - """Test that datasets created by DO are visible to DS via mock drive.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - ) - - mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() - - # DO creates a dataset - do_manager.create_dataset( - name="mock drive dataset", - mock_path=mock_dset_path, - private_path=private_dset_path, - summary="Test dataset via mock drive", - readme_path=readme_path, - tags=["test"], - users=[ds_manager.email], - ) - - # Verify DO can see it - do_datasets = do_manager.datasets.get_all() - assert len(do_datasets) == 1 - - # DS syncs - ds_manager.sync() - - # DS should see the dataset - ds_datasets = ds_manager.datasets.get_all() - assert len(ds_datasets) == 1 - - dataset = ds_manager.datasets.get( - "mock drive dataset", datasite=do_manager.email - ) - assert dataset is not None - assert len(dataset.mock_files) > 0 diff --git a/tests/unit/test_peer_requests.py b/tests/unit/test_peer_requests.py index 710bf63ab4f..441690cf986 100644 --- a/tests/unit/test_peer_requests.py +++ b/tests/unit/test_peer_requests.py @@ -2,6 +2,7 @@ from syft_client.sync.syftbox_manager import SyftboxManager from tests.unit.test_sync_manager import path_for_job +from tests.unit.utils import grant_job_inbox_access def test_peer_request_blocks_sync_until_approved(): @@ -44,8 +45,9 @@ def test_peer_request_blocks_sync_until_approved(): do_cache = do_manager.datasite_owner_syncer.event_cache assert len(do_cache.file_hashes) == 0, "Cache should be empty - peer not approved" - # Step 4: DO approves peer request (grants write to app_data/job/{ds_email}/) + # Step 4: DO approves peer request do_manager.approve_peer_request(ds_manager.email) + grant_job_inbox_access(do_manager, ds_manager.email) # Verify: Peer moved from requests to approved assert len(do_manager.peer_manager.requested_by_peer_peers) == 0 diff --git a/tests/unit/test_sync_file_lock.py b/tests/unit/test_sync_file_lock.py index 3d5c1844bff..6a5a3414d39 100644 --- a/tests/unit/test_sync_file_lock.py +++ b/tests/unit/test_sync_file_lock.py @@ -4,8 +4,8 @@ sync() concurrently. If the file lock is disabled or broken, this test fails. Failure modes caught: -1. sync() stops calling _sync_file_lock() → no log entries → fails on count -2. _sync_file_lock() is broken (no flock call) → intervals overlap → fails +1. sync() stops calling sync_file_lock() → no log entries → fails on count +2. sync_file_lock() is broken (no flock call) → intervals overlap → fails 3. Cross-process serialization is broken → intervals overlap → fails """ @@ -29,7 +29,7 @@ def _subprocess_sync_with_instrumented_lock( """Run inside a child process. Creates its own SyftboxManager pointing at the shared syftbox_folder, - patches _sync_file_lock to record interval timings, then calls sync() + patches sync_file_lock to record interval timings, then calls sync() multiple times. The intervals are written to the shared log file. """ log_path = Path(log_path_str) @@ -39,7 +39,7 @@ def _subprocess_sync_with_instrumented_lock( use_in_memory_cache=True, ) - original_lock = SyftboxManager._sync_file_lock + original_lock = SyftboxManager.sync_file_lock @contextmanager def instrumented_lock(self): @@ -53,7 +53,7 @@ def instrumented_lock(self): with open(log_path, "a") as f: f.write(f"{label},{start},{end}\n") - with patch.object(SyftboxManager, "_sync_file_lock", instrumented_lock): + with patch.object(SyftboxManager, "sync_file_lock", instrumented_lock): for _ in range(iterations): try: manager.sync(auto_checkpoint=False) @@ -105,13 +105,13 @@ def test_sync_prevents_concurrent_sync_across_processes(): intervals.append((label, float(start_s), float(end_s))) # Assertion 1: the lock was actually entered. - # If sync() doesn't call _sync_file_lock(), the instrumentation never + # If sync() doesn't call sync_file_lock(), the instrumentation never # fires and we get 0 intervals. expected_total = ITERATIONS * 2 assert len(intervals) == expected_total, ( f"Expected {expected_total} lock acquisitions " f"(ITERATIONS={ITERATIONS} × 2 processes), got {len(intervals)}. " - f"This means sync() is not calling _sync_file_lock()." + f"This means sync() is not calling sync_file_lock()." ) # Assertion 2: no two intervals overlap. diff --git a/tests/unit/test_sync_manager.py b/tests/unit/test_sync_manager.py index 57944534718..40364be891a 100644 --- a/tests/unit/test_sync_manager.py +++ b/tests/unit/test_sync_manager.py @@ -1,14 +1,10 @@ from pathlib import Path -import json -import tempfile -import shutil import pytest import yaml from syft_client.sync.connections.connection_router import ConnectionRouter from syft_client.sync.connections.drive.gdrive_transport import GDriveConnection -from syft_client.sync.connections.drive import mock_drive_service from syft_client.sync.peers.peer_store import PeerStore from syft_client.sync.messages.proposed_filechange import ProposedFileChangesMessage from syft_client.sync.messages.proposed_filechange import ProposedFileChange @@ -16,13 +12,13 @@ from syft_client.sync.sync.caches.datasite_owner_cache import ( ProposedEventFileOutdatedException, ) -from syft_datasets import Dataset +from syft_client.sync.sync.collection_spec import CollectionSyncSpec from tests.unit.utils import ( - create_tmp_dataset_files, - create_tmp_dataset_files_with_parquet, - create_test_project_folder, + TEST_COLLECTION_PREFIX, + TEST_COLLECTION_SUBPATH, get_mock_events_messages, get_mock_proposed_events_messages, + grant_job_inbox_access, ) @@ -33,6 +29,8 @@ def path_for_job(do_email: str, ds_email: str, filename: str = "test.job") -> st def test_sync_to_syftbox_eventlog(): ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection() + + grant_job_inbox_access(do_manager, ds_manager.email) file_path = path_for_job(do_manager.email, ds_manager.email, "my.job") events_in_backing_platform = do_manager._get_all_accepted_events_do() @@ -122,6 +120,8 @@ def test_valid_and_invalid_proposed_filechange_event(): def test_sync_back_to_ds_cache(): ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection() + + grant_job_inbox_access(do_manager, ds_manager.email) file_path = path_for_job(do_manager.email, ds_manager.email) ds_manager._send_file_change(file_path, "Hello, world!") @@ -164,12 +164,8 @@ def test_sync_existing_datasite_state_do(): n_outbox = connection_ds.watcher_get_outbox_file_metadatas(do_manager.email, None) assert n_messages_in_cache >= 1 # At least 1 message with the 2 file changes - assert ( - n_files_in_cache == 4 - ) # 2 data files + 2 syft.pub.yaml permission files (inbox + review) - assert ( - hashes_in_cache == 4 - ) # 2 data files + 2 syft.pub.yaml permission files (inbox + review) + assert n_files_in_cache == 2 # 2 data files + assert hashes_in_cache == 2 # 2 data files assert len(n_outbox) >= 1 @@ -204,12 +200,8 @@ def test_sync_existing_inbox_state_do(): n_files_in_cache = len(do_manager.datasite_owner_syncer.event_cache.file_connection) hashes_in_cache = len(do_manager.datasite_owner_syncer.event_cache.file_hashes) assert n_events_message_in_cache >= 1 # At least 1 message with 2 file changes - assert ( - n_files_in_cache == 5 - ) # 2 data files + 3 syft.pub.yaml permission files (inbox + review + root) - assert ( - hashes_in_cache == 5 - ) # 2 data files + 3 syft.pub.yaml permission files (inbox + review + root) + assert n_files_in_cache == 3 # 2 data files + root syft.pub.yaml + assert hashes_in_cache == 3 # 2 data files + root syft.pub.yaml def test_sync_existing_datasite_state_ds(): @@ -301,926 +293,6 @@ def test_file_connections(): assert (syftbox_dir_ds / do_manager.email / result_rel_path).exists() -def test_datasets(): - """Test dataset creation and sync between DO and DS.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False - ) - - mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() - - # Create dataset with specific users - do_manager.create_dataset( - name="my dataset", - mock_path=mock_dset_path, - private_path=private_dset_path, - summary="This is a summary", - readme_path=readme_path, - tags=["tag1", "tag2"], - users=[ds_manager.email], # Share with specific user - ) - - # Verify collection created - collections = do_manager._connection_router.owner_list_dataset_collections() - assert "my dataset" in collections - - datasets = do_manager.datasets.get_all() - assert len(datasets) == 1 - - # Retrieve dataset by name - dataset_do = do_manager.datasets["my dataset"] - assert isinstance(dataset_do, Dataset) - assert len(dataset_do.private_files) > 0 - assert len(dataset_do.mock_files) > 0 - - ds_manager.sync() - - # Verify DS can see collection - ds_collections = ds_manager._connection_router.watcher_list_dataset_collections() - assert any(c["tag"] == "my dataset" for c in ds_collections) - - assert len(ds_manager.datasets.get_all()) == 1 - - dataset_ds = ds_manager.datasets.get("my dataset", datasite=do_manager.email) - - assert dataset_ds.mock_files[0].exists() - - mock_content_ds = (dataset_ds.mock_dir / "mock.txt").read_text() - assert len(mock_content_ds) > 0 - - # test getting it via resolve path - from syft_client import resolve_dataset_file_path - - mock_file_path = resolve_dataset_file_path("my dataset", client=ds_manager) - assert mock_file_path.exists() - - mock_content_ds = mock_file_path.read_text() - assert len(mock_content_ds) > 0 - - def has_file(root_dir, filename): - return any(p.name == filename for p in Path(root_dir).rglob("*")) - - assert has_file(ds_manager.syftbox_folder, "mock.txt") - assert not has_file(ds_manager.syftbox_folder, "private.txt") - # Confirm that "private.txt" does not exist anywhere in the DS syftbox folder - for path in Path(ds_manager.syftbox_folder).rglob("*"): - assert path.name != "private.txt" - - -def test_datasets_with_parquet(): - """Test dataset creation and sync with parquet files (binary format).""" - import pandas as pd - - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False - ) - - mock_dset_path, private_dset_path, readme_path = ( - create_tmp_dataset_files_with_parquet() - ) - - # This should work without errors even though parquet files are binary - do_manager.create_dataset( - name="parquet dataset", - mock_path=mock_dset_path, - private_path=private_dset_path, - summary="This is a dataset with parquet files", - readme_path=readme_path, - tags=["parquet", "binary"], - users=[ds_manager.email], - ) - - datasets = do_manager.datasets.get_all() - assert len(datasets) == 1 - - # Dataset files are synced via collections, not the event log. - # Only the permission file (syft.pub.yaml) should appear as an event. - cached_events = do_manager.datasite_owner_syncer.event_cache.get_cached_events() - assert all(str(e.path_in_datasite).endswith("syft.pub.yaml") for e in cached_events) - - # Retrieve dataset by name - dataset_do = do_manager.datasets["parquet dataset"] - assert isinstance(dataset_do, Dataset) - assert len(dataset_do.private_files) > 0 - assert len(dataset_do.mock_files) > 0 - - # Verify parquet files are present - mock_files = [f.name for f in dataset_do.mock_files] - assert "mock_data.parquet" in mock_files - - private_files = [f.name for f in dataset_do.private_files] - assert "private_data.parquet" in private_files - - # Sync to datasite - ds_manager.sync() - - assert len(ds_manager.datasets.get_all()) == 1 - - dataset_ds = ds_manager.datasets.get("parquet dataset", datasite=do_manager.email) - - # Verify the parquet file exists and can be read - mock_parquet_path = dataset_ds.mock_dir / "mock_data.parquet" - assert mock_parquet_path.exists() - - # Verify we can read the parquet file back - df = pd.read_parquet(mock_parquet_path) - assert len(df) == 5 - assert "name" in df.columns - assert "age" in df.columns - - def has_file(root_dir, filename): - return any(p.name == filename for p in Path(root_dir).rglob("*")) - - assert has_file(ds_manager.syftbox_folder, "mock_data.parquet") - assert not has_file(ds_manager.syftbox_folder, "private_data.parquet") - - -def test_dataset_empty_permissions_no_access(): - """Test that empty permissions list means no one can access the dataset collection.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False - ) - - mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() - - # Create dataset with empty permissions list (share with no one) - do_manager.create_dataset( - name="private dataset", - mock_path=mock_dset_path, - private_path=private_dset_path, - summary="This is a private summary", - readme_path=readme_path, - tags=["private"], - users=[], # Empty list - no one has access - ) - - # Verify collection created - collections = do_manager._connection_router.owner_list_dataset_collections() - assert "private dataset" in collections - - # DO should be able to see their own dataset - datasets = do_manager.datasets.get_all() - assert len(datasets) == 1 - - # DS syncs - ds_manager.sync() - - # DS should NOT see the collection (no permissions) - ds_collections = ds_manager._connection_router.watcher_list_dataset_collections() - assert not any(c["tag"] == "private dataset" for c in ds_collections) - - # DS should not have downloaded any datasets - assert len(ds_manager.datasets.get_all()) == 0 - - -def test_dataset_only_mock_data_uploaded(): - """Test that only mock data is uploaded to the collection, not private data.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False - ) - - mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() - - do_manager.create_dataset( - name="test dataset", - mock_path=mock_dset_path, - private_path=private_dset_path, - summary="Test summary", - readme_path=readme_path, - tags=["test"], - users=[ds_manager.email], - ) - - # Sync so DS receives the dataset - ds_manager.sync() - - files = ds_manager._connection_router.connections[ - 0 - ].drive_service._backing_store.files - list(files.keys()) - file_objs = list(files.values()) - file_objs_ex_dataset_yaml = [ - file_obj for file_obj in file_objs if file_obj.name != "dataset.yaml" - ] - - assert not any("private" in file_obj.name for file_obj in file_objs) - # dataset.yaml does mention "private", but thats just the path - assert not any( - b"private" in file_obj.content for file_obj in file_objs_ex_dataset_yaml - ) - - assert any("mock" in file_obj.name for file_obj in file_objs) - assert any(b"Hello, world" in file_obj.content for file_obj in file_objs) - - mock_file = next(file_obj for file_obj in file_objs if file_obj.name == "mock.txt") - - # Verify mock content is correct - mock_content = mock_file.content.decode("utf-8") - assert len(mock_content) > 0, "Mock file should have content" - assert "Hello" in mock_content, "Mock file should contain expected data" - - -def test_jobs(): - """Test basic job submission, approval, execution, and result sync.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - sync_automatically=False, - ) - - test_py_path = "/tmp/test.py" - with open(test_py_path, "w") as f: - f.write(""" -with open("outputs/result.json", "w") as f: - f.write('{"result": 1}') -""") - - ds_manager.submit_python_job( - user=do_manager.email, - code_path=test_py_path, - job_name="test.job", - ) - - # We want to make sure that we only send one message for the multiple files in the job. - # this is to reduce the number of messages sent, which increases the speed of sync - # we do this by not always syncing on a file change, currently this logic is a bit of - # a short cut, but we could do this based on timing eventually (if there are items in the - # queue for longer than a certain time we start pushing) - connection_do = do_manager._connection_router.connections[0] - inbox_folder_id = connection_do._get_own_datasite_inbox_id(ds_manager.email) - inbox_file_metadatas = connection_do.get_file_metadatas_from_folder(inbox_folder_id) - assert len(inbox_file_metadatas) == 1 - - do_manager.sync() - - assert len(do_manager.job_client.jobs) == 1 - job = do_manager.job_client.jobs[0] - - job.approve() - - do_manager.job_runner.process_approved_jobs() - do_manager.job_runner.share_job_results( - "test.job", share_outputs=True, share_logs=False - ) - - do_manager.sync() - - ds_manager.sync() - - output_path = ds_manager.job_client.jobs[-1].output_paths[0] - with open(output_path, "r") as f: - json_content = json.loads(f.read()) - - assert json_content["result"] == 1 - - -def test_jobs_with_dataset(): - """Test job execution with dataset access using syft:// protocol.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - sync_automatically=False, - ) - - mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() - - do_manager.create_dataset( - name="my dataset", - mock_path=mock_dset_path, - private_path=private_dset_path, - summary="This is a summary", - readme_path=readme_path, - tags=["tag1", "tag2"], - users=[ds_manager.email], - ) - do_manager.sync() - - ds_manager.sync() - assert len(ds_manager.datasets.get_all()) == 1 - - dataset_ds = ds_manager.datasets.get("my dataset", datasite=do_manager.email) - assert dataset_ds.mock_files[0].exists() - import syft_client as sc - - assert ( - sc.resolve_dataset_file_path("my dataset", client=ds_manager) - == dataset_ds.mock_files[0] - ) - - test_py_path = "/tmp/test.py" - with open(test_py_path, "w") as f: - f.write(""" -import syft_client as sc -import json - -data_path = sc.resolve_dataset_file_path("my dataset") -with open(data_path, "r") as f: - data = f.read() -result = {"result": len(data)} -with open("outputs/result.json", "w") as f: - f.write(json.dumps(result)) -""") - - ds_manager.submit_python_job( - user=do_manager.email, - code_path=test_py_path, - job_name="test.job", - ) - - do_manager.sync() - assert len(do_manager.job_client.jobs) == 1 - job = do_manager.job_client.jobs[0] - - job.approve() - - do_manager.job_runner.process_approved_jobs() - do_manager.job_runner.share_job_results( - "test.job", share_outputs=True, share_logs=False - ) - - do_manager.sync() - - ds_manager.sync() - - output_path = ds_manager.job_client.jobs[-1].output_paths[0] - with open(output_path, "r") as f: - json_content = json.loads(f.read()) - - with open(private_dset_path, "r") as f: - private_data_length = len(f.read()) - - assert json_content["result"] == private_data_length - - -def test_single_file_job_submission_without_pyproject(): - """Test that code files are placed in job_dir/code/ subdirectory. - - Verifies code is at: - job_dir/code/main.py - """ - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - sync_automatically=False, - ) - - # Test with single file submission - test_py_path = "/tmp/test_direct_copy.py" - with open(test_py_path, "w") as f: - f.write('print("hello")') - - ds_manager.submit_python_job( - user=do_manager.email, - code_path=test_py_path, - job_name="test.direct.copy", - ) - - do_manager.sync() - - assert len(do_manager.job_client.jobs) == 1 - job = do_manager.job_client.jobs[0] - job_dir = job.job_submission_path - - # Verify code is in job_dir/code/ - assert (job_dir / "code" / "test_direct_copy.py").exists(), ( - "Code should be in job_dir/code/" - ) - assert (job_dir / "run.sh").exists(), "run.sh should exist" - assert (job_dir / "config.yaml").exists(), "config.yaml should exist" - - -def test_folder_job_submission_without_pyproject(): - """Test folder submission without pyproject.toml uses uv venv + uv pip install. - - Verifies: - - Folder without pyproject.toml works - - Folder is preserved with its name (not dumped at root) - - Generated run.sh uses 'uv venv' (not 'uv sync') - - Entrypoint path includes folder name - """ - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - sync_automatically=False, - ) - - # Create a folder without pyproject.toml - project_dir = tempfile.mkdtemp(prefix="test_no_pyproject_") - Path(project_dir).name - - try: - # Create main.py - main_path = Path(project_dir) / "main.py" - with open(main_path, "w") as f: - f.write(""" -with open("outputs/result.txt", "w") as f: - f.write("success") -""") - - # Create a helper module - helper_path = Path(project_dir) / "helper.py" - with open(helper_path, "w") as f: - f.write("VALUE = 42\n") - - # Submit folder (no pyproject.toml) - ds_manager.submit_python_job( - user=do_manager.email, - code_path=project_dir, - job_name="test.no.pyproject", - entrypoint="main.py", - ) - - do_manager.sync() - - assert len(do_manager.job_client.jobs) == 1 - job = do_manager.job_client.jobs[0] - job_dir = job.job_submission_path - - # Verify folder structure - code is in code/ subdirectory - assert (job_dir / "code").exists(), "code/ should exist in job_dir" - assert (job_dir / "code" / "main.py").exists(), "main.py should be inside code/" - assert (job_dir / "code" / "helper.py").exists(), ( - "helper.py should be inside code/" - ) - assert (job_dir / "run.sh").exists(), "run.sh should be at job_dir root" - assert (job_dir / "config.yaml").exists(), ( - "config.yaml should be at job_dir root" - ) - - # Verify run.sh uses uv venv (not uv sync) and correct entrypoint path - run_script = (job_dir / "run.sh").read_text() - assert "uv venv" in run_script, ( - "Should use 'uv venv' for folders without pyproject.toml" - ) - assert "uv sync" not in run_script, ( - "Should NOT use 'uv sync' without pyproject.toml" - ) - assert "cd code" in run_script, "Should cd into code folder" - assert "python main.py" in run_script, "Should run main.py from code/" - - finally: - shutil.rmtree(project_dir, ignore_errors=True) - - -def test_folder_job_submission_with_pyproject(): - """Test folder submission with pyproject.toml uses uv sync. - - Verifies: - - Folder is preserved with its name inside job_dir - - pyproject.toml is inside the folder, not at job_dir root - - run.sh uses 'uv sync' inside the folder - - Entrypoint path includes folder name - """ - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - sync_automatically=False, - ) - - # Create a folder with pyproject.toml - project_dir = tempfile.mkdtemp(prefix="test_with_pyproject_") - Path(project_dir).name - - try: - # Create pyproject.toml - pyproject_path = Path(project_dir) / "pyproject.toml" - with open(pyproject_path, "w") as f: - f.write(""" -[project] -name = "test-project" -version = "0.1.0" -dependencies = [] -""") - - # Create main.py - main_path = Path(project_dir) / "main.py" - with open(main_path, "w") as f: - f.write('print("hello from pyproject project")') - - # Submit folder (with pyproject.toml) - ds_manager.submit_python_job( - user=do_manager.email, - code_path=project_dir, - job_name="test.with.pyproject", - entrypoint="main.py", - ) - - do_manager.sync() - - assert len(do_manager.job_client.jobs) == 1 - job = do_manager.job_client.jobs[0] - job_dir = job.job_submission_path - - # Verify folder structure - code is in code/ subdirectory - assert (job_dir / "code").exists(), "code/ should exist in job_dir" - assert (job_dir / "code" / "main.py").exists(), "main.py should be inside code/" - assert (job_dir / "code" / "pyproject.toml").exists(), ( - "pyproject.toml should be inside code/" - ) - assert (job_dir / "run.sh").exists(), "run.sh should be at job_dir root" - assert (job_dir / "config.yaml").exists(), ( - "config.yaml should be at job_dir root" - ) - - # Verify run.sh uses uv sync inside code folder and correct entrypoint path - run_script = (job_dir / "run.sh").read_text() - assert "uv sync" in run_script, ( - "Should use 'uv sync' for folders with pyproject.toml" - ) - assert "cd code" in run_script, "Should cd into code folder for uv sync" - assert "python main.py" in run_script, "Should run main.py from code/" - - finally: - shutil.rmtree(project_dir, ignore_errors=True) - - -def test_folder_job_auto_detect_main_py(): - """Test that entrypoint is auto-detected when main.py exists.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - sync_automatically=False, - ) - - project_dir = tempfile.mkdtemp(prefix="test_auto_main_") - Path(project_dir).name - - try: - # Create main.py and another file - (Path(project_dir) / "main.py").write_text('print("main")') - (Path(project_dir) / "utils.py").write_text('print("utils")') - - # Submit without entrypoint - should auto-detect main.py - ds_manager.submit_python_job( - user=do_manager.email, - code_path=project_dir, - job_name="test.auto.main", - # No entrypoint specified - ) - - do_manager.sync() - job = do_manager.job_client.jobs[0] - job_dir = job.job_submission_path - - # Verify main.py was auto-detected - run_script = (job_dir / "run.sh").read_text() - assert "cd code" in run_script, "Should cd into code folder" - assert "python main.py" in run_script, ( - "Should auto-detect main.py as entrypoint" - ) - - finally: - shutil.rmtree(project_dir, ignore_errors=True) - - -def test_folder_job_auto_detect_single_py(): - """Test that entrypoint is auto-detected when only one .py file exists.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - sync_automatically=False, - ) - - project_dir = tempfile.mkdtemp(prefix="test_auto_single_") - Path(project_dir).name - - try: - # Create only one .py file (not named main.py) - (Path(project_dir) / "script.py").write_text('print("script")') - (Path(project_dir) / "README.md").write_text("# Readme") - - # Submit without entrypoint - should auto-detect script.py - ds_manager.submit_python_job( - user=do_manager.email, - code_path=project_dir, - job_name="test.auto.single", - ) - - do_manager.sync() - job = do_manager.job_client.jobs[0] - job_dir = job.job_submission_path - - # Verify script.py was auto-detected - run_script = (job_dir / "run.sh").read_text() - assert "cd code" in run_script, "Should cd into code folder" - assert "python script.py" in run_script, ( - "Should auto-detect single .py file as entrypoint" - ) - - finally: - shutil.rmtree(project_dir, ignore_errors=True) - - -def test_folder_job_no_auto_detect_multiple_py(): - """Test that auto-detection fails when multiple .py files and no main.py.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - sync_automatically=False, - ) - - project_dir = tempfile.mkdtemp(prefix="test_no_auto_") - - try: - # Create multiple .py files (no main.py) - (Path(project_dir) / "script1.py").write_text('print("1")') - (Path(project_dir) / "script2.py").write_text('print("2")') - - # Submit without entrypoint - should fail - with pytest.raises(ValueError, match="Could not auto-detect entrypoint"): - ds_manager.submit_python_job( - user=do_manager.email, - code_path=project_dir, - job_name="test.no.auto", - ) - - finally: - shutil.rmtree(project_dir, ignore_errors=True) - - -def test_single_file_job_flow_with_dataset(): - """Test complete job submission flow with dataset access. - - This test verifies the end-to-end flow of: - 1. Data Owner (DO) creates a dataset with mock and private data - 2. Data Scientist (DS) syncs and sees the dataset - 3. DS submits a Python job that accesses the private dataset - 4. DO approves and runs the job - 5. Job reads private data using syft:// protocol - 6. Results sync back to DS - - Test flow: - DO: create_dataset("my dataset") with private.txt containing "Hello, world!" - ↓ - DS: sync() → sees dataset - ↓ - DS: submit_python_job() with code that reads syft://private/... - ↓ - DO: sync() → receives job - ↓ - DO: job.approve() + process_approved_jobs() - ↓ - Job executes: reads private data → writes outputs/result.json - ↓ - DO: sync() → sends results - ↓ - DS: sync() → receives results - ↓ - Assert: result.json contains {"result": "Hello, world!"} - - Verifies: - - Dataset creation and sync between DO and DS - - Job submission with syft:// path resolution - - Job approval and execution workflow - - Output file sync back to DS - """ - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False - ) - - mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() - do_manager.create_dataset( - name="my dataset", - mock_path=mock_dset_path, - private_path=private_dset_path, - summary="This is a summary", - readme_path=readme_path, - tags=["tag1", "tag2"], - users=[ds_manager.email], # Share with DS so they can access the dataset - ) - - datasets = do_manager.datasets.get_all() - assert len(datasets) == 1 - - ds_manager.sync() - - assert len(ds_manager.datasets.get_all()) == 1 - - test_py_path = "/tmp/test.py" - with open(test_py_path, "w") as f: - f.write(""" -import json -import syft_client as sc - -data_path = sc.resolve_dataset_file_path("my dataset") - -with open(data_path, "r") as data_file: - data = data_file.read() - -result = {"result": data} - -with open("outputs/result.json", "w") as f: - f.write(json.dumps(result)) -""") - - ds_manager.submit_python_job( - user=do_manager.email, - code_path=test_py_path, - job_name="test.job", - ) - do_manager.sync() - - assert len(do_manager.job_client.jobs) == 1 - job = do_manager.job_client.jobs[0] - - job.approve() - - do_manager.job_runner.process_approved_jobs() - - # Before sharing: DS should not see outputs - do_manager.sync() - ds_manager.sync() - assert len(ds_manager.job_client.jobs[-1].output_paths) == 0 - - # After sharing: DS should see outputs - do_manager.job_runner.share_job_results( - "test.job", share_outputs=True, share_logs=False - ) - do_manager.sync() - ds_manager.sync() - - output_path = ds_manager.job_client.jobs[-1].output_paths[0] - with open(output_path, "r") as f: - json_content = json.loads(f.read()) - - assert json_content["result"] == "Hello, world private!" - - -def test_folder_job_flow_with_dataset(): - """Test job submission with a folder containing multiple Python files. - - Tests folder structure: - project_dir/ - ├── main.py # entrypoint, imports from helpers.helper - └── helpers/ - ├── __init__.py # package marker - └── helper.py # helper functions - - Verifies: - - Folder submission with entrypoint parameter works - - Nested package imports work (from helpers.helper import ...) - - Outputs created at job root (not inside code/) - - End-to-end flow: submit → approve → run → sync → verify output - """ - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False - ) - - mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() - do_manager.create_dataset( - name="my dataset", - mock_path=mock_dset_path, - private_path=private_dset_path, - summary="This is a summary", - readme_path=readme_path, - tags=["tag1", "tag2"], - users=[ds_manager.email], # Share with DS so they can access the dataset - ) - - ds_manager.sync() - assert len(ds_manager.datasets.get_all()) == 1 - - # Create test project folder (no pyproject.toml, multiplier=2) - project_dir = create_test_project_folder(with_pyproject=False, multiplier=2) - - try: - ds_manager.submit_python_job( - user=do_manager.email, - code_path=str(project_dir), - job_name="test.folder.job", - entrypoint="main.py", - ) - do_manager.sync() - - assert len(do_manager.job_client.jobs) == 1 - job = do_manager.job_client.jobs[0] - - job.approve() - do_manager.job_runner.process_approved_jobs() - - # Before sharing: DS should not see outputs - do_manager.sync() - ds_manager.sync() - assert len(ds_manager.job_client.jobs[-1].output_paths) == 0 - - # After sharing: DS should see outputs - do_manager.job_runner.share_job_results( - "test.folder.job", share_outputs=True, share_logs=False - ) - do_manager.sync() - ds_manager.sync() - - # Verify the job completed and produced output - output_path = ds_manager.job_client.jobs[-1].output_paths[0] - with open(output_path, "r") as f: - json_content = json.loads(f.read()) - - # Verify the helper module was imported and used correctly - assert json_content["original"] == "Hello, world private!" - assert json_content["processed"] == "Processed: Hello, world private!" - assert json_content["multiplier"] == 2 - - finally: - shutil.rmtree(project_dir, ignore_errors=True) - - -def test_pyproject_folder_job_flow_with_dataset(): - """Test job submission with a folder containing pyproject.toml. - - Tests folder structure: - project_dir/ - ├── pyproject.toml # project config with dependencies - ├── main.py # entrypoint, imports from helpers.helper - └── helpers/ - ├── __init__.py # package marker - └── helper.py # helper functions - - Verifies: - - Folder with pyproject.toml uses 'uv sync' (not 'uv venv') - - Folder is preserved with its name in job_dir - - .venv is created inside the code folder by uv sync - - Nested package imports work - - End-to-end flow: submit → approve → run → sync → verify output - """ - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False - ) - - mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() - do_manager.create_dataset( - name="my dataset", - mock_path=mock_dset_path, - private_path=private_dset_path, - summary="This is a summary", - readme_path=readme_path, - tags=["tag1", "tag2"], - users=[ds_manager.email], # Share with DS so they can access the dataset - ) - - ds_manager.sync() - assert len(ds_manager.datasets.get_all()) == 1 - - # Create test project folder with pyproject.toml, multiplier=3 - project_dir = create_test_project_folder( - with_pyproject=True, multiplier=3, prefix="test_pyproject_" - ) - - try: - ds_manager.submit_python_job( - user=do_manager.email, - code_path=str(project_dir), - job_name="test.pyproject.job", - entrypoint="main.py", - ) - - do_manager.sync() - assert len(do_manager.job_client.jobs) == 1 - job = do_manager.job_client.jobs[0] - job_dir = job.job_submission_path - - # Verify folder structure before running - code is in code/ subdirectory - assert (job_dir / "code").exists(), "code/ should exist in job_dir" - assert (job_dir / "code" / "pyproject.toml").exists(), ( - "pyproject.toml should be inside code/" - ) - assert (job_dir / "code" / "main.py").exists(), "main.py should be inside code/" - assert (job_dir / "run.sh").exists(), "run.sh should be at job_dir root" - - # Verify run.sh uses uv sync (pyproject.toml case) - run_script = (job_dir / "run.sh").read_text() - assert "uv sync" in run_script, ( - "Should use 'uv sync' for folders with pyproject.toml" - ) - assert "cd code" in run_script, "Should cd into code folder for uv sync" - assert "python main.py" in run_script, "Should run main.py from code/" - - # Run the job - job.approve() - do_manager.job_runner.process_approved_jobs() - - # Verify .venv was created inside the code folder (by uv sync) - assert (job_dir / "code" / ".venv").exists(), ( - ".venv should be created inside code/ folder by uv sync" - ) - - # Before sharing: DS should not see outputs - do_manager.sync() - ds_manager.sync() - assert len(ds_manager.job_client.jobs[-1].output_paths) == 0 - - # After sharing: DS should see outputs - do_manager.job_runner.share_job_results( - "test.pyproject.job", share_outputs=True, share_logs=False - ) - do_manager.sync() - ds_manager.sync() - - # Verify the job completed and produced output - output_path = ds_manager.job_client.jobs[-1].output_paths[0] - with open(output_path, "r") as f: - json_content = json.loads(f.read()) - - # Verify the helper module was imported and used correctly - assert json_content["original"] == "Hello, world private!" - assert json_content["processed"] == "Processed: Hello, world private!" - assert json_content["multiplier"] == 3 - - finally: - shutil.rmtree(project_dir, ignore_errors=True) - - def test_file_deletion_do_to_ds(): """Test that DO can delete a file and it syncs to DS""" ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( @@ -1285,6 +357,8 @@ def test_in_memory_deletion(): use_in_memory_cache=True ) + grant_job_inbox_access(do_manager, ds_manager.email) + # Create file via send_file_change job_path = path_for_job(do_manager.email, ds_manager.email) job_path_in_datasite = "/".join(job_path.split("/")[1:]) @@ -1311,14 +385,17 @@ def test_in_memory_deletion(): assert str(ds_path) not in [str(p) for p, _ in ds_cache.file_connection.get_items()] -def test_syft_datasets_excluded_from_outbox_sync(): - """Test that files in syft_datasets folder are excluded from outbox sync. +def test_collection_files_excluded_from_outbox_sync(): + """Test that files in a registered collection folder are excluded from outbox sync. - Datasets have their own dedicated sync channel with proper permissions, + Collections have their own dedicated sync channel with proper permissions, so they should not be broadcast to all peers via the general outbox. """ ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False + use_in_memory_cache=False, + collection_specs=[ + CollectionSyncSpec.public(TEST_COLLECTION_PREFIX, TEST_COLLECTION_SUBPATH) + ], ) datasite_dir_do = do_manager.syftbox_folder / do_manager.email @@ -1332,12 +409,12 @@ def test_syft_datasets_excluded_from_outbox_sync(): regular_file.parent.mkdir(parents=True, exist_ok=True) regular_file.write_text("regular content") - # Create a dataset file (should NOT be synced via outbox) - dataset_file = ( - datasite_dir_do / "public" / "syft_datasets" / "my_dataset" / "dataset.yaml" + # Create a collection file (should NOT be synced via outbox) + collection_file = ( + datasite_dir_do / TEST_COLLECTION_SUBPATH / "my_collection" / "data.yaml" ) - dataset_file.parent.mkdir(parents=True, exist_ok=True) - dataset_file.write_text("name: my_dataset") + collection_file.parent.mkdir(parents=True, exist_ok=True) + collection_file.write_text("name: my_collection") # Sync DO to generate events do_manager.sync() @@ -1351,9 +428,9 @@ def test_syft_datasets_excluded_from_outbox_sync(): "Regular files should be included in outbox sync" ) - # Dataset file should NOT be in cache (excluded from outbox) - assert not any("syft_datasets" in p for p in cached_paths), ( - "Files in syft_datasets should be excluded from outbox sync" + # Collection file should NOT be in cache (excluded from outbox) + assert not any("my_collection" in p for p in cached_paths), ( + "Files in a registered collection should be excluded from outbox sync" ) @@ -1453,197 +530,6 @@ def test_job_files_sync_to_submitter_only(): ) -def test_ds_dataset_cache_aware_sync(): - """Test that DS loads dataset hashes from disk and skips re-download on restart. - - This test verifies cache-aware dataset syncing: - 1. Create pair 1, DO creates dataset, DS syncs and downloads - 2. Create pair 2 with same directories and backing store (simulates restart) - 3. Verify hash is loaded from disk on startup and matches remote hash - 4. This ensures sync_down_datasets will skip downloading (hash comparison passes) - """ - from unittest.mock import patch - - # Create first pair - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False - ) - - mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() - - do_manager.create_dataset( - name="cached dataset", - mock_path=mock_dset_path, - private_path=private_dset_path, - summary="This is a cached dataset", - readme_path=readme_path, - tags=["cache", "test"], - users=[ds_manager.email], - ) - - # DS syncs and receives dataset - ds_manager.sync() - - # Verify dataset was downloaded - assert len(ds_manager.datasets.get_all()) == 1 - dataset = ds_manager.datasets.get("cached dataset", datasite=do_manager.email) - assert dataset.mock_files[0].exists() - - # Get the original hash from the collection - collections = ds_manager._connection_router.watcher_list_dataset_collections() - remote_hash = None - for c in collections: - if c["tag"] == "cached dataset": - remote_hash = c["content_hash"] - break - assert remote_hash is not None - - # Get the mock backing store and directories for creating second pair - mock_backing_store = ds_manager._connection_router.connections[ - 0 - ].drive_service._backing_store - ds_folder = ds_manager.syftbox_folder - do_folder = do_manager.syftbox_folder - ds_email = ds_manager.email - do_email = do_manager.email - - # Create second pair with same directories (simulates restart) - ds_manager2, do_manager2 = SyftboxManager.pair_with_mock_drive_service_connection( - email1=do_email, - email2=ds_email, - base_path1=do_folder, - base_path2=ds_folder, - use_in_memory_cache=False, - add_peers=False, - ) - - # Replace mock backing store to share dataset collections - ds_manager2._connection_router.connections[ - 0 - ].drive_service._backing_store = mock_backing_store - do_manager2._connection_router.connections[ - 0 - ].drive_service._backing_store = mock_backing_store - - # Load peers (already approved in shared backing store) - ds_manager2.load_peers() - do_manager2.load_peers() - - # Verify hash was loaded from disk on startup - ds_cache = ds_manager2.datasite_watcher_syncer.datasite_watcher_cache - # Cache uses full path as key: syftbox_folder / owner_email / collection_subpath / tag - cache_key = ds_cache.get_collection_path(do_email, "cached dataset") - assert cache_key in ds_cache.dataset_collection_hashes, ( - "Hash should be loaded from disk on startup" - ) - - # Verify the loaded hash matches the remote hash - # This ensures sync_down_datasets will skip the download (hash comparison at line 283-284) - loaded_hash = ds_cache.dataset_collection_hashes[cache_key] - assert loaded_hash == remote_hash, ( - "Loaded hash should match remote hash, ensuring no re-download is needed" - ) - - # Patch the download method to verify it's NOT called (hash match should skip download) - syncer = ds_manager2.datasite_watcher_syncer - original_method = syncer.download_dataset_file_with_new_connection - - with patch( - "syft_client.sync.sync.datasite_watcher_syncer.DatasiteWatcherSyncer.download_dataset_file_with_new_connection", - wraps=original_method, - ) as mock_download: - # Sync - no download should happen because hash matches - ds_manager2.sync() - - # Verify download_dataset_file_with_new_connection was NOT called - assert mock_download.call_count == 0, ( - "Should not download files when local hash matches remote" - ) - - # Verify dataset still accessible - assert len(ds_manager2.datasets.get_all()) == 1 - - -def test_do_dataset_cache_aware_sync(): - """Test that DO doesn't re-download datasets on restart when hash matches. - - This test verifies cache-aware dataset syncing for DO side: - 1. Create pair 1, DO creates dataset - 2. Create pair 2 with same directories and backing store (simulates restart) - 3. Verify _download_dataset_collections_parallel is NOT called on sync - """ - from unittest.mock import patch - - # Create first pair - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False - ) - - mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() - - do_manager.create_dataset( - name="do cached dataset", - mock_path=mock_dset_path, - private_path=private_dset_path, - summary="This is a DO cached dataset", - readme_path=readme_path, - tags=["cache", "do", "test"], - users=[ds_manager.email], - ) - - # Verify dataset was created locally - assert len(do_manager.datasets.get_all()) == 1 - - # Get the mock backing store and directories for creating second pair - mock_backing_store = ds_manager._connection_router.connections[ - 0 - ].drive_service._backing_store - ds_folder = ds_manager.syftbox_folder - do_folder = do_manager.syftbox_folder - ds_email = ds_manager.email - do_email = do_manager.email - - # Create second pair with same directories (simulates restart) - ds_manager2, do_manager2 = SyftboxManager.pair_with_mock_drive_service_connection( - email1=do_email, - email2=ds_email, - base_path1=do_folder, - base_path2=ds_folder, - use_in_memory_cache=False, - add_peers=False, - ) - - # Replace mock backing store to share dataset collections - ds_manager2._connection_router.connections[ - 0 - ].drive_service._backing_store = mock_backing_store - do_manager2._connection_router.connections[ - 0 - ].drive_service._backing_store = mock_backing_store - - # Load peers (already approved in shared backing store) - ds_manager2.load_peers() - do_manager2.load_peers() - - # Patch the download method to verify it's NOT called (hash match should skip download) - syncer = do_manager2.datasite_owner_syncer - with patch.object( - syncer, - "_download_file_with_new_connection", - wraps=syncer._download_file_with_new_connection, - ) as mock_download: - # Sync - should NOT trigger download since local hash matches remote - do_manager2.sync() - - # Verify _download_file_with_new_connection was NOT called - assert mock_download.call_count == 0, ( - "Should not download files when local hash matches remote" - ) - - # Verify dataset still accessible - assert len(do_manager2.datasets.get_all()) == 1 - - def test_in_memory_connection_syncing(): """Test basic syncing flow with mock drive service connection. @@ -1651,6 +537,8 @@ def test_in_memory_connection_syncing(): """ ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection() + grant_job_inbox_access(do_manager, ds_manager.email) + # DS sends a file change to DO's job folder (where DS has write access) ds_manager._send_file_change( path_for_job(do_manager.email, ds_manager.email, "my.job"), "Hello, world!" @@ -1669,251 +557,6 @@ def test_in_memory_connection_syncing(): assert len(events) > 0 -def test_in_memory_connection_load_state(): - """Test state persistence and loading with mock drive connection. - - Unit test equivalent of integration test_google_drive_connection_load_state. - - Workflow (matches integration test): - 1. Pair 1: Create peers, make changes, create dataset - 2. Pair 2: Load peers, sync DO → verify events processed - 3. Pair 3: Load peers, sync both → verify state loaded from storage - """ - from syft_client.sync.syftbox_manager import SyftboxManagerConfig - - # Get shared backing store and directories that will persist across pairs - ds_manager1, do_manager1 = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - add_peers=True, - ) - - # Get the backing store that will persist across "restarts" - backing_store = do_manager1._connection_router.connections[ - 0 - ].drive_service._backing_store - ds_folder = ds_manager1.syftbox_folder - do_folder = do_manager1.syftbox_folder - ds_email = ds_manager1.email - do_email = do_manager1.email - ds_manager1._connection_router.connections[0] - do_manager1._connection_router.connections[0] - - # Make some changes (submit to DS's job folder where DS has write access) - ds_manager1._send_file_change( - path_for_job(do_manager1.email, ds_manager1.email, "my.job"), "Hello, world!" - ) - ds_manager1._send_file_change( - path_for_job(do_manager1.email, ds_manager1.email, "my_second.job"), - "Hello, world!", - ) - - # Create a dataset with "any" permission - mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() - do_manager1.create_dataset( - name="load_state_dataset", - mock_path=mock_dset_path, - private_path=private_dset_path, - summary="Dataset for load state test", - readme_path=readme_path, - tags=["test"], - users="any", - ) - - # Verify dataset was created and cache populated - assert len(do_manager1._connection_router.owner_list_dataset_collections()) == 1 - assert len(do_manager1.datasite_owner_syncer._any_shared_datasets) == 1 - - # Create second pair (simulates restart, tests loading peers and processing inbox) - do_config2 = SyftboxManagerConfig._base_config_for_testing( - email=do_email, - syftbox_folder=do_folder, - has_ds_role=False, - has_do_role=True, - use_in_memory_cache=False, - check_versions=False, - ) - ds_config2 = SyftboxManagerConfig._base_config_for_testing( - email=ds_email, - syftbox_folder=ds_folder, - has_ds_role=True, - has_do_role=False, - use_in_memory_cache=False, - check_versions=False, - ) - - do_manager2 = SyftboxManager.from_config(do_config2) - ds_manager2 = SyftboxManager.from_config(ds_config2) - - # Connect to the same backing store - service_do = mock_drive_service.MockDriveService(backing_store, do_email) - do_connection2 = GDriveConnection.from_service(do_email, service_do) - - service_ds = mock_drive_service.MockDriveService(backing_store, ds_email) - ds_connection2 = GDriveConnection.from_service(ds_email, service_ds) - - do_manager2._add_connection(do_connection2) - ds_manager2._add_connection(ds_connection2) - - # Load peers - do_manager2.load_peers() - assert len(do_manager2.peers) == 1 - - ds_manager2.load_peers() - assert len(ds_manager2.peers) == 1 - - # Sync DO so we have something in the syftbox and do outbox - do_manager2.sync() - ds_manager2.sync() - - # Verify events in DO cache (inbox was processed) - # 2 data events + 1 permission file event (syft.pub.yaml from approve_peer_request) - assert len(do_manager2.datasite_owner_syncer.event_cache.get_cached_events()) == 4 - - # verify events in DS cache - loaded_events_ds = ( - ds_manager2.datasite_watcher_syncer.datasite_watcher_cache.get_cached_events() - ) - assert len(loaded_events_ds) == 2 - - # Verify datasets were loaded - loaded_datasets = do_manager2.datasets.get_all() - assert len(loaded_datasets) == 1 - assert loaded_datasets[0].name == "load_state_dataset" - assert len(do_manager2.datasite_owner_syncer._any_shared_datasets) == 1 - assert ( - do_manager2.datasite_owner_syncer._any_shared_datasets[0][0] - == "load_state_dataset" - ) - - -def test_datasets_shared_with_any(): - """Test that datasets shared with 'any' become discoverable after peer approval. - - Unit test equivalent of integration test_datasets_shared_with_any. - """ - # Create managers WITHOUT auto peer setup - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - add_peers=False, - ) - - mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() - - # DO creates dataset with users='any' BEFORE peer is approved - do_manager.create_dataset( - name="any dataset", - mock_path=mock_dset_path, - private_path=private_dset_path, - summary="Dataset shared with anyone", - readme_path=readme_path, - tags=["any"], - users="any", - ) - - # DS should NOT see the dataset yet (not approved) - ds_collections = ds_manager._connection_router.watcher_list_dataset_collections() - assert not any(c["tag"] == "any dataset" for c in ds_collections) - - # DS adds peer, DO approves (this should share 'any' datasets) - ds_manager.add_peer(do_manager.email) - do_manager.load_peers() - do_manager.approve_peer_request(ds_manager.email, peer_must_exist=False) - - # DS should now see the dataset - ds_collections = ds_manager._connection_router.watcher_list_dataset_collections() - assert any(c["tag"] == "any dataset" for c in ds_collections) - - -def test_datasets_shared_with_any_after_peer_approved(): - """Test that creating a dataset with users='any' after peers are approved - grants those peers access immediately. - - Workflow: - 1. Create managers without auto peers - 2. DS adds peer, DO approves - 3. DO creates dataset with users='any' - 4. DS can see the dataset (without needing another approve_peer_request) - """ - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - add_peers=False, - ) - - # DS adds peer, DO approves - ds_manager.add_peer(do_manager.email) - do_manager.load_peers() - do_manager.approve_peer_request(ds_manager.email, peer_must_exist=False) - - mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() - - # DO creates dataset with users='any' AFTER peer is already approved - do_manager.create_dataset( - name="late any dataset", - mock_path=mock_dset_path, - private_path=private_dset_path, - summary="Dataset shared with anyone, created after peer approval", - readme_path=readme_path, - tags=["any", "late"], - users="any", - ) - - # DS should see the dataset immediately (shared at creation time) - ds_collections = ds_manager._connection_router.watcher_list_dataset_collections() - assert any(c["tag"] == "late any dataset" for c in ds_collections) - - -def test_ds_stale_state_cleared_after_do_delete_syftbox(): - """Test that DS state is cleaned up after DO calls delete_syftbox(). - - Verifies: - 1. After initial setup, DS has datasets and file_hashes - 2. DO calls delete_syftbox() which broadcasts is_deleted events - 3. DS syncs and its file_hashes and datasets are empty - """ - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - ) - - # Create dataset on DO side - mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() - do_manager.create_dataset( - name="my dataset", - mock_path=mock_dset_path, - private_path=private_dset_path, - summary="Test dataset", - readme_path=readme_path, - users=[ds_manager.email], - ) - - # DS sends a file change to DO (use correct job path) - ds_manager._send_file_change( - path_for_job(do_manager.email, ds_manager.email), "print('hello')" - ) - do_manager.sync() - - # DS syncs to get dataset and file events - ds_manager.sync() - - # Verify initial state exists - ds_cache = ds_manager.datasite_watcher_syncer.datasite_watcher_cache - assert len(ds_manager.datasets.get_all()) == 1 - assert len(ds_cache.file_hashes) > 0 - - # DO deletes syftbox (broadcasts delete events to DS) - do_manager.delete_syftbox() - - # DS syncs again - should pick up delete events and stale dataset cleanup - ds_manager.sync() - - # Verify DS state is cleaned up - assert len(ds_cache.file_hashes) == 0, ( - "DS file_hashes should be empty after DO delete" - ) - assert len(ds_manager.datasets.get_all()) == 0, ( - "DS datasets should be empty after DO delete" - ) - - def test_incoming_syft_pub_yaml_write_requires_admin(): """Test that DS cannot write syft.pub.yaml unless they have admin access. @@ -1925,6 +568,8 @@ def test_incoming_syft_pub_yaml_write_requires_admin(): use_in_memory_cache=False, ) + grant_job_inbox_access(do_manager, ds_manager.email) + ds_email = ds_manager.email do_email = do_manager.email datasite_dir_do = do_manager.syftbox_folder / do_email @@ -2042,39 +687,16 @@ def test_permission_change_triggers_resend(): ) -def test_dataset_delete_propagates_to_ds(): - """Test that deleting a dataset on DO removes it from Drive and DS.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False - ) +def test_default_collections_folder_picks_shareable_spec_regardless_of_order(): + """The collections folder is chosen by owner_only, not by position.""" + from syft_client.sync.syftbox_manager import default_collections_folder - mock_dset_path, private_dset_path, readme_path = create_tmp_dataset_files() + shareable = CollectionSyncSpec.public("syft_pub", Path("public/pub")) + owner_only = CollectionSyncSpec.private("syft_priv", Path("private/priv")) + expected = Path("/sb/me@t.com/public/pub") - # DO creates dataset shared with DS - do_manager.create_dataset( - name="deleteme", - mock_path=mock_dset_path, - private_path=private_dset_path, - summary="To be deleted", - readme_path=readme_path, - users=[ds_manager.email], - ) + for specs in ([shareable, owner_only], [owner_only, shareable]): + assert default_collections_folder("/sb", "me@t.com", specs) == expected - # Verify collection exists on Drive - collections = do_manager._connection_router.owner_list_dataset_collections() - assert "deleteme" in collections - - # DS syncs and sees the dataset - ds_manager.sync() - assert len(ds_manager.datasets.get_all()) == 1 - - # DO deletes the dataset - do_manager.delete_dataset(name="deleteme", require_confirmation=False, sync=True) - - # Verify collection is gone from Drive - collections_after = do_manager._connection_router.owner_list_dataset_collections() - assert "deleteme" not in collections_after - - # DS syncs again — should pick up the deletion - ds_manager.sync() - assert len(ds_manager.datasets.get_all()) == 0 + assert default_collections_folder("/sb", "me@t.com", [owner_only]) is None + assert default_collections_folder("/sb", "me@t.com", []) is None diff --git a/tests/unit/test_version_negotiation.py b/tests/unit/test_version_negotiation.py index 36bb593f4ac..846c4032d54 100644 --- a/tests/unit/test_version_negotiation.py +++ b/tests/unit/test_version_negotiation.py @@ -6,7 +6,6 @@ from syft_client.sync.syftbox_manager import SyftboxManager from syft_client.sync.version.exceptions import ( VersionMismatchError, - VersionUnknownError, ) from syft_client.sync.version.peer_manager import CompatAction from syft_client.sync.version.version_info import CompatibilityStatus, VersionInfo @@ -256,69 +255,6 @@ def test_load_peer_versions_parallel(self): ) -class TestForceSubmission: - """Tests for force_submission parameter.""" - - def test_job_submission_blocked_without_version(self): - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - add_peers=False, - sync_automatically=False, - check_versions=True, - ) - - ds_manager.add_peer(do_manager.email) - - test_py_path = "/tmp/test_version.py" - with open(test_py_path, "w") as f: - f.write('print("hello")') - - with pytest.raises(VersionUnknownError): - ds_manager.submit_python_job( - user=do_manager.email, - code_path=test_py_path, - job_name="test.job", - ) - - def test_job_submission_allowed_with_force(self): - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - add_peers=False, - sync_automatically=False, - check_versions=True, - ) - - ds_manager.add_peer(do_manager.email) - - test_py_path = "/tmp/test_version_force.py" - with open(test_py_path, "w") as f: - f.write('print("hello")') - - with pytest.raises(VersionUnknownError): - ds_manager.submit_python_job( - user=do_manager.email, - code_path=test_py_path, - job_name="test.fail.job", - ) - - ds_manager.submit_python_job( - user=do_manager.email, - code_path=test_py_path, - job_name="test.force.job", - force_submission=True, - ) - - job_dir = ( - ds_manager.syftbox_folder - / do_manager.email - / "app_data" - / "job" - / "inbox" - / ds_manager.email - / "v1" - / "test.force.job" - ) - assert job_dir.exists() - - def _set_peer_version(manager: SyftboxManager, peer_email: str, version: VersionInfo): """Override the cached version for a peer without a Drive round-trip.""" peer = manager.peer_manager.get_cached_peer(peer_email) @@ -545,43 +481,6 @@ def test_sync_skips_incompatible_peers(self): ) assert ds_manager.email not in compatible_peers - def test_job_execution_forced_with_incompatible_version(self): - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - sync_automatically=False, - use_in_memory_cache=False, - ) - - test_py_path = "/tmp/test_exec_force.py" - with open(test_py_path, "w") as f: - f.write('print("hello")') - - ds_manager.submit_python_job( - user=do_manager.email, - code_path=test_py_path, - job_name="test.exec.force.job", - ) - - do_manager.sync() - assert len(do_manager.job_client.jobs) == 1 - job = do_manager.job_client.jobs[0] - job.approve() - - _set_peer_version(do_manager, ds_manager.email, build_client_version("0.0.1")) - - executed_jobs = [] - - def mock_process_approved_jobs( - stream_output=True, timeout=None, skip_job_names=None, **kwargs - ): - executed_jobs.append(skip_job_names) - - do_manager.job_runner.process_approved_jobs = mock_process_approved_jobs - - do_manager.process_approved_jobs(force_execution=True) - - assert len(executed_jobs) == 1 - assert executed_jobs[0] is None # No jobs skipped when force=True - def test_version_upgrade_breaks_communication(self): """Major-bump upgrade should now make peers incompatible.""" ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( diff --git a/tests/unit/utils.py b/tests/unit/utils.py index fb530ec3a38..d8a1d37fcea 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -194,3 +194,16 @@ def get_multiplier(): """) return project_dir + + +# Domain-free collection used by engine-level tests, in place of the RDS +# dataset specs (syft-client must not depend on the product layer). +TEST_COLLECTION_PREFIX = "syft_testcollection" +TEST_COLLECTION_SUBPATH = Path("public/test_collections") + + +def grant_job_inbox_access(do_manager, ds_email: str) -> None: + """Grant DS write access to the DO's job inbox.""" + do_manager.datasite_owner_syncer.perm_context.open( + f"app_data/job/inbox/{ds_email}/" + ).grant_write_access(ds_email) diff --git a/uv.lock b/uv.lock index 51ca3f63bbc..2bdbc64d949 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'win32'", @@ -29,6 +29,7 @@ members = [ "syft-notebook-ui", "syft-permissions", "syft-perms", + "syft-rds", "syft-restrict", ] @@ -986,7 +987,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1506,17 +1507,17 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version < '3.11'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, - { name = "jedi", marker = "python_full_version < '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version < '3.11'" }, - { name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version < '3.11'" }, - { name = "pygments", marker = "python_full_version < '3.11'" }, - { name = "stack-data", marker = "python_full_version < '3.11'" }, - { name = "traitlets", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "exceptiongroup" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/40/18/f8598d287006885e7136451fdea0755af4ebcbfe342836f24deefaed1164/ipython-8.39.0.tar.gz", hash = "sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624", size = 5513971, upload-time = "2026-03-27T10:02:13.94Z" } wheels = [ @@ -1542,18 +1543,18 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version >= '3.11'" }, - { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, - { name = "jedi", marker = "python_full_version >= '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, - { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, - { name = "psutil", marker = "python_full_version >= '3.11'" }, - { name = "pygments", marker = "python_full_version >= '3.11'" }, - { name = "stack-data", marker = "python_full_version >= '3.11'" }, - { name = "traitlets", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "psutil" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/87cda5842cf5c31837c06ddb588e11c3c35d8ece89b7a0108c06b8c9b00a/ipython-9.13.0.tar.gz", hash = "sha256:7e834b6afc99f020e3f05966ced34792f40267d64cb1ea9043886dab0dde5967", size = 4430549, upload-time = "2026-04-24T12:24:55.221Z" } wheels = [ @@ -1565,7 +1566,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -2089,15 +2090,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/de/88b3be5c31b22333b3ca2f6ff1de4e863d8fe45aaea7485f591970ec1d3e/linkify_it_py-2.1.0-py3-none-any.whl", hash = "sha256:0d252c1594ecba2ecedc444053db5d3a9b7ec1b0dd929c8f1d74dce89f86c05e", size = 19878, upload-time = "2026-03-01T07:48:46.098Z" }, ] -[[package]] -name = "lockfile" -version = "0.12.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/17/47/72cb04a58a35ec495f96984dddb48232b551aafb95bde614605b754fe6f7/lockfile-0.12.2.tar.gz", hash = "sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799", size = 20874, upload-time = "2015-11-25T18:29:58.279Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/22/9460e311f340cb62d26a38c419b1381b8593b0bb6b5d1f056938b086d362/lockfile-0.12.2-py2.py3-none-any.whl", hash = "sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa", size = 13564, upload-time = "2015-11-25T18:29:51.462Z" }, -] - [[package]] name = "loguru" version = "0.7.3" @@ -2840,10 +2832,10 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "python-dateutil", marker = "python_full_version < '3.11'" }, - { name = "pytz", marker = "python_full_version < '3.11'" }, - { name = "tzdata", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -2915,9 +2907,9 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.11'" }, - { name = "tzdata", marker = "(python_full_version >= '3.11' and sys_platform == 'emscripten') or (python_full_version >= '3.11' and sys_platform == 'win32')" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } wheels = [ @@ -3002,7 +2994,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "ptyprocess" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -3733,18 +3725,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, ] -[[package]] -name = "python-daemon" -version = "3.1.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "lockfile" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3d/37/4f10e37bdabc058a32989da2daf29e57dc59dbc5395497f3d36d5f5e2694/python_daemon-3.1.2.tar.gz", hash = "sha256:f7b04335adc473de877f5117e26d5f1142f4c9f7cd765408f0877757be5afbf4", size = 71576, upload-time = "2024-12-03T08:41:07.843Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/45/3c/b88167e2d6785c0e781ee5d498b07472aeb9b6765da3b19e7cc9e0813841/python_daemon-3.1.2-py3-none-any.whl", hash = "sha256:b906833cef63502994ad48e2eab213259ed9bb18d54fa8774dcba2ff7864cec6", size = 30872, upload-time = "2024-12-03T08:41:03.322Z" }, -] - [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -4472,6 +4452,7 @@ dependencies = [ { name = "pydantic" }, { name = "pyyaml" }, { name = "syft-job" }, + { name = "syft-rds" }, { name = "textual" }, ] @@ -4486,6 +4467,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.0.0" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "syft-job", editable = "packages/syft-job" }, + { name = "syft-rds", editable = "packages/syft-rds" }, { name = "textual", specifier = ">=0.40.0" }, ] @@ -4493,38 +4475,18 @@ requires-dist = [ name = "syft-client" source = { editable = "." } dependencies = [ - { name = "click" }, { name = "google-api-python-client" }, { name = "google-auth", extra = ["pyjwt"] }, { name = "google-auth-oauthlib" }, - { name = "jinja2" }, - { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "portalocker" }, { name = "pydantic-settings" }, - { name = "python-daemon" }, - { name = "pyyaml" }, { name = "rich" }, - { name = "syft-bg" }, { name = "syft-crypto-python" }, { name = "syft-dataset" }, - { name = "syft-job" }, { name = "syft-permissions" }, { name = "syft-perms" }, ] -[package.optional-dependencies] -all = [ - { name = "syft-dataset" }, - { name = "syft-job" }, -] -datasets = [ - { name = "syft-dataset" }, -] -job = [ - { name = "syft-job" }, -] - [package.dev-dependencies] dev = [ { name = "ipython", version = "8.39.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -4548,6 +4510,7 @@ test = [ { name = "pytest-cov" }, { name = "pytest-rerunfailures" }, { name = "pytest-xdist" }, + { name = "syft-bg" }, { name = "syft-enclave" }, { name = "syft-migration" }, { name = "syft-restrict" }, @@ -4555,28 +4518,17 @@ test = [ [package.metadata] requires-dist = [ - { name = "click", specifier = ">=8.0.0" }, { name = "google-api-python-client", specifier = ">=2.95.0" }, { name = "google-auth", extras = ["pyjwt"], specifier = ">=2.22.0" }, { name = "google-auth-oauthlib", specifier = ">=1.0.0" }, - { name = "jinja2", specifier = ">=3.1.0" }, - { name = "pandas" }, { name = "portalocker", specifier = ">=2.8.0" }, { name = "pydantic-settings", specifier = ">=2.11.0" }, - { name = "python-daemon", specifier = ">=3.0.0" }, - { name = "pyyaml", specifier = ">=6.0" }, { name = "rich", specifier = ">=13.0.0" }, - { name = "syft-bg", editable = "packages/syft-bg" }, - { name = "syft-client", extras = ["datasets", "job"], marker = "extra == 'all'" }, { name = "syft-crypto-python", specifier = ">=0.1.2b2" }, { name = "syft-dataset", editable = "packages/syft-datasets" }, - { name = "syft-dataset", marker = "extra == 'datasets'", editable = "packages/syft-datasets" }, - { name = "syft-job", editable = "packages/syft-job" }, - { name = "syft-job", marker = "extra == 'job'", editable = "packages/syft-job" }, { name = "syft-permissions", editable = "packages/syft-permissions" }, { name = "syft-perms", editable = "packages/syft-perms" }, ] -provides-extras = ["datasets", "job", "all"] [package.metadata.requires-dev] dev = [ @@ -4600,6 +4552,7 @@ test = [ { name = "pytest-cov", specifier = ">=4.0.0" }, { name = "pytest-rerunfailures", specifier = ">=14.0" }, { name = "pytest-xdist", specifier = ">=3.0.0" }, + { name = "syft-bg", editable = "packages/syft-bg" }, { name = "syft-enclave", editable = "packages/syft-enclave" }, { name = "syft-migration", editable = "packages/syft-migration" }, { name = "syft-restrict", editable = "packages/syft-restrict" }, @@ -4654,6 +4607,7 @@ dependencies = [ { name = "pydantic-settings" }, { name = "requests" }, { name = "syft-client" }, + { name = "syft-rds" }, ] [package.metadata] @@ -4662,6 +4616,7 @@ requires-dist = [ { name = "pydantic-settings", specifier = ">=2.11.0" }, { name = "requests", specifier = ">=2.32.0" }, { name = "syft-client", editable = "." }, + { name = "syft-rds", editable = "packages/syft-rds" }, ] [[package]] @@ -4751,6 +4706,23 @@ dependencies = [ [package.metadata] requires-dist = [{ name = "syft-permissions", editable = "packages/syft-permissions" }] +[[package]] +name = "syft-rds" +version = "0.1.0" +source = { editable = "packages/syft-rds" } +dependencies = [ + { name = "syft-client" }, + { name = "syft-dataset" }, + { name = "syft-job" }, +] + +[package.metadata] +requires-dist = [ + { name = "syft-client", editable = "." }, + { name = "syft-dataset", editable = "packages/syft-datasets" }, + { name = "syft-job", editable = "packages/syft-job" }, +] + [[package]] name = "syft-restrict" version = "0.1.0"