From aada93267d8164e3e70ecf3f2f1946c1f1353b50 Mon Sep 17 00:00:00 2001 From: Alex Wolf Date: Wed, 12 Aug 2026 15:08:14 +0200 Subject: [PATCH 01/14] =?UTF-8?q?=E2=9C=A8=20Introduce=20using=20in=20ln.s?= =?UTF-8?q?ave()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lamindb/models/save.py | 15 +++++++-------- sub/lamindb-setup | 2 +- tests/pydata/test_save.py | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/lamindb/models/save.py b/lamindb/models/save.py index 5c44ac6de..95bb4b41b 100644 --- a/lamindb/models/save.py +++ b/lamindb/models/save.py @@ -30,6 +30,7 @@ def save( records: Iterable[SQLRecord], ignore_conflicts: bool | None = False, batch_size: int = 10000, + using: str | None = None, ) -> None: """Bulk save objects. @@ -49,6 +50,7 @@ def save( If you need records with ids, you need to query them from the database. batch_size: Number of records to process in each batch. Large batch sizes can improve performance but may lead to memory issues. + using: Optional database slug for the target database. Examples -------- @@ -117,8 +119,7 @@ def save( ): record._storage_ongoing = True record._save_skip_storage() - using_key = settings._using_key - store_artifacts(artifacts, using_key=using_key) + store_artifacts(artifacts, using=using) # this function returns None as potentially 10k records might be saved # refreshing all of them from the DB would mean a severe performance penalty @@ -410,9 +411,7 @@ def check_and_attempt_clearing( return None -def store_artifacts( - artifacts: Iterable[Artifact], using_key: str | None = None -) -> None: +def store_artifacts(artifacts: Iterable[Artifact], using: str | None = None) -> None: """Upload artifacts in a list of database-committed artifacts to storage. If any upload fails, subsequent artifacts are cleaned up from the DB. @@ -427,7 +426,7 @@ def store_artifacts( for artifact in artifacts: # failure here sets ._clear_storagekey # for cleanup below - exception = check_and_attempt_upload(artifact, using_key) + exception = check_and_attempt_upload(artifact, using) if exception is not None: break @@ -445,7 +444,7 @@ def store_artifacts( # if check_and_attempt_upload was successful # then this can have only ._clear_storagekey from .replace exception = check_and_attempt_clearing( - artifact, raise_file_not_found_error=True, using_key=using_key + artifact, raise_file_not_found_error=True, using_key=using ) if exception is not None: logger.warning(f"clean up of {artifact._clear_storagekey} failed") # type: ignore @@ -459,7 +458,7 @@ def store_artifacts( artifact._delete_skip_storage() # clean up storage after failure in check_and_attempt_upload exception_clear = check_and_attempt_clearing( - artifact, raise_file_not_found_error=False, using_key=using_key + artifact, raise_file_not_found_error=False, using_key=using ) if exception_clear is not None: logger.warning( diff --git a/sub/lamindb-setup b/sub/lamindb-setup index 194e2e491..307c46e1e 160000 --- a/sub/lamindb-setup +++ b/sub/lamindb-setup @@ -1 +1 @@ -Subproject commit 194e2e4912922298ebee609f13d303b8e339cad0 +Subproject commit 307c46e1e64b73def27c9fd3d137a850ab1eaa87 diff --git a/tests/pydata/test_save.py b/tests/pydata/test_save.py index b58ee0e7d..847589e43 100644 --- a/tests/pydata/test_save.py +++ b/tests/pydata/test_save.py @@ -49,7 +49,7 @@ def test_store_artifacts_acid(get_mini_csv): artifact.save() with pytest.raises(RuntimeError) as error: - store_artifacts([artifact], using_key=None) + store_artifacts([artifact], using=None) assert str(error.exconly()).startswith( "RuntimeError: The following entries have been successfully uploaded" ) From f35c1b643207c52defb65ae2c95bdff325e0ddb2 Mon Sep 17 00:00:00 2001 From: Alex Wolf Date: Wed, 12 Aug 2026 15:13:02 +0200 Subject: [PATCH 02/14] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Harmonize=20docstrin?= =?UTF-8?q?gs=20and=20naming=20convention?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lamindb/models/save.py | 2 +- lamindb/models/sqlrecord.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/lamindb/models/save.py b/lamindb/models/save.py index 95bb4b41b..c2feb3496 100644 --- a/lamindb/models/save.py +++ b/lamindb/models/save.py @@ -50,7 +50,7 @@ def save( If you need records with ids, you need to query them from the database. batch_size: Number of records to process in each batch. Large batch sizes can improve performance but may lead to memory issues. - using: Optional database slug for the target database. + using: Optional database slug for a target database that differs from the default database. Examples -------- diff --git a/lamindb/models/sqlrecord.py b/lamindb/models/sqlrecord.py index 0255522a3..ee7aa6137 100644 --- a/lamindb/models/sqlrecord.py +++ b/lamindb/models/sqlrecord.py @@ -1335,11 +1335,12 @@ def _field_changed(self, field_name: str, check_is_saved: bool = True) -> bool: def save(self: T, *args, **kwargs) -> T: """Save. - Always saves to the default database. + Args: + using: Optional database slug for a target database that differs from the default database. """ - using_key = None + using = None if "using" in kwargs: - using_key = kwargs["using"] + using = kwargs["using"] transfer_config = kwargs.pop("transfer", None) db = self._state.db pk_on_db = self.pk @@ -1348,7 +1349,7 @@ def save(self: T, *args, **kwargs) -> T: and transfer_config is None and db is not None and db != "default" - and using_key is None + and using is None ): transfer_config = "annotations" artifacts: list = [] @@ -1362,14 +1363,14 @@ def save(self: T, *args, **kwargs) -> T: "transferred": [], "run": None, } - if db is not None and db != "default" and using_key is None: + if db is not None and db != "default" and using is None: if isinstance(self, IsVersioned): if not self.is_latest: raise NotImplementedError( "You are attempting to transfer a record that's not the latest in its version history. This is currently not supported." ) pre_existing_record = transfer_to_default_db( - self, using_key, transfer_logs=transfer_logs + self, using, transfer_logs=transfer_logs ) self._revises: IsVersioned if pre_existing_record is not None: @@ -1543,7 +1544,7 @@ def save(self: T, *args, **kwargs) -> T: track_current_name_value(self) # perform transfer of many-to-many fields # only supported for Artifact and Collection records - if db is not None and db != "default" and using_key is None: + if db is not None and db != "default" and using is None: if self.__class__.__name__ == "Collection": if len(artifacts) > 0: logger.info("transfer artifacts") @@ -1554,7 +1555,7 @@ def save(self: T, *args, **kwargs) -> T: from .schema import transfer_schema_members transfer_schema_members( - self, db, pk_on_db, using_key, transfer_logs=transfer_logs + self, db, pk_on_db, using, transfer_logs=transfer_logs ) if hasattr(self, "labels") and transfer_config == "annotations": from copy import copy From a0dbe406666940b1654f148d1ab3e170e5ba9b92 Mon Sep 17 00:00:00 2001 From: Alex Wolf Date: Wed, 12 Aug 2026 15:15:08 +0200 Subject: [PATCH 03/14] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Better=20docstring?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lamindb/models/save.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lamindb/models/save.py b/lamindb/models/save.py index c2feb3496..be357e2a5 100644 --- a/lamindb/models/save.py +++ b/lamindb/models/save.py @@ -34,9 +34,8 @@ def save( ) -> None: """Bulk save objects. - Note: - - This is a much faster than saving objects using `object.save()`. + This is a much faster than saving a list of objects + and repeatedly calling `object.save()`. Warning: From 7ac08ec0f712ec8082d465f1d48bd0a473ae54c2 Mon Sep 17 00:00:00 2001 From: Alex Wolf Date: Wed, 12 Aug 2026 15:27:51 +0200 Subject: [PATCH 04/14] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Introduce=20using=20?= =?UTF-8?q?everywhere?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lamindb/models/save.py | 36 +++++++++++++------ tests/pydata/test_save.py | 75 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 100 insertions(+), 11 deletions(-) diff --git a/lamindb/models/save.py b/lamindb/models/save.py index be357e2a5..3661728fe 100644 --- a/lamindb/models/save.py +++ b/lamindb/models/save.py @@ -86,10 +86,13 @@ def save( lambda r: r._state.adding or r.pk is None, non_artifacts ) bulk_create( - non_artifacts_new, ignore_conflicts=ignore_conflicts, batch_size=batch_size + non_artifacts_new, + ignore_conflicts=ignore_conflicts, + batch_size=batch_size, + using=using, ) if non_artifacts_old: - bulk_update(non_artifacts_old, batch_size=batch_size) + bulk_update(non_artifacts_old, batch_size=batch_size, using=using) non_artifacts_with_parents = [ r for r in non_artifacts_new if hasattr(r, "_parents") ] @@ -117,7 +120,7 @@ def save( record, "_to_store", False ): record._storage_ongoing = True - record._save_skip_storage() + record._save_skip_storage(using=using) store_artifacts(artifacts, using=using) # this function returns None as potentially 10k records might be saved @@ -130,6 +133,7 @@ def bulk_create( records: Iterable[SQLRecord], ignore_conflicts: bool | None = False, batch_size: int = 10000, + using: str | None = None, ): """Create records in batches for safety and performance. @@ -137,6 +141,7 @@ def bulk_create( records: Iterable of SQLRecord objects to create ignore_conflicts: Whether to ignore conflicts during creation batch_size: Number of records to process in each batch. + using: Optional database alias used for bulk operations. """ records_by_orm = defaultdict(list) for record in records: @@ -145,6 +150,9 @@ def bulk_create( for registry, records_list in records_by_orm.items(): total_records = len(records_list) model_name = registry.__name__ + manager = ( + registry.objects.using(using) if using is not None else registry.objects + ) if total_records > batch_size: logger.important( f"starting creation of {total_records} {model_name} records in batches of {batch_size}" @@ -161,7 +169,7 @@ def bulk_create( f"processing batch {batch_num}/{total_batches} for {model_name}: {len(batch)} records" ) try: - registry.objects.bulk_create(batch, ignore_conflicts=ignore_conflicts) + manager.bulk_create(batch, ignore_conflicts=ignore_conflicts) # handle unique constraint violations due to non-default branches except IntegrityError as e: error_msg = str(e) @@ -189,7 +197,7 @@ def bulk_create( q_objects |= Q(**field_kwargs) # Query against non-default branches - pre_existing_records_not_main_branch = registry.objects.filter( + pre_existing_records_not_main_branch = manager.filter( q_objects ).exclude(branch_id=1) @@ -206,7 +214,7 @@ def bulk_create( if tuple(getattr(r, field) for field in unique_fields) not in pre_existing_value_tuples ] - save(records_main_branch) + save(records_main_branch, using=using) # Now move the pre-existing records to the main branch if pre_existing_value_tuples: @@ -221,7 +229,10 @@ def bulk_create( in pre_existing_value_tuples ] for record in pre_existing_records_to_move: - record.save() + if using is None: + record.save() + else: + record.save(using=using) else: raise e @@ -231,6 +242,7 @@ def bulk_update( ignore_conflicts: bool | None = False, batch_size: int = 10000, update_fields: list[str] | None = None, + using: str | None = None, ): """Update records in batches for safety and performance. @@ -239,6 +251,7 @@ def bulk_update( ignore_conflicts: Whether to ignore conflicts during update (currently unused but kept for consistency) batch_size: Number of records to process in each batch. If None, processes all at once. update_fields: Specific fields to update. If None, updates all fields except created_at and id. + using: Optional database alias used for bulk operations. """ records_by_orm = defaultdict(list) for record in records: @@ -247,6 +260,9 @@ def bulk_update( for registry, records_list in records_by_orm.items(): total_records = len(records_list) model_name = registry.__name__ + manager = ( + registry.objects.using(using) if using is not None else registry.objects + ) if total_records > batch_size: logger.warning( f"starting update for {total_records} {model_name} records in batches of {batch_size}" @@ -268,7 +284,7 @@ def bulk_update( logger.info( f"processing batch {batch_num}/{total_batches} for {model_name}: {len(batch)} records" ) - registry.objects.bulk_update(batch, field_names) + manager.bulk_update(batch, field_names) # This is also used within Artifact.save() @@ -439,7 +455,7 @@ def store_artifacts(artifacts: Iterable[Artifact], using: str | None = None) -> if artifact._storage_ongoing: artifact._storage_ongoing = False # each .save() is a separate transaction below - super(Artifact, artifact).save() + super(Artifact, artifact).save(using=using) # if check_and_attempt_upload was successful # then this can have only ._clear_storagekey from .replace exception = check_and_attempt_clearing( @@ -454,7 +470,7 @@ def store_artifacts(artifacts: Iterable[Artifact], using: str | None = None) -> with transaction.atomic(): for artifact in artifacts: if artifact not in stored_artifacts: - artifact._delete_skip_storage() + artifact._delete_skip_storage(using=using) # clean up storage after failure in check_and_attempt_upload exception_clear = check_and_attempt_clearing( artifact, raise_file_not_found_error=False, using_key=using diff --git a/tests/pydata/test_save.py b/tests/pydata/test_save.py index 847589e43..b07fbdf92 100644 --- a/tests/pydata/test_save.py +++ b/tests/pydata/test_save.py @@ -1,11 +1,17 @@ # ruff: noqa: F811 import lamindb as ln +import lamindb.models.save as save_module import pytest from _dataset_fixtures import ( # noqa get_mini_csv, ) -from lamindb.models.save import prepare_error_message, store_artifacts +from lamindb.models.save import ( + bulk_create, + bulk_update, + prepare_error_message, + store_artifacts, +) def test_bulk_save_and_update(): @@ -160,6 +166,73 @@ def test_bulk_save_lazy_record_features_requires_schema(): ln.Record.filter(name="lazy-no-schema-type").delete(permanent=True) +def test_bulk_save_passes_using_to_bulk_operations(monkeypatch): + captured: dict[str, str | None] = {"create": None, "update": None} + + def _bulk_create(*args, **kwargs): + captured["create"] = kwargs.get("using") + + def _bulk_update(*args, **kwargs): + captured["update"] = kwargs.get("using") + + monkeypatch.setattr(save_module, "bulk_create", _bulk_create) + monkeypatch.setattr(save_module, "bulk_update", _bulk_update) + + existing = ln.Record(name="bulk-using-existing").save() + existing.name = "bulk-using-existing-updated" + new = ln.Record(name="bulk-using-new") + + ln.save([new, existing], using="default") + + assert captured["create"] == "default" + assert captured["update"] == "default" + existing.delete(permanent=True) + + +def test_bulk_helpers_use_manager_using(): + class DummyManager: + def __init__(self): + self.current_alias = None + self.bulk_create_alias = None + self.bulk_update_alias = None + + def using(self, alias): + self.current_alias = alias + return self + + def bulk_create(self, _batch, ignore_conflicts=False): + self.bulk_create_alias = self.current_alias + + def bulk_update(self, _batch, _field_names): + self.bulk_update_alias = self.current_alias + + class DummyField: + def __init__(self, name): + self.name = name + + class DummyRecord: + objects = DummyManager() + _meta = type( + "Meta", + (), + { + "fields": [ + DummyField("id"), + DummyField("created_at"), + DummyField("name"), + ] + }, + )() + + records = [DummyRecord(), DummyRecord()] + + bulk_create(records, using="analytics") + bulk_update(records, using="analytics") + + assert DummyRecord.objects.bulk_create_alias == "analytics" + assert DummyRecord.objects.bulk_update_alias == "analytics" + + def test_bulk_resave_trashed_records(): import bionty as bt From 33fa098ff392a780201e0a1f5e30bc5b1ea1fded Mon Sep 17 00:00:00 2001 From: Alex Wolf Date: Thu, 13 Aug 2026 12:44:57 +0200 Subject: [PATCH 05/14] =?UTF-8?q?=E2=8F=AA=EF=B8=8F=20Do=20not=20bump=20la?= =?UTF-8?q?mindb-setup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sub/lamindb-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sub/lamindb-setup b/sub/lamindb-setup index 307c46e1e..194e2e491 160000 --- a/sub/lamindb-setup +++ b/sub/lamindb-setup @@ -1 +1 @@ -Subproject commit 307c46e1e64b73def27c9fd3d137a850ab1eaa87 +Subproject commit 194e2e4912922298ebee609f13d303b8e339cad0 From 49ebaddc3db9268d017c9b90b234032669a11472 Mon Sep 17 00:00:00 2001 From: Alex Wolf Date: Thu, 13 Aug 2026 12:45:12 +0200 Subject: [PATCH 06/14] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Register=20the=20sec?= =?UTF-8?q?ond=20database?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lamindb/models/save.py | 23 ++++++++++++++++++++++- lamindb/models/sqlrecord.py | 11 +++++++++++ tests/pydata/test_save.py | 6 ++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/lamindb/models/save.py b/lamindb/models/save.py index 3661728fe..f9a2bbf7b 100644 --- a/lamindb/models/save.py +++ b/lamindb/models/save.py @@ -8,7 +8,7 @@ from datetime import datetime from typing import TYPE_CHECKING -from django.db import IntegrityError, transaction +from django.db import IntegrityError, connections, transaction from django.utils.functional import partition from lamin_utils import logger from lamindb_setup.core.upath import LocalPathClasses, UPath @@ -26,6 +26,20 @@ from .artifact import Artifact +def _ensure_using_connection(registry: type[SQLRecord], using: str | None) -> None: + if using is None or using == "default" or using in connections: + return + registry.connect(using) + + +def _prepare_cross_instance_create(record: SQLRecord, using: str | None) -> None: + if using is None or using == "default": + return + if (record._state.adding or record.pk is None) and hasattr(record, "run_id"): + record.run = None + record.run_id = None + + def save( records: Iterable[SQLRecord], ignore_conflicts: bool | None = False, @@ -82,6 +96,8 @@ def save( # for artifacts, we want to bulk-upload rather than upload one-by-one non_artifacts, artifacts = partition(lambda r: isinstance(r, Artifact), records) if non_artifacts: + for record in non_artifacts: + _prepare_cross_instance_create(record, using) non_artifacts_old, non_artifacts_new = partition( lambda r: r._state.adding or r.pk is None, non_artifacts ) @@ -113,6 +129,9 @@ def save( bulk_set_features_in_records(records_with_lazy_features) if artifacts: + for record in artifacts: + _prepare_cross_instance_create(record, using) + _ensure_using_connection(Artifact, using) with transaction.atomic(): for record in artifacts: # will switch to True after the successful upload / saving @@ -148,6 +167,7 @@ def bulk_create( records_by_orm[record.__class__].append(record) for registry, records_list in records_by_orm.items(): + _ensure_using_connection(registry, using) total_records = len(records_list) model_name = registry.__name__ manager = ( @@ -258,6 +278,7 @@ def bulk_update( records_by_orm[record.__class__].append(record) for registry, records_list in records_by_orm.items(): + _ensure_using_connection(registry, using) total_records = len(records_list) model_name = registry.__name__ manager = ( diff --git a/lamindb/models/sqlrecord.py b/lamindb/models/sqlrecord.py index ee7aa6137..fe0ca6464 100644 --- a/lamindb/models/sqlrecord.py +++ b/lamindb/models/sqlrecord.py @@ -1341,6 +1341,17 @@ def save(self: T, *args, **kwargs) -> T: using = None if "using" in kwargs: using = kwargs["using"] + if using != "default" and using not in connections: + self.__class__.connect(using) + # cross-instance writes can inherit a run from the active local context; + # that run id does not exist in the target instance. + if ( + using != "default" + and (self._state.adding or self.pk is None) + and hasattr(self, "run_id") + ): + self.run = None + self.run_id = None transfer_config = kwargs.pop("transfer", None) db = self._state.db pk_on_db = self.pk diff --git a/tests/pydata/test_save.py b/tests/pydata/test_save.py index b07fbdf92..4bd18d9e4 100644 --- a/tests/pydata/test_save.py +++ b/tests/pydata/test_save.py @@ -211,6 +211,7 @@ def __init__(self, name): self.name = name class DummyRecord: + connect_alias = None objects = DummyManager() _meta = type( "Meta", @@ -224,11 +225,16 @@ class DummyRecord: }, )() + @classmethod + def connect(cls, using): + cls.connect_alias = using + records = [DummyRecord(), DummyRecord()] bulk_create(records, using="analytics") bulk_update(records, using="analytics") + assert DummyRecord.connect_alias == "analytics" assert DummyRecord.objects.bulk_create_alias == "analytics" assert DummyRecord.objects.bulk_update_alias == "analytics" From e213765a204cc61319a61f7b0d27c2990af54c79 Mon Sep 17 00:00:00 2001 From: Alex Wolf Date: Thu, 13 Aug 2026 13:10:42 +0200 Subject: [PATCH 07/14] =?UTF-8?q?=E2=9C=85=20Add=20reasonable=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/pydata/test_save.py | 76 ----------------------- tests/transfer/test_save_to_another_db.py | 42 +++++++++++++ 2 files changed, 42 insertions(+), 76 deletions(-) create mode 100644 tests/transfer/test_save_to_another_db.py diff --git a/tests/pydata/test_save.py b/tests/pydata/test_save.py index 4bd18d9e4..3151f3451 100644 --- a/tests/pydata/test_save.py +++ b/tests/pydata/test_save.py @@ -1,14 +1,11 @@ # ruff: noqa: F811 import lamindb as ln -import lamindb.models.save as save_module import pytest from _dataset_fixtures import ( # noqa get_mini_csv, ) from lamindb.models.save import ( - bulk_create, - bulk_update, prepare_error_message, store_artifacts, ) @@ -166,79 +163,6 @@ def test_bulk_save_lazy_record_features_requires_schema(): ln.Record.filter(name="lazy-no-schema-type").delete(permanent=True) -def test_bulk_save_passes_using_to_bulk_operations(monkeypatch): - captured: dict[str, str | None] = {"create": None, "update": None} - - def _bulk_create(*args, **kwargs): - captured["create"] = kwargs.get("using") - - def _bulk_update(*args, **kwargs): - captured["update"] = kwargs.get("using") - - monkeypatch.setattr(save_module, "bulk_create", _bulk_create) - monkeypatch.setattr(save_module, "bulk_update", _bulk_update) - - existing = ln.Record(name="bulk-using-existing").save() - existing.name = "bulk-using-existing-updated" - new = ln.Record(name="bulk-using-new") - - ln.save([new, existing], using="default") - - assert captured["create"] == "default" - assert captured["update"] == "default" - existing.delete(permanent=True) - - -def test_bulk_helpers_use_manager_using(): - class DummyManager: - def __init__(self): - self.current_alias = None - self.bulk_create_alias = None - self.bulk_update_alias = None - - def using(self, alias): - self.current_alias = alias - return self - - def bulk_create(self, _batch, ignore_conflicts=False): - self.bulk_create_alias = self.current_alias - - def bulk_update(self, _batch, _field_names): - self.bulk_update_alias = self.current_alias - - class DummyField: - def __init__(self, name): - self.name = name - - class DummyRecord: - connect_alias = None - objects = DummyManager() - _meta = type( - "Meta", - (), - { - "fields": [ - DummyField("id"), - DummyField("created_at"), - DummyField("name"), - ] - }, - )() - - @classmethod - def connect(cls, using): - cls.connect_alias = using - - records = [DummyRecord(), DummyRecord()] - - bulk_create(records, using="analytics") - bulk_update(records, using="analytics") - - assert DummyRecord.connect_alias == "analytics" - assert DummyRecord.objects.bulk_create_alias == "analytics" - assert DummyRecord.objects.bulk_update_alias == "analytics" - - def test_bulk_resave_trashed_records(): import bionty as bt diff --git a/tests/transfer/test_save_to_another_db.py b/tests/transfer/test_save_to_another_db.py new file mode 100644 index 000000000..3def55b1e --- /dev/null +++ b/tests/transfer/test_save_to_another_db.py @@ -0,0 +1,42 @@ +import lamindb as ln + + +def test_save_ulabel_to_another_db_via_model_save(): + assert ln.setup.settings.instance.name == "testdb2" + + using = f"{ln.setup.settings.user.handle}/testdb1" + db2 = ln.DB(using) + name = "save-using-single" + remote_labels = db2.ULabel.filter(name=name) + if remote_labels.exists(): + remote_labels.delete(permanent=True) + + try: + ulabel = ln.ULabel(name=name).save(using=using) + + assert ulabel._state.db == using + assert db2.ULabel.get(name=name) + assert ln.ULabel.filter(name=name).count() == 0 + finally: + db2.ULabel.filter(name=name).delete(permanent=True) + + +def test_save_ulabels_to_another_db_via_ln_save(): + assert ln.setup.settings.instance.name == "testdb2" + + using = f"{ln.setup.settings.user.handle}/testdb1" + db2 = ln.DB(using) + names = [f"save-using-bulk-{i}" for i in range(3)] + remote_labels = db2.ULabel.filter(name__in=names) + if remote_labels.exists(): + remote_labels.delete(permanent=True) + + try: + ulabels = [ln.ULabel(name=name) for name in names] + ln.save(ulabels, using=using) + + remote_matches = db2.ULabel.filter(name__in=names) + assert remote_matches.count() == len(names) + assert ln.ULabel.filter(name__in=names).count() == 0 + finally: + db2.ULabel.filter(name__in=names).delete(permanent=True) From af68bb24f2d6d874b6a2b7e6c6faaa06f8ed36ad Mon Sep 17 00:00:00 2001 From: Raaghav-Pillai Date: Thu, 13 Aug 2026 17:40:03 +0530 Subject: [PATCH 08/14] Adding testing for record saving and linking when connected to different database --- tests/transfer/test_save_to_another_db.py | 24 +++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/transfer/test_save_to_another_db.py b/tests/transfer/test_save_to_another_db.py index 3def55b1e..364ac9e6e 100644 --- a/tests/transfer/test_save_to_another_db.py +++ b/tests/transfer/test_save_to_another_db.py @@ -40,3 +40,27 @@ def test_save_ulabels_to_another_db_via_ln_save(): assert ln.ULabel.filter(name__in=names).count() == 0 finally: db2.ULabel.filter(name__in=names).delete(permanent=True) + + +def test_save_record_with_relation_to_another_db_via_model_save(): + assert ln.setup.settings.instance.name == "testdb2" + + using = f"{ln.setup.settings.user.handle}/testdb1" + db2 = ln.DB(using) + type_name = "save-using-relation-type" + rec_name = "save-using-relation-child" + + for qs in (db2.Record.filter(name=rec_name), db2.Record.filter(name=type_name)): + if qs.exists(): + qs.delete(permanent=True) + + try: + rec_type = ln.Record(name=type_name, is_type=True).save(using=using) + child = ln.Record(name=rec_name, type=rec_type).save(using=using) + + assert child._state.db == using + assert db2.Record.get(name=rec_name) + assert ln.Record.filter(name=rec_name).count() == 0 + finally: + for qs in (db2.Record.filter(name=rec_name), db2.Record.filter(name=type_name)): + qs.delete(permanent=True) From 1d25d535dbfa487aeccf52cb71a1687069ff16dc Mon Sep 17 00:00:00 2001 From: Raaghav-Pillai Date: Thu, 13 Aug 2026 17:58:01 +0530 Subject: [PATCH 09/14] :recycle:adding feature so that databse linking works while being connected to different database --- lamindb/models/sqlrecord.py | 22 ++++++++++++++++++++++ tests/transfer/conftest.py | 8 ++++++++ 2 files changed, 30 insertions(+) diff --git a/lamindb/models/sqlrecord.py b/lamindb/models/sqlrecord.py index fe0ca6464..ef09cf7bf 100644 --- a/lamindb/models/sqlrecord.py +++ b/lamindb/models/sqlrecord.py @@ -2388,6 +2388,27 @@ def get_name_field( return field +class LaminDBRouter: + def allow_relation(self, obj1, obj2, **hints) -> bool | None: + if isinstance(obj1, SQLRecord) and isinstance(obj2, SQLRecord): + return True + return None + + +def _ensure_lamindb_router() -> None: + from django.conf import settings as django_settings + from django.db import router as django_router + + router_path = "lamindb.models.sqlrecord.LaminDBRouter" + existing = list(getattr(django_settings, "DATABASE_ROUTERS", [])) + if router_path in existing or any( + isinstance(r, LaminDBRouter) for r in existing + ): + return + django_settings.DATABASE_ROUTERS = [*existing, router_path] + django_router.__dict__.pop("routers", None) + + def add_db_connection(db: str, using: str): db_config = dj_database_url.config( default=db, conn_max_age=600, conn_health_checks=True @@ -2396,6 +2417,7 @@ def add_db_connection(db: str, using: str): db_config["OPTIONS"] = {} db_config["AUTOCOMMIT"] = True connections.settings[using] = db_config + _ensure_lamindb_router() REGISTRY_UNIQUE_FIELD = {"storage": "root", "ulabel": "name"} diff --git a/tests/transfer/conftest.py b/tests/transfer/conftest.py index fa247c8b6..fdfd6bc87 100644 --- a/tests/transfer/conftest.py +++ b/tests/transfer/conftest.py @@ -4,10 +4,17 @@ import pytest +def _close_all_connections(): + from django.db import connections + + connections.close_all() + + @pytest.fixture(scope="session", autouse=True) def setup_testdb1(): ln.setup.init(storage="./testdb1") yield + _close_all_connections() shutil.rmtree("./testdb1") ln.setup.delete("testdb1", force=True) @@ -16,6 +23,7 @@ def setup_testdb1(): def setup_testdb2(): ln.setup.init(storage="./testdb2") yield + _close_all_connections() shutil.rmtree("./testdb2") ln.setup.delete("testdb2", force=True) From df95170763cf2ae7a63ae153b346a2bb87f7bad2 Mon Sep 17 00:00:00 2001 From: Raaghav-Pillai Date: Thu, 13 Aug 2026 18:14:41 +0530 Subject: [PATCH 10/14] adding testing for multivalue linking in records --- tests/transfer/test_save_to_another_db.py | 42 +++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/transfer/test_save_to_another_db.py b/tests/transfer/test_save_to_another_db.py index 364ac9e6e..e778c062e 100644 --- a/tests/transfer/test_save_to_another_db.py +++ b/tests/transfer/test_save_to_another_db.py @@ -64,3 +64,45 @@ def test_save_record_with_relation_to_another_db_via_model_save(): finally: for qs in (db2.Record.filter(name=rec_name), db2.Record.filter(name=type_name)): qs.delete(permanent=True) + +def test_save_record_with_multivalued_relation_to_another_db_via_model_save(): + assert ln.setup.settings.instance.name == "testdb2" + + using = f"{ln.setup.settings.user.handle}/testdb1" + db2 = ln.DB(using) + + type_name = "mv-intervention-type" + gene_type_name = "mv-gene-type" + gene_feat_name = "mv-genes" + gene_names = [f"mv-gene-{i}" for i in range(5)] + rec_name = "mv-child" + + def _clean(): + for qs in ( + db2.Record.filter(name=rec_name), + db2.Record.filter(name__in=gene_names), + db2.Record.filter(name=type_name), + db2.Record.filter(name=gene_type_name), + db2.Feature.filter(name=gene_feat_name), + ): + if qs.exists(): + qs.delete(permanent=True) + + _clean() + try: + intervention_t = ln.Record(name=type_name, is_type=True).save(using=using) + gene_t = ln.Record(name=gene_type_name, is_type=True).save(using=using) + gene_feat = ln.Feature(name=gene_feat_name, dtype=gene_t).save(using=using) + gene_pool = [ln.Record(name=g, type=gene_t).save(using=using) for g in gene_names] + + child = ln.Record( + name=rec_name, + type=intervention_t, + features={gene_feat: gene_pool[:3]}, + ).save(using=using) + + assert child._state.db == using + assert db2.Record.get(name=rec_name) + assert ln.Record.filter(name=rec_name).count() == 0 + finally: + _clean() From 6f4977f09b3d12ebd761d049dea5c38d0c40699a Mon Sep 17 00:00:00 2001 From: Raaghav-Pillai Date: Thu, 13 Aug 2026 18:31:41 +0530 Subject: [PATCH 11/14] adding functionality for multivalue linking in records with using --- lamindb/curators/core.py | 38 ++++++++++++++++++++++++++---- lamindb/models/_feature_manager.py | 16 +++++++++---- lamindb/models/_from_values.py | 5 +++- lamindb/models/feature.py | 7 +++++- 4 files changed, 54 insertions(+), 12 deletions(-) diff --git a/lamindb/curators/core.py b/lamindb/curators/core.py index 68dbd9d30..886ec32a7 100644 --- a/lamindb/curators/core.py +++ b/lamindb/curators/core.py @@ -664,17 +664,21 @@ def __init__( schema: Schema, slot: str | None = None, require_saved_schema: bool = True, + instance: str | None = None, ) -> None: super().__init__( dataset=dataset, schema=schema, require_saved_schema=require_saved_schema ) + self._instance = instance categoricals = [] features = [] feature_ids: set[int] = set() if schema.flexible: - features += Feature.filter(name__in=self._dataset.keys()).to_list() + features += Feature.filter( + name__in=self._dataset.keys(), _using_key=instance + ).to_list() feature_ids = {feature.id for feature in features} if schema.n_members and schema.n_members > 0: @@ -847,6 +851,7 @@ def __init__( categoricals=categoricals, index=schema.index, slot=slot, + instance=instance, maximal_set=schema.maximal_set, schema=schema, ) @@ -978,6 +983,7 @@ def __init__( slot: str | None = None, features: dict[str, Any] | None = None, require_saved_schema: bool = True, + instance: str | None = None, ) -> None: # loads or opens dataset, dataset may be an artifact super().__init__( @@ -986,12 +992,14 @@ def __init__( features=features, require_saved_schema=require_saved_schema, ) + self._instance = instance # uses open dataset at self._dataset self._atomic_curator = ComponentCurator( dataset=self._dataset, schema=schema, slot=slot, require_saved_schema=require_saved_schema, + instance=instance, ) # Handle (nested) attrs if slot is None and schema.slots: @@ -1013,6 +1021,7 @@ def __init__( slot_schema, slot=slot_name, require_saved_schema=require_saved_schema, + instance=instance, ) elif slot_name != "__external__": raise ValueError( @@ -1066,6 +1075,7 @@ def __init__( schema: Schema, slot: str | None = None, require_saved_schema: bool = False, + instance: str | None = None, ) -> None: if not isinstance(dataset, dict) and not isinstance(dataset, Artifact): raise InvalidArgument("The dataset must be a dict or dict-like artifact.") @@ -1076,7 +1086,11 @@ def __init__( d = dataset df = convert_dict_to_dataframe_for_validation(d, schema) # type: ignore super().__init__( - df, schema, slot=slot, require_saved_schema=require_saved_schema + df, + schema, + slot=slot, + require_saved_schema=require_saved_schema, + instance=instance, ) @@ -1545,6 +1559,7 @@ def __init__( type_uid: str | None = None, maximal_set: bool = True, # whether unvalidated categoricals cause validation failure. schema: Schema = None, + instance: str | None = None, # target instance for cross-instance curation ) -> None: self._values_getter = values_getter self._values_setter = values_setter @@ -1561,6 +1576,7 @@ def __init__( self.records = None self._maximal_set = maximal_set self._type_record = None + self._instance = instance self._registry = self._field.field.model self._field_name = self._field.field.name self._filter_kwargs = {} @@ -1590,6 +1606,7 @@ def __init__( self._type_record = get_record_type_from_uid( self._registry, self._type_uid, + using=self._instance, ) if hasattr(self._registry, "_name_field"): @@ -1774,7 +1791,7 @@ def _add_validated(self) -> tuple[list, list]: # When we have a Schema with typed members, # scope the query to the types present in the schema's members (plus untyped features) # to avoid ambiguous matches across different feature types. - qs = registry.filter() + qs = registry.filter(_using_key=self._instance) if self._schema and self._schema.n_members: type_ids = { m.type_id @@ -1783,7 +1800,8 @@ def _add_validated(self) -> tuple[list, list]: } if type_ids: qs = registry.filter( - Q(type_id__in=type_ids) | Q(type_id__isnull=True) + Q(type_id__in=type_ids) | Q(type_id__isnull=True), + _using_key=self._instance, ) self._subtype_query_set = qs else: @@ -1837,6 +1855,7 @@ def _add_validated(self) -> tuple[list, list]: remaining_values, field=field, mute=True, + using=self._instance, **filter_kwargs, # type: ignore ) existing_and_public_values = [ @@ -1944,7 +1963,11 @@ def _validate( if self._subtype_query_set is not None and registry == self._registry: registry_or_queryset = self._subtype_query_set # first inspect against the registry - inspect_result = registry_or_queryset.filter(**filter_kwargs).inspect( + if registry_or_queryset is registry: + queryset = registry.filter(_using_key=self._instance, **filter_kwargs) + else: + queryset = registry_or_queryset.filter(**filter_kwargs) + inspect_result = queryset.inspect( non_validated, field=field, mute=True, @@ -2074,10 +2097,12 @@ def __init__( slot: str | None = None, maximal_set: bool = False, schema: Schema | None = None, + instance: str | None = None, ) -> None: self._non_validated = None self._index = index self._schema = schema + self._instance = instance self._artifact: Artifact = None # pass the dataset as an artifact self._dataset: Any = df # pass the dataset as an AnyPathStr or data object if isinstance(self._dataset, Artifact): @@ -2115,6 +2140,7 @@ def __init__( if schema.id is None else f"schemas__id={schema.id}", schema=schema, + instance=instance, ) for feature in self._categoricals: result = parse_dtype(feature._dtype_str)[0] @@ -2135,6 +2161,7 @@ def __init__( cat_manager=self, filter_str=result["filter_str"], type_uid=result.get("type_uid"), + instance=instance, ) if index is not None and index._dtype_str.startswith("cat"): result = parse_dtype(index._dtype_str)[0] @@ -2150,6 +2177,7 @@ def __init__( cat_manager=self, filter_str=result["filter_str"], type_uid=result.get("type_uid"), + instance=instance, ) @property diff --git a/lamindb/models/_feature_manager.py b/lamindb/models/_feature_manager.py index 82b7d0018..5617518c4 100644 --- a/lamindb/models/_feature_manager.py +++ b/lamindb/models/_feature_manager.py @@ -1700,7 +1700,10 @@ def add_values( ) schema = Schema(feature_objects) ExperimentalDictCurator( - dictionary, schema, require_saved_schema=False + dictionary, + schema, + require_saved_schema=False, + instance=self._host._state.db, ).validate() if host_is_record and schema.index is not None: from .record import strip_index_for_record_persistence @@ -1730,6 +1733,7 @@ def _add_values( from ..base.dtypes import is_iterable_of_sqlrecord from .can_curate import CanCurate + host_db = self._host._state.db host_is_record = self._host.__class__.__name__ == "Record" if host_is_record: feature_json_values: list[SQLRecord] = [] @@ -1746,12 +1750,12 @@ def _add_values( ) self._raise_not_validated_values(record_not_validated_values) if feature_json_values: - save(feature_json_values) + save(feature_json_values, using=host_db) for links in links_by_model.values(): try: - save(links, ignore_conflicts=False) + save(links, ignore_conflicts=False, using=host_db) except Exception: - save(links, ignore_conflicts=True) + save(links, ignore_conflicts=True, using=host_db) from .record import get_type_schema_index, persist_record_name if ( @@ -1918,7 +1922,9 @@ def set_values( if host_is_artifact: schema = self._get_external_schema() if schema is not None: - ExperimentalDictCurator(dictionary, schema).validate() + ExperimentalDictCurator( + dictionary, schema, instance=self._host._state.db + ).validate() member_ids = set(schema.members.values_list("id", flat=True)) features_not_in_schema = [ feature.name diff --git a/lamindb/models/_from_values.py b/lamindb/models/_from_values.py index d7123f2ff..05be6131e 100644 --- a/lamindb/models/_from_values.py +++ b/lamindb/models/_from_values.py @@ -57,6 +57,7 @@ def _from_values( standardize: bool = True, from_source: bool = True, mute: bool = False, + using: str | None = None, **filter_kwargs, ) -> SQLRecordList: """Get or create records from iterables.""" @@ -84,6 +85,7 @@ def _from_values( field=field, organism=organism_record, mute=mute, + using=using, **filter_kwargs, ) @@ -135,6 +137,7 @@ def get_existing_records( organism: SQLRecord | None = None, standardize: bool = True, mute: bool = False, + using: str | None = None, **filter_kwargs, ) -> tuple[list, Index, str]: """Get existing records from the database.""" @@ -144,7 +147,7 @@ def get_existing_records( # NOTE: existing records matching is agnostic to the source registry = field.field.model # type: ignore - queryset = registry.filter(**filter_kwargs) + queryset = registry.filter(_using_key=using, **filter_kwargs) if standardize: # log synonyms mapped terms diff --git a/lamindb/models/feature.py b/lamindb/models/feature.py index ae0078b7d..86861d909 100644 --- a/lamindb/models/feature.py +++ b/lamindb/models/feature.py @@ -171,8 +171,13 @@ def transfer_feature_dtypes( def get_record_type_from_uid( registry: Registry, type_uid: str, + using: str | None = None, ) -> SQLRecord: - type_record: SQLRecord = registry.get(type_uid) + type_record: SQLRecord = ( + registry.get(type_uid) + if using is None + else registry.connect(using).get(type_uid) + ) if type_record.branch_id == -1: warning_msg = f"retrieving {registry.__name__} type '{type_record.name}' (uid='{type_uid}') from trash" From 1397eca1071a5703f809fb81e8c2170d71f76929 Mon Sep 17 00:00:00 2001 From: Raaghav-Pillai Date: Thu, 13 Aug 2026 19:52:16 +0530 Subject: [PATCH 12/14] Adding functionaly so that default is hte current database and when using using it searches for links in that database --- lamindb/models/schema.py | 99 +++++++++++++++++++++-- tests/transfer/test_save_to_another_db.py | 67 +++++++++++++++ 2 files changed, 158 insertions(+), 8 deletions(-) diff --git a/lamindb/models/schema.py b/lamindb/models/schema.py index c92a49554..76ff1d962 100644 --- a/lamindb/models/schema.py +++ b/lamindb/models/schema.py @@ -93,6 +93,22 @@ def validate_features(features: list[SQLRecord]) -> SQLRecord: return next(iter(feature_types)) # return value in set of cardinality 1 +def _resolve_pks_on_instance( + registry: type[SQLRecord], uids: list[str], using: str | None +) -> list[int]: + """Resolve primary keys on a target instance from stable `uid`s (order-preserving). + + Primary keys are per-instance, whereas `uid` is stable across instances. When + writing link-table rows to a different instance via `using=`, the in-memory + `.id` of a related record is not guaranteed to match that instance's row, so we + look up the ids by `uid` on the target connection. + """ + uid_to_pk = dict( + registry.objects.using(using).filter(uid__in=uids).values_list("uid", "id") + ) + return [uid_to_pk[uid] for uid in uids] + + def get_features_config( features: list[SQLRecord] | tuple[SQLRecord, dict], ) -> tuple[list[SQLRecord], list[tuple[SQLRecord, dict]]]: @@ -1093,6 +1109,33 @@ def from_df( ) -> Schema | None: return cls.from_dataframe(df, field, name, mute, organism, source) + def _infer_members_instance(self) -> str | None: + """Return the non-default instance the pending members live on, if any. + + Pending schema members (``_features``) and slot components (``_slots``) + carry ``_state.db``. When they were created on another instance via + ``using=``, a bare ``.save()`` should follow them there. Returns ``None`` + when members are local (the same-instance fast path) and raises when they + span more than one instance. + """ + candidates: set[str] = set() + if hasattr(self, "_features"): + for record in self._features[1]: + db = record._state.db + if db is not None and db != "default": + candidates.add(db) + if hasattr(self, "_slots"): + for component in self._slots.values(): + db = component._state.db + if db is not None and db != "default": + candidates.add(db) + if len(candidates) > 1: + raise InvalidArgument( + f"schema members live on multiple instances ({sorted(candidates)}); " + "a schema must be saved to a single instance." + ) + return next(iter(candidates)) if candidates else None + def save(self, *args, **kwargs) -> Schema: """Save schema. @@ -1121,6 +1164,23 @@ def save(self, *args, **kwargs) -> Schema: index_name_conflict = kwargs.pop("index_name_conflict", None) using = kwargs.get("using") or self._state.db + # a schema must live on the same instance as its members. When they were + # created on another instance via `using=` and the schema is saved without + # an explicit target, follow the members there (bare `.save()` should not + # split the schema from its features). Same-instance members are local, so + # this is a no-op for the normal path. + members_instance = self._infer_members_instance() + if members_instance is not None: + if using is None or using == "default": + using = members_instance + kwargs["using"] = members_instance + elif using != members_instance: + raise InvalidArgument( + f"schema is being saved to '{using}' but its members live on " + f"'{members_instance}'; save the schema to the same instance as " + "its members." + ) + if self.pk is not None: existing_features = self.members.to_list() if self.members.exists() else [] if hasattr(self, "_features"): @@ -1191,14 +1251,30 @@ def save(self, *args, **kwargs) -> Schema: index_name_conflict=index_name_conflict, ) super().save(*args, **kwargs) + # link-table rows reference per-instance primary keys; when writing to a + # different instance via `using=`, resolve both endpoints by their stable + # `uid` on the target connection. Same-instance saves keep the fast path. + cross_instance = using is not None and using != "default" + schema_id = ( + _resolve_pks_on_instance(type(self), [self.uid], using)[0] + if cross_instance + else self.id + ) if hasattr(self, "_slots"): # analogous to save_schema_links in core._data.py # which is called to save feature sets in artifact.save() + slots = list(self._slots.items()) + if cross_instance: + component_ids = _resolve_pks_on_instance( + Schema, [component.uid for _, component in slots], using + ) + else: + component_ids = [component.id for _, component in slots] links = [] - for slot, component in self._slots.items(): + for (slot, _), component_id in zip(slots, component_ids): kwargs = { - "composite_id": self.id, - "component_id": component.id, + "composite_id": schema_id, + "component_id": component_id, "slot": slot, } links.append(Schema.components.through(**kwargs)) @@ -1223,18 +1299,25 @@ def save(self, *args, **kwargs) -> Schema: else: related_field = related_model_split[1].lower() related_field_id = f"{related_field}_id" - new_member_ids = [record.id for record in records] + if cross_instance: + new_member_ids = _resolve_pks_on_instance( + records[0].__class__, [record.uid for record in records], using + ) + else: + new_member_ids = [record.id for record in records] existing_member_ids = list( through_model.objects.using(using) - .filter(schema_id=self.id) + .filter(schema_id=schema_id) .order_by("id") .values_list(related_field_id, flat=True) ) if new_member_ids != existing_member_ids: - through_model.objects.using(using).filter(schema_id=self.id).delete() + through_model.objects.using(using).filter(schema_id=schema_id).delete() links = [ - through_model(**{"schema_id": self.id, related_field_id: record.id}) - for record in records + through_model( + **{"schema_id": schema_id, related_field_id: member_id} + ) + for member_id in new_member_ids ] through_model.objects.using(using).bulk_create(links) delattr(self, "_features") diff --git a/tests/transfer/test_save_to_another_db.py b/tests/transfer/test_save_to_another_db.py index e778c062e..ebf7f2fd0 100644 --- a/tests/transfer/test_save_to_another_db.py +++ b/tests/transfer/test_save_to_another_db.py @@ -106,3 +106,70 @@ def _clean(): assert ln.Record.filter(name=rec_name).count() == 0 finally: _clean() + + +def test_save_schema_with_features_to_another_db_via_model_save(): + assert ln.setup.settings.instance.name == "testdb2" + + using = f"{ln.setup.settings.user.handle}/testdb1" + db2 = ln.DB(using) + + feat_names = [f"save-using-schema-feat-{i}" for i in range(3)] + schema_name = "save-using-schema" + + def _clean(): + for qs in ( + db2.Schema.filter(name=schema_name), + db2.Feature.filter(name__in=feat_names), + ): + if qs.exists(): + qs.delete(permanent=True) + + _clean() + try: + features = [ + ln.Feature(name=name, dtype=str).save(using=using) for name in feat_names + ] + schema = ln.Schema(name=schema_name, features=features).save(using=using) + + assert schema._state.db == using + remote_schema = db2.Schema.get(name=schema_name) + assert set(remote_schema.members.to_list("name")) == set(feat_names) + assert ln.Schema.filter(name=schema_name).count() == 0 + assert ln.Feature.filter(name__in=feat_names).count() == 0 + finally: + _clean() + + +def test_save_schema_with_remote_features_infers_instance_via_bare_save(): + assert ln.setup.settings.instance.name == "testdb2" + + using = f"{ln.setup.settings.user.handle}/testdb1" + db2 = ln.DB(using) + + feat_names = [f"save-using-schema-infer-feat-{i}" for i in range(3)] + schema_name = "save-using-schema-infer" + + def _clean(): + for qs in ( + db2.Schema.filter(name=schema_name), + db2.Feature.filter(name__in=feat_names), + ): + if qs.exists(): + qs.delete(permanent=True) + + _clean() + try: + features = [ + ln.Feature(name=name, dtype=str).save(using=using) for name in feat_names + ] + # bare save: instance is inferred from the (remote) member features + schema = ln.Schema(name=schema_name, features=features).save() + + assert schema._state.db == using + remote_schema = db2.Schema.get(name=schema_name) + assert set(remote_schema.members.to_list("name")) == set(feat_names) + assert ln.Schema.filter(name=schema_name).count() == 0 + assert ln.Feature.filter(name__in=feat_names).count() == 0 + finally: + _clean() From 0a6cd462d4dd39efc874ef92b4ad81077766e5ab Mon Sep 17 00:00:00 2001 From: Raaghav-Pillai Date: Fri, 14 Aug 2026 23:41:43 +0530 Subject: [PATCH 13/14] :recycle: Final fixes to get the feature work --- lamindb/curators/core.py | 14 +- lamindb/models/__init__.py | 7 + lamindb/models/_feature_manager.py | 39 ++++- lamindb/models/artifact_set.py | 8 +- lamindb/models/feature.py | 12 +- lamindb/models/save.py | 2 +- lamindb/models/schema.py | 23 --- lamindb/models/sqlrecord.py | 8 + tests/transfer/test_save_to_another_db.py | 184 ++++++++++++++++++++++ 9 files changed, 259 insertions(+), 38 deletions(-) diff --git a/lamindb/curators/core.py b/lamindb/curators/core.py index 886ec32a7..60d001992 100644 --- a/lamindb/curators/core.py +++ b/lamindb/curators/core.py @@ -1584,7 +1584,9 @@ def __init__( if filter_str and filter_str != "unsaved": self._filter_kwargs.update( resolve_relation_filters( - parse_filter_string(filter_str), self._registry + parse_filter_string(filter_str), + self._registry, + using=self._instance, ) # type: ignore ) if self._registry.__base__.__name__ == "BioRecord": @@ -1765,7 +1767,9 @@ def _add_validated(self) -> tuple[list, list]: if filter_str: parsed_filters = parse_filter_string(filter_str) filter_kwargs.update( - resolve_relation_filters(parsed_filters, registry) + resolve_relation_filters( + parsed_filters, registry, using=self._instance + ) ) if registry.__base__.__name__ == "BioRecord": organism_record = get_organism_record_from_field( @@ -1958,7 +1962,11 @@ def _validate( filter_str = result.get("filter_str", "") if filter_str: parsed_filters = parse_filter_string(filter_str) - filter_kwargs.update(resolve_relation_filters(parsed_filters, registry)) + filter_kwargs.update( + resolve_relation_filters( + parsed_filters, registry, using=self._instance + ) + ) registry_or_queryset = registry if self._subtype_query_set is not None and registry == self._registry: registry_or_queryset = self._subtype_query_set diff --git a/lamindb/models/__init__.py b/lamindb/models/__init__.py index 69d26f83a..ec50d8964 100644 --- a/lamindb/models/__init__.py +++ b/lamindb/models/__init__.py @@ -239,3 +239,10 @@ ) FeatureValue = JsonValue # backward compatibility + +# register the cross-instance relation router as early as possible, so that +# `allow_relation` is active before any relational FK assignment (which Django +# checks at object construction, i.e. before a first cross-instance save) +from .sqlrecord import _ensure_lamindb_router + +_ensure_lamindb_router() diff --git a/lamindb/models/_feature_manager.py b/lamindb/models/_feature_manager.py index 5617518c4..b8f605447 100644 --- a/lamindb/models/_feature_manager.py +++ b/lamindb/models/_feature_manager.py @@ -2227,7 +2227,9 @@ def _add_from(self, data: Artifact | Collection, transfer_logs: dict = None): self._host.features._add_schema(schema_self, slot) -def bulk_set_features_in_records(records: Iterable[Record]) -> None: +def bulk_set_features_in_records( + records: Iterable[Record], using: str | None = None +) -> None: import numpy as np """Bulk-set lazy feature dictionaries for records. @@ -2247,6 +2249,11 @@ def bulk_set_features_in_records(records: Iterable[Record]) -> None: if len(records_with_features) == 0: return None + # for a cross-instance bulk save, validate and write on the target instance + # (records were bulk-created there, so their `_state.db` is the alias). Same + # convention as the per-record `add_values` path; None keeps the default. + instance = using if using not in (None, "default") else None + batch_schema: Schema | None = None batch_schema_index: Feature | None = None prepared_records: list[ @@ -2314,6 +2321,7 @@ def bulk_set_features_in_records(records: Iterable[Record]) -> None: ) data: dict[str, pd.Series] = {} + multivalued_columns: list[str] = [] for column in ordered_columns: # None from entirely-null columns is not a valid sentinel for extension # dtypes (StringDtype, BooleanDtype, Int64Dtype) — convert to pd.NA @@ -2322,22 +2330,37 @@ def bulk_set_features_in_records(records: Iterable[Record]) -> None: for v in (row.get(column, pd.NA) for row in prepared_rows) ] target_dtype = feature_dtype_by_name.get(column) - if target_dtype is not None: + # multi-valued features carry a list of values per record; a scalar + # categorical column cannot hold (unhashable) lists, so keep such columns + # as object dtype and flag them as `list_of_categories` (same as the + # per-record `convert_dict_to_dataframe_for_validation` path) so the + # curator validates list cells and `_collect_record_feature_writes` fans + # them out to per-value link rows. Single-valued columns keep the fast, + # declared-dtype Series path. + is_multivalued = any( + isinstance(v, (list, tuple, set, np.ndarray)) for v in values + ) + if target_dtype is not None and not is_multivalued: data[column] = pd.Series(values, dtype=target_dtype) else: - data[column] = pd.Series(values) + data[column] = pd.Series(values, dtype="object") + if is_multivalued: + multivalued_columns.append(column) if data: dataframe = pd.DataFrame(data) else: dataframe = pd.DataFrame(index=range(len(prepared_rows))) dataframe = move_schema_index_column_to_dataframe_index(dataframe, batch_schema) + # set after the index move, which does not preserve `.attrs` + for column in multivalued_columns: + dataframe.attrs[column] = "list_of_categories" # Single-pass dataframe curation: # validate schema and resolve categoricals once for the entire batch. # # The resolved label records are then reused below when creating per-record # link rows, avoiding repeated registry calls for each row. - curator = DataFrameCurator(dataframe, batch_schema) + curator = DataFrameCurator(dataframe, batch_schema, instance=instance) curator.validate() members_by_name: dict[str, list[Feature]] = defaultdict(list) @@ -2418,18 +2441,18 @@ def bulk_set_features_in_records(records: Iterable[Record]) -> None: ) FeatureManager._raise_not_validated_values(not_validated_values) if feature_json_values: - save(feature_json_values) + save(feature_json_values, using=instance) for links in links_by_model.values(): try: - save(links, ignore_conflicts=False) + save(links, ignore_conflicts=False, using=instance) except Exception: - save(links, ignore_conflicts=True) + save(links, ignore_conflicts=True, using=instance) from .save import bulk_update if batch_schema_index is not None: # only `name` was modified (via strip_index_for_record_persistence) # updating all fields generates a massive CASE WHEN SQL for large batches - bulk_update(records_with_features, update_fields=["name"]) + bulk_update(records_with_features, update_fields=["name"], using=instance) for record in records_with_features: del record._features return None diff --git a/lamindb/models/artifact_set.py b/lamindb/models/artifact_set.py index 0664f2932..af3d90c3c 100644 --- a/lamindb/models/artifact_set.py +++ b/lamindb/models/artifact_set.py @@ -225,7 +225,13 @@ def to_dataframe( ) # `type_id` points to record types (`is_type=True`) by model design. - record_type = Record.get(id=type_ids[0]) + # `type_ids` were read from `qs.db`; resolve the type on the same + # instance so cross-instance reads don't hit the default connection. + record_type = ( + Record.get(id=type_ids[0]) + if qs.db in (None, "default") + else Record.connect(qs.db).get(id=type_ids[0]) + ) qs._record_export_type = record_type logger.important(f"exporting {qs.count()} records of '{record_type.name}'") diff --git a/lamindb/models/feature.py b/lamindb/models/feature.py index 86861d909..a0fd9997a 100644 --- a/lamindb/models/feature.py +++ b/lamindb/models/feature.py @@ -640,13 +640,17 @@ def parse_filter_string(filter_str: str) -> dict[str, tuple[str, str | None, str def resolve_relation_filters( - parsed_filters: dict[str, tuple[str, str | None, str]], registry: SQLRecord + parsed_filters: dict[str, tuple[str, str | None, str]], + registry: SQLRecord, + using: str | None = None, ) -> dict[str, str | SQLRecord]: """Resolve relation filters actual model objects. Args: parsed_filters: Django filters like output from :func:`lamindb.models.feature.parse_filter_string` registry: Model class to resolve relationships against + using: Target instance to resolve related objects on (per-instance PKs); + ``None`` resolves against the default connection unchanged. Returns: Dict with resolved objects for successful relations, original values for direct fields and failed resolutions. @@ -661,7 +665,11 @@ def resolve_relation_filters( and relation_field.field.is_relation ): related_model = relation_field.field.related_model - related_obj = related_model.get(**{field_name: value}) + related_obj = ( + related_model.get(**{field_name: value}) + if using is None + else related_model.connect(using).get(**{field_name: value}) + ) resolved[relation_name] = related_obj else: resolved[filter_key] = value diff --git a/lamindb/models/save.py b/lamindb/models/save.py index f9a2bbf7b..911b72edd 100644 --- a/lamindb/models/save.py +++ b/lamindb/models/save.py @@ -126,7 +126,7 @@ def save( if records_with_lazy_features: from ._feature_manager import bulk_set_features_in_records - bulk_set_features_in_records(records_with_lazy_features) + bulk_set_features_in_records(records_with_lazy_features, using=using) if artifacts: for record in artifacts: diff --git a/lamindb/models/schema.py b/lamindb/models/schema.py index 76ff1d962..aac1af31f 100644 --- a/lamindb/models/schema.py +++ b/lamindb/models/schema.py @@ -96,13 +96,6 @@ def validate_features(features: list[SQLRecord]) -> SQLRecord: def _resolve_pks_on_instance( registry: type[SQLRecord], uids: list[str], using: str | None ) -> list[int]: - """Resolve primary keys on a target instance from stable `uid`s (order-preserving). - - Primary keys are per-instance, whereas `uid` is stable across instances. When - writing link-table rows to a different instance via `using=`, the in-memory - `.id` of a related record is not guaranteed to match that instance's row, so we - look up the ids by `uid` on the target connection. - """ uid_to_pk = dict( registry.objects.using(using).filter(uid__in=uids).values_list("uid", "id") ) @@ -1110,14 +1103,6 @@ def from_df( return cls.from_dataframe(df, field, name, mute, organism, source) def _infer_members_instance(self) -> str | None: - """Return the non-default instance the pending members live on, if any. - - Pending schema members (``_features``) and slot components (``_slots``) - carry ``_state.db``. When they were created on another instance via - ``using=``, a bare ``.save()`` should follow them there. Returns ``None`` - when members are local (the same-instance fast path) and raises when they - span more than one instance. - """ candidates: set[str] = set() if hasattr(self, "_features"): for record in self._features[1]: @@ -1164,11 +1149,6 @@ def save(self, *args, **kwargs) -> Schema: index_name_conflict = kwargs.pop("index_name_conflict", None) using = kwargs.get("using") or self._state.db - # a schema must live on the same instance as its members. When they were - # created on another instance via `using=` and the schema is saved without - # an explicit target, follow the members there (bare `.save()` should not - # split the schema from its features). Same-instance members are local, so - # this is a no-op for the normal path. members_instance = self._infer_members_instance() if members_instance is not None: if using is None or using == "default": @@ -1251,9 +1231,6 @@ def save(self, *args, **kwargs) -> Schema: index_name_conflict=index_name_conflict, ) super().save(*args, **kwargs) - # link-table rows reference per-instance primary keys; when writing to a - # different instance via `using=`, resolve both endpoints by their stable - # `uid` on the target connection. Same-instance saves keep the fast path. cross_instance = using is not None and using != "default" schema_id = ( _resolve_pks_on_instance(type(self), [self.uid], using)[0] diff --git a/lamindb/models/sqlrecord.py b/lamindb/models/sqlrecord.py index ef09cf7bf..5fc5d5fc9 100644 --- a/lamindb/models/sqlrecord.py +++ b/lamindb/models/sqlrecord.py @@ -2397,6 +2397,14 @@ def allow_relation(self, obj1, obj2, **hints) -> bool | None: def _ensure_lamindb_router() -> None: from django.conf import settings as django_settings + + # Django enforces `allow_relation` at FK assignment (object construction), + # which can happen before any cross-instance save. Registering the router only + # in `add_db_connection` would be too late, so this is also called at import + # time (see `lamindb/models/__init__.py`). It is a no-op until Django settings + # are configured (i.e. once an instance is connected). + if not django_settings.configured: + return from django.db import router as django_router router_path = "lamindb.models.sqlrecord.LaminDBRouter" diff --git a/tests/transfer/test_save_to_another_db.py b/tests/transfer/test_save_to_another_db.py index ebf7f2fd0..0205ccc65 100644 --- a/tests/transfer/test_save_to_another_db.py +++ b/tests/transfer/test_save_to_another_db.py @@ -1,6 +1,44 @@ +import subprocess +import sys + import lamindb as ln +def test_lamindb_router_registered_on_fresh_import(): + # Bug A: the cross-instance relation router must be active on a plain + # `import lamindb`, before any cross-instance save/connect in the process. + # A subprocess guarantees a cold router state regardless of test ordering. + code = ( + "import lamindb as ln\n" + "from django.conf import settings\n" + "print('LaminDBRouter' in str(settings.DATABASE_ROUTERS))\n" + ) + result = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True + ) + assert "True" in result.stdout, result.stdout + result.stderr + + +def test_construct_relation_to_remote_type_before_any_save(): + # Bug A (functional): assigning an FK to a record living on another instance + # must not raise the router error at construction time. + using = f"{ln.setup.settings.user.handle}/testdb1" + db2 = ln.DB(using) + type_name = "router-remote-type" + + if db2.Record.filter(name=type_name).exists(): + db2.Record.filter(name=type_name).delete(permanent=True) + try: + remote_type = ln.Record(name=type_name, is_type=True).save(using=using) + # read it back so the FK target is a record whose _state.db is the alias + remote_type = db2.Record.get(name=type_name) + # construction with a cross-instance FK must not raise + child = ln.Record(name="router-remote-child", type=remote_type) + assert child.type.uid == remote_type.uid + finally: + db2.Record.filter(name=type_name).delete(permanent=True) + + def test_save_ulabel_to_another_db_via_model_save(): assert ln.setup.settings.instance.name == "testdb2" @@ -173,3 +211,149 @@ def _clean(): assert ln.Feature.filter(name__in=feat_names).count() == 0 finally: _clean() + + +def test_save_record_with_single_valued_feature_to_another_db_via_model_save(): + # Bug B: single-valued categorical feature whose dtype-type lives on testdb1. + # The type lookup must resolve on the target instance, not the default. + assert ln.setup.settings.instance.name == "testdb2" + + using = f"{ln.setup.settings.user.handle}/testdb1" + db2 = ln.DB(using) + + gene_type_name = "sv-gene-type" + gene_feat_name = "sv-gene" + gene_name = "sv-gene-value" + rec_name = "sv-child" + + def _clean(): + for qs in ( + db2.Record.filter(name=rec_name), + db2.Record.filter(name=gene_name), + db2.Record.filter(name=gene_type_name), + db2.Feature.filter(name=gene_feat_name), + ): + if qs.exists(): + qs.delete(permanent=True) + + _clean() + try: + gene_t = ln.Record(name=gene_type_name, is_type=True).save(using=using) + gene_feat = ln.Feature(name=gene_feat_name, dtype=gene_t).save(using=using) + gene = ln.Record(name=gene_name, type=gene_t).save(using=using) + + child = ln.Record(name=rec_name, features={gene_feat: gene}).save(using=using) + + assert child._state.db == using + remote_child = db2.Record.get(name=rec_name) + assert remote_child.features.get_values()[gene_feat_name] == gene_name + assert ln.Record.filter(name=rec_name).count() == 0 + finally: + _clean() + + +def test_bulk_save_records_with_multivalued_features_to_another_db(): + # Bug C: `ln.save([...], using=)` funnels multi-valued features through + # `bulk_set_features_in_records`; a scalar categorical column cannot hold + # list values (pandas `unhashable type: 'list'`), and the writes must land + # on the target instance. + assert ln.setup.settings.instance.name == "testdb2" + + using = f"{ln.setup.settings.user.handle}/testdb1" + db2 = ln.DB(using) + + gene_type_name = "bulk-mv-gene-type" + int_type_name = "bulk-mv-int-type" + gene_feat_name = "bulk-mv-genes" + schema_name = "bulk-mv-schema" + gene_names = [f"bulk-mv-gene-{i}" for i in range(5)] + rec_names = [f"bulk-mv-child-{i}" for i in range(3)] + + def _clean(): + for qs in ( + db2.Record.filter(name__in=rec_names), + db2.Record.filter(name__in=gene_names), + db2.Record.filter(name=int_type_name), + db2.Schema.filter(name=schema_name), + db2.Feature.filter(name=gene_feat_name), + db2.Record.filter(name=gene_type_name), + ): + if qs.exists(): + qs.delete(permanent=True) + + _clean() + try: + gene_t = ln.Record(name=gene_type_name, is_type=True).save(using=using) + gene_feat = ln.Feature(name=gene_feat_name, dtype=gene_t).save(using=using) + schema = ln.Schema(name=schema_name, features=[gene_feat]).save(using=using) + int_t = ln.Record( + name=int_type_name, is_type=True, schema=schema + ).save(using=using) + gene_pool = [ + ln.Record(name=g, type=gene_t).save(using=using) for g in gene_names + ] + + records = [ + ln.Record(name=rn, type=int_t, features={gene_feat: gene_pool[:3]}) + for rn in rec_names + ] + ln.save(records, using=using) + + for rn in rec_names: + remote = db2.Record.get(name=rn) + assert set(remote.features.get_values()[gene_feat_name]) == set( + gene_names[:3] + ) + assert ln.Record.filter(name__in=rec_names).count() == 0 + finally: + _clean() + + +def test_read_features_from_another_db_via_to_dataframe(): + # read-path gap: `.to_dataframe(include="features")` on a remote queryset must + # resolve the record type on the queryset's instance, not the default. + assert ln.setup.settings.instance.name == "testdb2" + + using = f"{ln.setup.settings.user.handle}/testdb1" + db2 = ln.DB(using) + + gene_type_name = "rd-gene-type" + int_type_name = "rd-int-type" + gene_feat_name = "rd-genes" + schema_name = "rd-schema" + gene_names = [f"rd-gene-{i}" for i in range(4)] + rec_name = "rd-child" + + def _clean(): + for qs in ( + db2.Record.filter(name=rec_name), + db2.Record.filter(name__in=gene_names), + db2.Record.filter(name=int_type_name), + db2.Schema.filter(name=schema_name), + db2.Feature.filter(name=gene_feat_name), + db2.Record.filter(name=gene_type_name), + ): + if qs.exists(): + qs.delete(permanent=True) + + _clean() + try: + gene_t = ln.Record(name=gene_type_name, is_type=True).save(using=using) + gene_feat = ln.Feature(name=gene_feat_name, dtype=gene_t).save(using=using) + schema = ln.Schema(name=schema_name, features=[gene_feat]).save(using=using) + int_t = ln.Record( + name=int_type_name, is_type=True, schema=schema + ).save(using=using) + gene_pool = [ + ln.Record(name=g, type=gene_t).save(using=using) for g in gene_names + ] + ln.Record( + name=rec_name, type=int_t, features={gene_feat: gene_pool[:3]} + ).save(using=using) + + # cross-instance read with feature reassembly must not raise + df = db2.Record.filter(name=rec_name).to_dataframe(include="features") + assert len(df) == 1 + assert gene_feat_name in df.columns + finally: + _clean() From 665b55b5c537bb59f09ce5e27844458ec85e15a8 Mon Sep 17 00:00:00 2001 From: Raaghav-Pillai Date: Sat, 15 Aug 2026 01:12:17 +0530 Subject: [PATCH 14/14] Fixing run so it passes test cases --- lamindb/models/__init__.py | 3 --- lamindb/models/_feature_manager.py | 10 ------- lamindb/models/sqlrecord.py | 19 +++++++++----- tests/transfer/test_save_to_another_db.py | 32 +++++++++++++++-------- 4 files changed, 34 insertions(+), 30 deletions(-) diff --git a/lamindb/models/__init__.py b/lamindb/models/__init__.py index ec50d8964..c529e0d54 100644 --- a/lamindb/models/__init__.py +++ b/lamindb/models/__init__.py @@ -240,9 +240,6 @@ FeatureValue = JsonValue # backward compatibility -# register the cross-instance relation router as early as possible, so that -# `allow_relation` is active before any relational FK assignment (which Django -# checks at object construction, i.e. before a first cross-instance save) from .sqlrecord import _ensure_lamindb_router _ensure_lamindb_router() diff --git a/lamindb/models/_feature_manager.py b/lamindb/models/_feature_manager.py index b8f605447..b2bcc5773 100644 --- a/lamindb/models/_feature_manager.py +++ b/lamindb/models/_feature_manager.py @@ -2249,9 +2249,6 @@ def bulk_set_features_in_records( if len(records_with_features) == 0: return None - # for a cross-instance bulk save, validate and write on the target instance - # (records were bulk-created there, so their `_state.db` is the alias). Same - # convention as the per-record `add_values` path; None keeps the default. instance = using if using not in (None, "default") else None batch_schema: Schema | None = None @@ -2330,13 +2327,6 @@ def bulk_set_features_in_records( for v in (row.get(column, pd.NA) for row in prepared_rows) ] target_dtype = feature_dtype_by_name.get(column) - # multi-valued features carry a list of values per record; a scalar - # categorical column cannot hold (unhashable) lists, so keep such columns - # as object dtype and flag them as `list_of_categories` (same as the - # per-record `convert_dict_to_dataframe_for_validation` path) so the - # curator validates list cells and `_collect_record_feature_writes` fans - # them out to per-value link rows. Single-valued columns keep the fast, - # declared-dtype Series path. is_multivalued = any( isinstance(v, (list, tuple, set, np.ndarray)) for v in values ) diff --git a/lamindb/models/sqlrecord.py b/lamindb/models/sqlrecord.py index 5fc5d5fc9..73ca23931 100644 --- a/lamindb/models/sqlrecord.py +++ b/lamindb/models/sqlrecord.py @@ -2390,19 +2390,26 @@ def get_name_field( class LaminDBRouter: def allow_relation(self, obj1, obj2, **hints) -> bool | None: + # Permit assigning a relation when one side is not yet saved, e.g. + # `ln.Record(type=remote_type)` while building a record for a + # cross-instance `using=` save: the new record's `_state.db` may have + # been pinned to "default" by its default branch/space FKs before the + # cross-instance type FK is assigned, which Django's default + # (same-database) check would otherwise reject at construction time. + # + # For two *already-saved* records we defer to Django's default check + # (return None) so that genuinely cross-instance links, such as + # `artifact.records.add(label)` across two different instances, remain + # blocked with the usual friendly error. if isinstance(obj1, SQLRecord) and isinstance(obj2, SQLRecord): - return True + if obj1._state.adding or obj2._state.adding: + return True return None def _ensure_lamindb_router() -> None: from django.conf import settings as django_settings - # Django enforces `allow_relation` at FK assignment (object construction), - # which can happen before any cross-instance save. Registering the router only - # in `add_db_connection` would be too late, so this is also called at import - # time (see `lamindb/models/__init__.py`). It is a no-op until Django settings - # are configured (i.e. once an instance is connected). if not django_settings.configured: return from django.db import router as django_router diff --git a/tests/transfer/test_save_to_another_db.py b/tests/transfer/test_save_to_another_db.py index 0205ccc65..601e5755e 100644 --- a/tests/transfer/test_save_to_another_db.py +++ b/tests/transfer/test_save_to_another_db.py @@ -2,12 +2,30 @@ import sys import lamindb as ln +import pytest + + +def test_cross_instance_m2m_add_still_blocked(): + # the cross-instance relation router must NOT let `.add()` link two + # already-saved records that live on different instances; that guardrail + # (a friendly ValueError) must still fire. Mirrors the network-dependent + # tests/pydata test_unsaved_model_different_instance, but runs locally. + using = f"{ln.setup.settings.user.handle}/testdb1" + db2 = ln.DB(using) + af = db2.Artifact.get(key="README.md") # lives on testdb1 (see conftest) + assert af._state.db == using + + new_label = ln.Record(name="guardrail-testlabel").save() # on default testdb2 + try: + with pytest.raises(ValueError) as excinfo: + af.records.add(new_label) + assert "Cannot label a record from instance" in str(excinfo.value) + assert using in str(excinfo.value) + finally: + new_label.delete(permanent=True) def test_lamindb_router_registered_on_fresh_import(): - # Bug A: the cross-instance relation router must be active on a plain - # `import lamindb`, before any cross-instance save/connect in the process. - # A subprocess guarantees a cold router state regardless of test ordering. code = ( "import lamindb as ln\n" "from django.conf import settings\n" @@ -20,8 +38,6 @@ def test_lamindb_router_registered_on_fresh_import(): def test_construct_relation_to_remote_type_before_any_save(): - # Bug A (functional): assigning an FK to a record living on another instance - # must not raise the router error at construction time. using = f"{ln.setup.settings.user.handle}/testdb1" db2 = ln.DB(using) type_name = "router-remote-type" @@ -214,8 +230,6 @@ def _clean(): def test_save_record_with_single_valued_feature_to_another_db_via_model_save(): - # Bug B: single-valued categorical feature whose dtype-type lives on testdb1. - # The type lookup must resolve on the target instance, not the default. assert ln.setup.settings.instance.name == "testdb2" using = f"{ln.setup.settings.user.handle}/testdb1" @@ -253,10 +267,6 @@ def _clean(): def test_bulk_save_records_with_multivalued_features_to_another_db(): - # Bug C: `ln.save([...], using=)` funnels multi-valued features through - # `bulk_set_features_in_records`; a scalar categorical column cannot hold - # list values (pandas `unhashable type: 'list'`), and the writes must land - # on the target instance. assert ln.setup.settings.instance.name == "testdb2" using = f"{ln.setup.settings.user.handle}/testdb1"