diff --git a/lamindb_setup/core/upath.py b/lamindb_setup/core/upath.py index 4d00c2326..bb24473c9 100644 --- a/lamindb_setup/core/upath.py +++ b/lamindb_setup/core/upath.py @@ -531,8 +531,8 @@ def synchronize_to( # no need to cast local_stat.st_mtime to int # because if it has the fractional part and cloud_mtime doesn't # and they have the same integer part then cloud_mtime can't be bigger - is_sync_needed = ( - lambda cloud_mtime, local_stat: cloud_mtime > local_stat.st_mtime + is_sync_needed = lambda cloud_mtime, local_stat: ( + cloud_mtime > local_stat.st_mtime ) local_paths: list[Path] = [] @@ -808,27 +808,56 @@ def view_tree( logger.print(message) -def to_url(upath: S3Path) -> str: - """Public storage URL. +def to_url(upath: UPath) -> str: + """Generates a URL for an object represented by `UPath`. - Generates a public URL for an object in an S3 bucket using fsspec's UPath, - considering the bucket's region. + For S3/GCS paths, this returns a public URL considering the bucket region. + If the S3 path is not publicly hosted, it returns a LaminHub URL if the artifact is hosted on LaminHub. Args: - upath: A `UPath` object representing an S3 path. + upath: A `UPath` object. Returns: - A string containing the public URL to the S3 object. + A string containing the URL to the object. """ - if upath.protocol != "s3": - raise ValueError("The provided UPath must be an S3 path.") - key = "/".join(upath.parts[1:]) - bucket = upath.drive - region = get_storage_region(upath) - if region == "us-east-1": - return f"https://{bucket}.s3.amazonaws.com/{key}" - else: - return f"https://{bucket}.s3-{region}.amazonaws.com/{key}" + from ._settings import settings + + if upath.protocol == "s3": + key = "/".join(upath.parts[1:]) + bucket = upath.drive + if _is_publicly_accessible_path(upath): + region = get_storage_region(upath) + if region == "us-east-1": + return f"https://{bucket}.s3.amazonaws.com/{key}" + return f"https://{bucket}.s3-{region}.amazonaws.com/{key}" + elif settings.instance.is_on_hub: + origin = settings.instance.ui_url + if origin is not None: + common = f"{origin}/storage/s3/{bucket}%2F/{key}" + return common + else: + raise ValueError( + "The provided S3 UPath must be publicly accessible or the artifact must be hosted on LaminHub." + ) + if upath.protocol == "gs": + if _is_publicly_accessible_path(upath): + return f"https://storage.googleapis.com/{str(upath).removeprefix('gs://')}" + else: + raise ValueError( + "This function only supports publicly accessible GCS paths." + ) + if upath.protocol in {"http", "https"}: + return str(upath) + raise ValueError("The provided UPath must be an S3, GCS, HTTP, or HTTPS path.") + + +def _is_publicly_accessible_path(upath: UPath) -> bool: + """Check whether an S3/GCS path is anonymously readable.""" + anon_path = UPath(upath, anon=True) + try: + return anon_path.exists() + except Exception: + return False def from_auth(cls, path: AnyPathStr) -> UPath: diff --git a/tests/core/test_to_url.py b/tests/core/test_to_url.py index 53072329a..2c7ed091d 100644 --- a/tests/core/test_to_url.py +++ b/tests/core/test_to_url.py @@ -1,32 +1,115 @@ from __future__ import annotations +from types import SimpleNamespace + import lamindb_setup as ln_setup +import pytest +from lamindb_setup.core._settings import settings + + +def test_to_url_s3_public_us_east_1(monkeypatch): + monkeypatch.setattr( + ln_setup.core.upath, "_is_publicly_accessible_path", lambda _: True + ) + monkeypatch.setattr( + ln_setup.core.upath, "get_storage_region", lambda _: "us-east-1" + ) + upath = ln_setup.core.upath.UPath("s3://lamindata/test-folder") + assert upath.to_url() == "https://lamindata.s3.amazonaws.com/test-folder" -def test_to_url(): - # us-east-1 / AWS Dev - # public bucket +def test_to_url_s3_public_regional(monkeypatch): + monkeypatch.setattr( + ln_setup.core.upath, "_is_publicly_accessible_path", lambda _: True + ) + monkeypatch.setattr( + ln_setup.core.upath, "get_storage_region", lambda _: "eu-central-1" + ) + upath = ln_setup.core.upath.UPath("s3://lamindata-eu/test-folder") assert ( - ln_setup.core.upath.create_path("s3://lamindata/test-folder").to_url() - == "https://lamindata.s3.amazonaws.com/test-folder" + upath.to_url() + == "https://lamindata-eu.s3-eu-central-1.amazonaws.com/test-folder" + ) + + +def test_to_url_gcs_root(monkeypatch): + monkeypatch.setattr( + ln_setup.core.upath, "_is_publicly_accessible_path", lambda _: True + ) + upath = ln_setup.core.upath.UPath( + "gs://rxrx1-europe-west4/images/test/HEPG2-08/Plate1/B02_s1_w1.png" ) - # private bucket assert ( - ln_setup.core.upath.create_path( - "s3://lamindb-setup-private-bucket/test-folder" - ).to_url() - == "https://lamindb-setup-private-bucket.s3.amazonaws.com/test-folder" + upath.to_url() + == "https://storage.googleapis.com/rxrx1-europe-west4/images/test/HEPG2-08/Plate1/B02_s1_w1.png" + ) + + +def test_to_url_https_root(): + upath = ln_setup.core.upath.UPath("https://example.com/files/document.txt") + assert upath.to_url() == "https://example.com/files/document.txt" + + +def test_to_url_s3_hub_private_route(monkeypatch): + monkeypatch.setattr( + ln_setup.core.upath, "_is_publicly_accessible_path", lambda _: False + ) + monkeypatch.setattr( + settings, + "_instance_settings", + SimpleNamespace(is_on_hub=True, ui_url="https://app.lamin.ai"), + raising=False, ) - # eu-central-1 / AWS Dev + upath = ln_setup.core.upath.UPath("s3://lamindb-ci/test-data/test.parquet") assert ( - ln_setup.core.upath.create_path("s3://lamindata-eu/test-folder").to_url() - == "https://lamindata-eu.s3-eu-central-1.amazonaws.com/test-folder" + upath.to_url() + == "https://app.lamin.ai/storage/s3/lamindb-ci%2F/test-data/test.parquet" + ) + + +def test_to_url_s3_public_stays_native(monkeypatch): + monkeypatch.setattr( + ln_setup.core.upath, "_is_publicly_accessible_path", lambda _: True + ) + monkeypatch.setattr( + ln_setup.core.upath, "get_storage_region", lambda _: "us-east-1" + ) + monkeypatch.setattr( + settings, + "_instance_settings", + SimpleNamespace(is_on_hub=True, ui_url="https://app.lamin.ai"), + raising=False, ) - # eu-central-1 / AWS Hosted - # below is the default storage of the lamin-dev instance + upath = ln_setup.core.upath.UPath("s3://lamindb-ci/test-data/test.parquet") assert ( - ln_setup.core.upath.create_path( - "s3://lamin-eu-central-1/9fm7UN13/test-folder" - ).to_url() - == "https://lamin-eu-central-1.s3-eu-central-1.amazonaws.com/9fm7UN13/test-folder" + upath.to_url() == "https://lamindb-ci.s3.amazonaws.com/test-data/test.parquet" + ) + + +def test_to_url_s3_private_not_hub_raises(monkeypatch): + monkeypatch.setattr( + ln_setup.core.upath, "_is_publicly_accessible_path", lambda _: False + ) + monkeypatch.setattr( + settings, + "_instance_settings", + SimpleNamespace(is_on_hub=False, ui_url=None), + raising=False, + ) + upath = ln_setup.core.upath.UPath("s3://private-bucket/secret/file.csv") + with pytest.raises( + ValueError, + match="must be publicly accessible or the artifact must be hosted on LaminHub", + ): + upath.to_url() + + +def test_to_url_gcs_private_raises(monkeypatch): + monkeypatch.setattr( + ln_setup.core.upath, "_is_publicly_accessible_path", lambda _: False + ) + upath = ln_setup.core.upath.UPath( + "gs://rxrx1-europe-west4/images/test/HEPG2-08/Plate1/B02_s1_w1.png" ) + with pytest.raises(ValueError, match="only supports publicly accessible GCS paths"): + upath.to_url()