Reliable release

.github/workflows/release.yml

@@ -0,0 +1,120 @@
+name: Release
+
+# Every push to main builds and validates the distributions, so a packaging
+# mistake surfaces at merge time rather than at release time. Publishing to PyPI
+# happens when a GitHub Release is published: PyPI versions are immutable and
+# unique, so a release has to be tied to a version tag rather than to every merge.
+on:
+  push:
+    branches: ["main"]
+  release:
+    types: [published]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build and check distributions
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # setuptools_scm derives the version from the tags, it needs all of them
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build tooling
+        run: python -m pip install --upgrade build twine
+
+      - name: Build sdist and wheel
+        run: python -m build
+
+      - name: Check distribution metadata
+        run: python -m twine check dist/*
+
+      - name: Check the precomputed response norms are packaged
+        # they are loaded from inside the installed package, so a wheel without
+        # them silently falls back to simulating naturalistic stimuli responses
+        run: |
+          python - <<'PY'
+          import glob, sys, tarfile, zipfile
+
+          expected = "flyvis/data/responses_norm.h5"
+          wheel = glob.glob("dist/*.whl")[0]
+          sdist = glob.glob("dist/*.tar.gz")[0]
+
+          in_wheel = expected in zipfile.ZipFile(wheel).namelist()
+          in_sdist = any(
+              name.split("/", 1)[-1] == expected for name in tarfile.open(sdist).getnames()
+          )
+          print(f"{expected} in wheel: {in_wheel}, in sdist: {in_sdist}")
+          if not (in_wheel and in_sdist):
+              sys.exit(f"{expected} is missing from the distributions")
+          PY
+
+      - name: Check the version matches the release tag
+        if: github.event_name == 'release'
+        run: |
+          python - <<'PY'
+          import glob, os, sys
+
+          tag = os.environ["TAG"].lstrip("v")
+          built = glob.glob("dist/*.whl")[0].split("/")[-1].split("-")[1]
+          print(f"tag: {tag}, built version: {built}")
+          if built != tag:
+              sys.exit(
+                  f"built version {built} does not match release tag {tag}; "
+                  "the release must point at an annotated version tag"
+              )
+          PY
+        env:
+          TAG: ${{ github.event.release.tag_name }}
+
+      - name: Smoke test the wheel
+        run: |
+          python -m venv /tmp/smoke
+          /tmp/smoke/bin/pip install --upgrade pip
+          /tmp/smoke/bin/pip install dist/*.whl
+          /tmp/smoke/bin/python - <<'PY'
+          from flyvis.analysis import response_norms
+
+          norms = response_norms.read_response_norms(
+              response_norms.PRECOMPUTED_FILE, "flow/0000"
+          )
+          assert norms is not None, f"{response_norms.PRECOMPUTED_FILE} is not installed"
+          assert len(norms.model_names) == 50, norms
+          assert len(norms.checkpoint_hashes) == 50, norms
+          print("installed package ships constants for", norms)
+          PY
+          /tmp/smoke/bin/flyvis --help > /dev/null
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: distributions
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    if: github.event_name == 'release'
+    runs-on: ubuntu-latest
+    # Configure this environment name in the PyPI trusted publisher, and add
+    # required reviewers here if releases should be approved by a human.
+    environment:
+      name: pypi
+      url: https://pypi.org/p/flyvis
+    permissions:
+      # required for trusted publishing, no API token needs to be stored
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: distributions
+          path: dist/
+
+      - uses: pypa/gh-action-pypi-publish@release/v1

CHANGELOG.md

@@ -22,6 +22,17 @@
 - Added `examples/figure_04_top_models.py`, which reproduces figure 4a,b for the
   models with the lowest task error instead of the task-optimal cluster.
 
+### Infrastructure
+- Added a `Release` workflow that builds and checks the distributions on every push
+  to `main` and publishes them to PyPI via trusted publishing when a GitHub Release
+  is published. It verifies that the version matches the release tag and that the
+  precomputed response norms are present in both the wheel and the source
+  distribution before uploading.
+- Made the test suite deterministic. Five tests asserted on unseeded randomness and
+  failed on roughly one run in seven between them. The global generators are now
+  seeded before every test, `FLYVIS_TEST_SEED` re-runs the suite under a different
+  seed, and the assertions that were only true for most draws were corrected.
+
 ## [v1.1.3] - 2026-03-07
 
 ### Bug Fixes

docs/docs/release.md

@@ -1,5 +1,50 @@
 # Release Process
 
+Releases are published to PyPI by the
+[`Release` workflow](https://github.com/TuragaLab/flyvis/blob/main/.github/workflows/release.yml).
+The manual steps further down are the fallback for when the workflow cannot be used.
+
+## Automated release
+
+Every push to `main` builds the source distribution and the wheel and checks them,
+so a packaging mistake --- a data file that stopped being included, for instance ---
+surfaces at merge time. Nothing is uploaded on a merge: PyPI versions are immutable
+and can never be reused, so publishing is tied to a version tag rather than to every
+commit that lands on `main`.
+
+To cut a release:
+
+1. Merge to `main` and let the tests and the build check pass.
+2. Update `CHANGELOG.md` with the new version.
+3. Tag and push:
+   ```bash
+   git tag -a v1.1.4 -F CHANGELOG.md
+   git push origin main v1.1.4
+   ```
+4. Publish a GitHub Release pointing at that tag. That triggers the upload.
+
+The workflow refuses to publish if the version derived by `setuptools_scm` does not
+match the release tag, if the distributions fail `twine check`, or if the
+precomputed response norms are missing from them. It then installs the built wheel
+in a clean virtualenv and verifies that the constants can be read from the installed
+package before uploading.
+
+### One-time setup
+
+Uploads use [PyPI trusted publishing](https://docs.pypi.org/trusted-publishers/), so
+no API token is stored in the repository. On PyPI, under the `flyvis` project's
+*Publishing* settings, add a GitHub publisher with:
+
+| Field | Value |
+| --- | --- |
+| Owner | `TuragaLab` |
+| Repository | `flyvis` |
+| Workflow name | `release.yml` |
+| Environment name | `pypi` |
+
+Adding required reviewers to the `pypi` environment in the repository settings makes
+every upload wait for a human approval.
+
 ## Prerequisites
 
 1. Ensure all tests pass:
@@ -16,7 +61,9 @@ The deployment to github can be done last or via workflow.
 python -m pip install build twine
 ```
 
-## Release Steps
+## Manual release steps
+
+These are the fallback for when the automated release cannot be used.
 
 0. **Test PyPi before committing (optional)**
 

tests/conftest.py

@@ -1,13 +1,41 @@
+import os
 import tempfile
 from pathlib import Path
 
 import numpy as np
 import pytest
+import torch
 from PIL import Image
 
 from flyvis import connectome_file
 from flyvis.connectome import ConnectomeFromAvgFilters
 
+SEED = int(os.environ.get("FLYVIS_TEST_SEED", 0))
+"""Seed used to make the test suite deterministic.
+
+Override with `FLYVIS_TEST_SEED` to re-run the suite under a different seed. An
+assertion that only holds for the default seed is a broken assertion, so sweeping
+the seed is how to find one.
+"""
+
+
+@pytest.fixture(autouse=True)
+def deterministic_rng():
+    """Reset the global random number generators before every test.
+
+    Augmentations draw from the global numpy stream, so without this a test's
+    outcome depends on how many draws the tests before it happened to consume, and
+    changes from run to run. Tests whose assertions only hold for most draws then
+    fail sporadically, which in CI is indistinguishable from a real regression.
+
+    Note:
+        This makes a test reproducible, it does not make an assertion that only
+        holds for most draws correct. Assert on what the code guarantees, or set
+        the random parameters explicitly, rather than relying on a lucky seed.
+    """
+    np.random.seed(SEED)
+    torch.manual_seed(SEED)
+
 
 @pytest.fixture(scope="session")
 def connectome(tmp_path_factory):
@@ -19,7 +47,10 @@ def connectome(tmp_path_factory):
 
 @pytest.fixture(scope="session")
 def sequence_path(tmp_path_factory):
-    sequences = np.random.rand(20, 10, 64, 64)
+    # session fixtures are built before the per-test seeding above runs, so they
+    # seed their own generator to keep the mock data identical across runs
+    rng = np.random.default_rng(SEED)
+    sequences = rng.random((20, 10, 64, 64))
     sequences = np.transpose(sequences, (1, 0, 2, 3)) / 255.0
     path = tmp_path_factory.mktemp("tmp") / "sequences.npy"
     np.save(path, sequences)
@@ -29,6 +60,7 @@ def sequence_path(tmp_path_factory):
 @pytest.fixture(scope="session")
 def mock_sintel_data():
     """Create a minimal mock Sintel dataset structure with original dimensions."""
+    rng = np.random.default_rng(SEED)
     with tempfile.TemporaryDirectory() as tmp_dir:
         tmp_path = Path(tmp_dir)
 
@@ -44,7 +76,7 @@ def mock_sintel_data():
             # Create dummy files with original dimensions
             for i in range(5):
                 # Luminance (final) - (436, 1024)
-                img = (np.random.uniform(0, 1, (HEIGHT, WIDTH)) * 255).astype(np.uint8)
+                img = (rng.uniform(0, 1, (HEIGHT, WIDTH)) * 255).astype(np.uint8)
                 Image.fromarray(img).save(
                     tmp_path / f"training/final/{seq_name}/frame_{i:04d}.png"
                 )
@@ -57,7 +89,7 @@ def mock_sintel_data():
                     np.array([202021.25], dtype=np.float32).tofile(f)  # Magic number
                     np.array([WIDTH, HEIGHT], dtype=np.int32).tofile(f)  # Dimensions
                     # Write flow data
-                    np.random.randn(HEIGHT, WIDTH, 2).astype(np.float32).tofile(f)
+                    rng.standard_normal((HEIGHT, WIDTH, 2)).astype(np.float32).tofile(f)
 
                 # Depth - (436, 1024)
                 with open(
@@ -66,6 +98,6 @@ def mock_sintel_data():
                     # Write header
                     np.array([1, WIDTH, HEIGHT], dtype=np.int32).tofile(f)  # Dimensions
                     # Write depth data
-                    np.random.randn(HEIGHT, WIDTH).astype(np.float32).tofile(f)
+                    rng.standard_normal((HEIGHT, WIDTH)).astype(np.float32).tofile(f)
 
         yield tmp_path

tests/test_augmentation.py

@@ -170,10 +170,13 @@ def _is_monotonous(sequence):
     assert cropped2.min() in sequence and cropped2.max() in sequence
     assert np.allclose(cropped1, cropped2)
 
-    # random start frame
+    # random start frame. A sampled start of 0 would legitimately reproduce
+    # cropped1, so assert that the sampled start is what was applied rather than
+    # that the crop differs.
     random_crop.set_or_sample(start=None, total_sequence_length=len(sequence))
     cropped3 = random_crop(sequence).cpu().numpy().flatten()
-    assert not np.allclose(cropped1, cropped3)
+    assert len(cropped3) == 10 and _is_monotonous(cropped3)
+    assert np.allclose(cropped3, cropped1 + random_crop.start)
 
     # provided start frame
     random_crop.set_or_sample(start=42, total_sequence_length=len(sequence))

tests/test_ensemble.py

@@ -63,8 +63,11 @@ def test_validation_losses(ensemble):
 def test_rank_by_validation_error(ensemble):
     sorted_names = deepcopy(ensemble.names)
 
-    # destroy current task sorting in place
-    np.random.shuffle(ensemble.names)
+    # Destroy the current task sorting. A random shuffle reproduces the sorted
+    # order once in len(ensemble)! draws -- 1 in 24 here -- which would make the
+    # `!= random_names` assertion below fail. Rotating is a derangement for any
+    # ensemble with more than one model, so it always destroys the sorting.
+    ensemble.names = sorted_names[1:] + sorted_names[:1]
 
     random_names = deepcopy(ensemble.names)
 

tests/test_rendering.py

@@ -31,13 +31,19 @@ def test_call(boxeye: rendering.BoxEye):
 
     sequence = torch.ones((2, 2, 100, 100)).random_(0, 11)
 
+    # the median of uniform integers on [0, 10] is 5 in expectation, but this is a
+    # finite sample: the mean of the filtered frames has a standard deviation of
+    # about 0.02, so the tolerance is ~7 sigma. A tighter one fails for a few
+    # percent of inputs, and the draw differs between CPU and GPU even when seeded.
+    atol = 0.15
+
     rendered = boxeye(sequence, ftype="median", hex_sample=True)
     assert rendered.shape == (2, 2, 1, boxeye.hexals)
-    assert np.isclose(rendered.cpu().numpy().mean(), 5, atol=0.05)
+    assert np.isclose(rendered.cpu().numpy().mean(), 5, atol=atol)
 
     rendered = boxeye(sequence.clone(), ftype="median", hex_sample=False)
     assert rendered.shape == (*sequence.shape[:2], *boxeye.min_frame_size.cpu().numpy())
-    assert np.isclose(rendered.cpu().numpy().mean(), 5, atol=0.05)
+    assert np.isclose(rendered.cpu().numpy().mean(), 5, atol=atol)
 
     with pytest.raises(ValueError):
         boxeye(sequence, ftype="invalid", hex_sample=True)

tests/test_sintel.py

@@ -199,10 +199,17 @@ def test_getitem(dataset):
     dataset.augment = True
     data1 = dataset[0]
     assert set(data1.keys()) == set(["lum", "flow"])
-    assert (data0["lum"] != data1["lum"]).any()
-    assert (data0["flow"] != data1["flow"]).any()
     assert data1["lum"].shape == (3, 1, 7)
     assert data1["flow"].shape == (3, 2, 7)
+    # the input is always changed, it is jittered and noised
+    assert (data0["lum"] != data1["lum"]).any()
+    # targets are only rotated, flipped and temporally cropped -- they are neither
+    # jittered nor noised -- so a sampled augmentation leaves them untouched
+    # whenever it draws neither a rotation nor a flip, which happens with
+    # probability (1 - p_rot) * (1 - p_flip) = 1/12. Ask for a rotation explicitly
+    # rather than asserting on the draw.
+    rotated = dataset.apply_augmentation(dataset.cached_sequences[0], n_rot=1, flip_axis=0)
+    assert (data0["flow"] != rotated["flow"]).any()
 
     # change dt to 1/50
     dataset.dt = 1 / 50
@@ -224,8 +231,13 @@ def test_apply_augmentation(dataset):
     data = dataset[0]
     data1 = dataset.apply_augmentation(data)
     assert set(data1.keys()) == set(data.keys())
+    # the input is always changed, it is jittered and noised
     assert (data["lum"] != data1["lum"]).any()
-    assert (data["flow"] != data1["flow"]).any()
+    # targets are only rotated, flipped and temporally cropped, so a sampled
+    # augmentation leaves them untouched whenever it draws neither a rotation nor
+    # a flip. Request a rotation explicitly rather than asserting on the draw.
+    rotated = dataset.apply_augmentation(data, n_rot=1, flip_axis=0)
+    assert (data["flow"] != rotated["flow"]).any()
 
 
 def test_original_sequence_index(dataset):