Add a test verifying only one downstairs required to activate and read from a RO volume.

integration_tests/src/lib.rs

@@ -487,6 +487,17 @@ mod integration_tests {
             downstairs.clone_region(source).await
         }
 
+        // Stop this downstairs, freeing the port it was listening on.  Used
+        // to simulate a downstairs that is not running.  The address that was
+        // assigned during spawn is still recorded in any CrucibleOpts we
+        // handed out, so the upstairs will try (and fail) to connect to it.
+        pub async fn stop(&mut self) -> Result<()> {
+            if let Some(downstairs) = self.downstairs.take() {
+                downstairs.stop().await?;
+            }
+            Ok(())
+        }
+
         pub fn address(&self) -> SocketAddr {
             // If start_downstairs returned Ok, then address will be populated
             self.downstairs.as_ref().unwrap().address()
@@ -1160,6 +1171,61 @@ mod integration_tests {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn integration_test_just_read_one_downstairs() -> Result<()> {
+        // Create three read-only downstairs, but only leave one of them
+        // running.  A single read-only downstairs is enough to activate a
+        // read-only upstairs, so both activation and a read should succeed.
+        const BLOCK_SIZE: usize = 512;
+
+        // small(true) creates and starts all three downstairs read only.  We
+        // capture the opts (which record all three addresses) before stopping
+        // two of them, leaving only downstairs1 running.
+        let mut tds = DefaultTestDownstairsSet::small(true).await?;
+        let opts = tds.opts();
+
+        tds.downstairs2.stop().await?;
+        tds.downstairs3.stop().await?;
+
+        // Put the region under sub_volumes (not as a read_only_parent) so that
+        // flushes are actually sent to it.
+        let vcr = VolumeConstructionRequest::Volume {
+            id: Uuid::new_v4(),
+            block_size: BLOCK_SIZE as u64,
+            sub_volumes: vec![VolumeConstructionRequest::Region {
+                block_size: BLOCK_SIZE as u64,
+                blocks_per_extent: tds.blocks_per_extent(),
+                extent_count: tds.extent_count(),
+                opts,
+                generation: 1,
+            }],
+            read_only_parent: None,
+        };
+
+        let volume = Volume::construct(vcr, None, csl()).await?;
+        volume.activate().await?;
+
+        // Read one block: should be all 0x00
+        let mut buffer = Buffer::new(1, BLOCK_SIZE);
+        volume.read(BlockIndex(0), &mut buffer).await?;
+
+        assert_eq!(vec![0x00; BLOCK_SIZE], &buffer[..]);
+
+        // Manually send a flush.  With only one downstairs running, the flush
+        // still completes: the two stopped downstairs have their jobs moved to
+        // Skipped, so the flush is complete on all clients and acks back to us.
+        // This should not hang.
+        volume.flush(None).await?;
+
+        // A second read after the flush should also complete successfully.
+        let mut buffer = Buffer::new(1, BLOCK_SIZE);
+        volume.read(BlockIndex(0), &mut buffer).await?;
+
+        assert_eq!(vec![0x00; BLOCK_SIZE], &buffer[..]);
+
+        Ok(())
+    }
+
     #[tokio::test]
     async fn integration_test_volume_write_unwritten_1() -> Result<()> {
         // Test a simple single layer volume, verify write_unwritten