From 7d4e416dd51eb97aacaf56c8a8c2f0b7f6cd29ce Mon Sep 17 00:00:00 2001 From: Daniel McCarney Date: Fri, 17 Jul 2026 11:40:12 -0400 Subject: [PATCH] revocation: support for V1 index format The "upkiidx1" magic bytes identify a new V1 index.bin format that is almost identical to the existing "upkiidx0" (V0) format with two main exceptions: 1. The header `num_filenames` count is now a u16 instead of u8 2. The per-entry `filter_index` is now a u16 instead of a u8 These changes are necessary in order to facilitate having more than 255 filter files in the revocation cache. We believe the combination of the webpki decreasing certificate maximum validity periods, and a potential increase in the frequency of filter file production could require more headroom than a maximum of 255 would allow. To accommodate these changes when writing a new index.bin upki will always write the V1 format & data-types. When _reading_ an existing index.bin, both formats are supported and use the correct data type based on the header magic bytes. In the future this can be simplified to only support V1 on the read-path once all extent data is updated to the new format. --- upki/src/revocation/index.rs | 229 +++++++++++++++++++++++++++++------ 1 file changed, 192 insertions(+), 37 deletions(-) diff --git a/upki/src/revocation/index.rs b/upki/src/revocation/index.rs index 92cf38e7..18ef7f98 100644 --- a/upki/src/revocation/index.rs +++ b/upki/src/revocation/index.rs @@ -28,9 +28,9 @@ use crate::Config; /// All integers are big-endian. Filenames are fixed 32-byte slots, NULL-padded. /// /// ```text -/// HEADER (first read, 13 bytes): -/// magic: [u8; 8] "upkiidx0" -/// num_filenames: u8 +/// HEADER (first reads, 14 bytes): +/// magic: [u8; 8] "upkiidx1" +/// num_filenames: u16 /// num_log_ids: u32 /// /// TABLES (second read): @@ -43,17 +43,23 @@ use crate::Config; /// /// ENTRY SECTIONS (seek + third read): /// Per entry: -/// filter_index: u8 +/// filter_index: u16 /// min_timestamp: u64 /// max_timestamp: u64 /// ``` /// +/// Indexes with the legacy "upkiidx0" magic encode `num_filenames` and +/// `filter_index` as `u8`. Both versions are read, but writing always produces +/// "upkiidx1". +/// /// This type stores a [`File`] for on-demand reading of entry sections. pub struct Index { cache_dir: PathBuf, num_filenames: usize, num_logs: usize, logs_offset: usize, + /// Size of one entry section entry, determined by the index magic version. + entry_size: usize, /// Contains the filename table followed by the log table. tables: Vec, file: File, @@ -72,17 +78,31 @@ impl Index { path: Some(index_path.clone()), })?; - // Read 1: magic + num_filenames + num_log_ids - let mut header = [0u8; HEADER_SIZE]; - file.read_exact(&mut header) + // Read 1: magic, determining the version-dependent header and entry sizes + let mut magic = [0u8; 8]; + file.read_exact(&mut magic) .map_err(|e| Error::IndexDecode(Box::new(e)))?; - if header[..8] != *INDEX_MAGIC { + let (header_size, entry_size) = if magic == *INDEX_MAGIC_V1 { + (HEADER_SIZE_V1, ENTRY_SIZE_V1) + } else if magic == *INDEX_MAGIC_V0 { + (HEADER_SIZE_V0, ENTRY_SIZE_V0) + } else { return Err(Error::IndexDecode("invalid index magic".into())); - } - let num_filenames = header[8] as usize; - let num_logs = u32::from_be_bytes([header[9], header[10], header[11], header[12]]) as usize; + }; - // Read 2: filename table + log table + // Read 2: num_filenames + num_log_ids + let mut header = [0u8; HEADER_SIZE_V1 - 8]; + let header = &mut header[..header_size - 8]; + file.read_exact(header) + .map_err(|e| Error::IndexDecode(Box::new(e)))?; + let mut data = &*header; + let num_filenames = match entry_size { + ENTRY_SIZE_V0 => u8::read_be(&mut data)? as usize, + _ => u16::read_be(&mut data)? as usize, + }; + let num_logs = u32::read_be(&mut data)? as usize; + + // Read 3: filename table + log table let logs_offset = num_filenames * FILENAME_SIZE; let tables_len = logs_offset + num_logs * LOG_DIR_ENTRY_SIZE; @@ -95,7 +115,7 @@ impl Index { path: Some(index_path), })? .len(); - if (HEADER_SIZE + tables_len) as u64 > file_len { + if (header_size + tables_len) as u64 > file_len { return Err(Error::IndexDecode("index tables truncated".into())); } @@ -108,6 +128,7 @@ impl Index { num_filenames, num_logs, logs_offset, + entry_size, tables, file, }) @@ -118,7 +139,7 @@ impl Index { /// Returns `None` if any filter file cannot be read or decoded. #[cfg(feature = "__fetch")] pub(super) fn write(manifest: &Manifest, dir: &Path) -> Option> { - let mut by_log_id: BTreeMap> = BTreeMap::new(); + let mut by_log_id: BTreeMap> = BTreeMap::new(); for (filter_idx, filter) in manifest.files.iter().enumerate() { if filter.filename.len() > FILENAME_SIZE { @@ -150,19 +171,19 @@ impl Index { by_log_id .entry(*log_id) .or_default() - .push((filter_idx as u8, *interval)); + .push((filter_idx as u16, *interval)); } } // Compute header size to determine entry section offsets - let header_size = HEADER_SIZE + let header_size = HEADER_SIZE_V1 + manifest.files.len() * FILENAME_SIZE + by_log_id.len() * LOG_DIR_ENTRY_SIZE; // Write header let mut buf = Vec::new(); - buf.extend_from_slice(INDEX_MAGIC); - buf.push(manifest.files.len() as u8); + buf.extend_from_slice(INDEX_MAGIC_V1); + buf.extend_from_slice(&(manifest.files.len() as u16).to_be_bytes()); buf.extend_from_slice(&(by_log_id.len() as u32).to_be_bytes()); // Write filename table (fixed 32-byte slots, NULL-padded) @@ -178,7 +199,7 @@ impl Index { let mut dir_entries: Vec<(&LogId, u64, u16)> = Vec::new(); for (log_id, entries) in &by_log_id { dir_entries.push((log_id, current_offset as u64, entries.len() as u16)); - current_offset += entries.len() * ENTRY_SIZE; + current_offset += entries.len() * ENTRY_SIZE_V1; } // Write log_id directory @@ -191,7 +212,7 @@ impl Index { // Write entry sections for entries in by_log_id.values() { for (filter_idx, interval) in entries { - buf.push(*filter_idx); + buf.extend_from_slice(&filter_idx.to_be_bytes()); buf.extend_from_slice(&interval.low.0.to_be_bytes()); buf.extend_from_slice(&interval.high.0.to_be_bytes()); } @@ -210,7 +231,7 @@ impl Index { let key = input.key(); let dir_data = &self.tables[self.logs_offset..]; let mut maybe_good = false; - let mut seen = [false; 256]; + let mut seen = vec![false; self.num_filenames]; 'timestamp: for sct in &input.sct_timestamps { // Binary search the sorted log_id directory (stride LOG_DIR_ENTRY_SIZE) @@ -239,20 +260,27 @@ impl Index { .seek(SeekFrom::Start(offset)) .map_err(|e| Error::IndexDecode(Box::new(e)))?; - let mut buf = vec![0u8; count as usize * ENTRY_SIZE]; + let mut buf = vec![0u8; count as usize * self.entry_size]; self.file .read_exact(&mut buf) .map_err(|e| Error::IndexDecode(Box::new(e)))?; let mut data = &buf[..]; for _ in 0..count { - let filter_index = u8::read_be(&mut data)? as usize; + let filter_index = match self.entry_size { + ENTRY_SIZE_V0 => u8::read_be(&mut data)? as usize, + _ => u16::read_be(&mut data)? as usize, + }; let min_ts = u64::read_be(&mut data)?; let max_ts = u64::read_be(&mut data)?; if min_ts > sct.timestamp || sct.timestamp > max_ts { continue; } + // Errors on `filter_index >= num_filenames`, so the `seen` + // indexing below is in range. + let filename = self.filename(filter_index)?; + // A filter is queried with every SCT timestamp, so consulting it // again for a later SCT cannot produce a different answer. if seen[filter_index] { @@ -260,7 +288,6 @@ impl Index { } seen[filter_index] = true; - let filename = self.filename(filter_index)?; let path = self.cache_dir.join(filename); let bytes = match fs::read(&path) { Ok(bytes) => bytes, @@ -324,6 +351,7 @@ impl fmt::Debug for Index { num_filenames, num_logs, logs_offset, + entry_size: _, tables: _, file: _, } = self; @@ -343,6 +371,12 @@ impl FromBeBytes<8> for u64 { } } +impl FromBeBytes<4> for u32 { + fn from_be_bytes(bytes: &[u8; 4]) -> Self { + Self::from_be_bytes(*bytes) + } +} + impl FromBeBytes<2> for u16 { fn from_be_bytes(bytes: &[u8; 2]) -> Self { Self::from_be_bytes(*bytes) @@ -369,13 +403,16 @@ trait FromBeBytes: Sized { fn from_be_bytes(bytes: &[u8; N]) -> Self; } -const HEADER_SIZE: usize = 8 + 1 + 4; +const HEADER_SIZE_V1: usize = 8 + 2 + 4; +const HEADER_SIZE_V0: usize = 8 + 1 + 4; const FILENAME_SIZE: usize = 32; const LOG_DIR_ENTRY_SIZE: usize = 32 + 8 + 2; -const ENTRY_SIZE: usize = 1 + 8 + 8; +const ENTRY_SIZE_V1: usize = 2 + 8 + 8; +const ENTRY_SIZE_V0: usize = 1 + 8 + 8; pub(super) const INDEX_BIN: &str = "index.bin"; -const INDEX_MAGIC: &[u8; 8] = b"upkiidx0"; +const INDEX_MAGIC_V1: &[u8; 8] = b"upkiidx1"; +const INDEX_MAGIC_V0: &[u8; 8] = b"upkiidx0"; #[cfg(test)] mod tests { @@ -450,7 +487,7 @@ mod tests { fn truncated_after_magic() { let dir = tempfile::tempdir().unwrap(); let config = test_config(dir.path()); - write_file(dir.path(), INDEX_BIN, INDEX_MAGIC); + write_file(dir.path(), INDEX_BIN, INDEX_MAGIC_V1); let err = Index::from_cache(&config).unwrap_err(); assert!(matches!(err, Error::IndexDecode(_))); } @@ -470,8 +507,8 @@ mod tests { fn oversized_table_counts() { let dir = tempfile::tempdir().unwrap(); let config = test_config(dir.path()); - let mut data = INDEX_MAGIC.to_vec(); - data.push(u8::MAX); + let mut data = INDEX_MAGIC_V1.to_vec(); + data.extend_from_slice(&u16::MAX.to_be_bytes()); data.extend_from_slice(&u32::MAX.to_be_bytes()); write_file(dir.path(), INDEX_BIN, &data); let err = Index::from_cache(&config).unwrap_err(); @@ -908,27 +945,142 @@ mod tests { Ok(()) } + // Checking a legacy "upkiidx0" index for a revoked result. + #[test] + fn check_v0_index_revoked() -> Result<(), Box> { + let dir = tempfile::tempdir()?; + let config = test_config(dir.path()); + + let (log_a, log_b) = ([0xb1; 32], [0xb2; 32]); + // f0 covers log_a but enrolls a different issuer -> NotEnrolled for us. + let f0 = build_filter([0xcc; 32], &[&[7, 7]], &[(log_a, 0, 2000)]); + // f1 covers log_b and revokes our serial. + let f1 = build_filter([0xaa; 32], &[&[1, 2, 3]], &[(log_b, 0, 2000)]); + write_file(dir.path(), "f0.filter", &f0); + write_file(dir.path(), "f1.filter", &f1); + write_file( + dir.path(), + INDEX_BIN, + &build_index_v0(&[ + ("f0.filter", &[(log_a, 0, 2000)]), + ("f1.filter", &[(log_b, 0, 2000)]), + ]), + ); + + assert_eq!( + Index::from_cache(&config)?.check(&multi_sct_input(&[(log_a, 1000), (log_b, 1000)]))?, + RevocationStatus::CertainlyRevoked, + ); + + Ok(()) + } + + // Checking a legacy "upkiidx0" index for a not revoked result. + #[test] + fn check_v0_index_not_revoked() -> Result<(), Box> { + let dir = tempfile::tempdir()?; + let config = test_config(dir.path()); + + let filter = build_filter([0xaa; 32], &[&[9, 9, 9]], &[([0xbb; 32], 0, 2000)]); + write_file(dir.path(), "f0.filter", &filter); + write_file( + dir.path(), + INDEX_BIN, + &build_index_v0(&[("f0.filter", &[([0xbb; 32], 0, 2000)])]), + ); + + assert_eq!( + Index::from_cache(&config)?.check(&test_input())?, + RevocationStatus::NotRevoked, + ); + + Ok(()) + } + + // Checking a legacy "upkiidx0" index that's empty shoud produce not covered. + #[test] + fn check_empty_v0_index() { + let dir = tempfile::tempdir().unwrap(); + let config = test_config(dir.path()); + write_file(dir.path(), INDEX_BIN, &build_index_v0(&[])); + assert_eq!( + Index::from_cache(&config) + .unwrap() + .check(&test_input()) + .unwrap(), + RevocationStatus::NotCoveredByRevocationData, + ); + } + + // Checking a malformed filter that has a filter index beyond the filename table must err. + #[test] + fn check_filter_index_out_of_bounds() { + let dir = tempfile::tempdir().unwrap(); + let config = test_config(dir.path()); + + let mut data = build_index(&[("f0.filter", &[([0xbb; 32], 0, 2000)])]); + // Overwrite the sole entry's filter_index (first two bytes of the entry + // section, which build_index places at the end of the tables). + let entry_offset = data.len() - ENTRY_SIZE_V1; + data[entry_offset..entry_offset + 2].copy_from_slice(&500u16.to_be_bytes()); + write_file(dir.path(), INDEX_BIN, &data); + + let err = Index::from_cache(&config) + .unwrap() + .check(&test_input()) + .unwrap_err(); + assert!(matches!(err, Error::IndexDecode(_))); + } + + /// Build a test index with INDEX_MAGIC_V1 with u16 filter indexes. #[expect(clippy::type_complexity)] fn build_index(filters: &[(&str, &[([u8; 32], u64, u64)])]) -> Vec { + build_index_with_magic(INDEX_MAGIC_V1, filters) + } + + /// Build a test legacy INDEX_MAGIC_V0 index with u8 filter indexes. + #[expect(clippy::type_complexity)] + fn build_index_v0(filters: &[(&str, &[([u8; 32], u64, u64)])]) -> Vec { + build_index_with_magic(INDEX_MAGIC_V0, filters) + } + + /// Build a test index with the given magic header bytes. + /// + /// The data types used for `num_filenames` and the entry `filter_index` are + /// determined by the magic bytes, using u8 for INDEX_MAGIC_V0, and u16 + /// otherwise. + #[expect(clippy::type_complexity)] + fn build_index_with_magic( + magic: &[u8; 8], + filters: &[(&str, &[([u8; 32], u64, u64)])], + ) -> Vec { + let (base_size, entry_size) = match magic { + INDEX_MAGIC_V0 => (HEADER_SIZE_V0, ENTRY_SIZE_V0), + _ => (HEADER_SIZE_V1, ENTRY_SIZE_V1), + }; + // Aggregate by log_id using BTreeMap for sorted order - let mut by_log_id: BTreeMap<[u8; 32], Vec<(u8, u64, u64)>> = BTreeMap::new(); + let mut by_log_id: BTreeMap<[u8; 32], Vec<(u16, u64, u64)>> = BTreeMap::new(); for (filter_idx, (_, entries)) in filters.iter().enumerate() { for (log_id, min_ts, max_ts) in *entries { by_log_id .entry(*log_id) .or_default() - .push((filter_idx as u8, *min_ts, *max_ts)); + .push((filter_idx as u16, *min_ts, *max_ts)); } } // Compute header size let header_size = - HEADER_SIZE + filters.len() * FILENAME_SIZE + by_log_id.len() * LOG_DIR_ENTRY_SIZE; + base_size + filters.len() * FILENAME_SIZE + by_log_id.len() * LOG_DIR_ENTRY_SIZE; // Write header let mut buf = Vec::new(); - buf.extend_from_slice(INDEX_MAGIC); - buf.push(filters.len() as u8); + buf.extend_from_slice(magic); + match entry_size { + ENTRY_SIZE_V0 => buf.push(filters.len() as u8), + _ => buf.extend_from_slice(&(filters.len() as u16).to_be_bytes()), + } buf.extend_from_slice(&(by_log_id.len() as u32).to_be_bytes()); // Write filename table (fixed 32-byte slots, NULL-padded) @@ -946,14 +1098,17 @@ mod tests { buf.extend_from_slice(log_id); buf.extend_from_slice(&(current_offset as u64).to_be_bytes()); buf.extend_from_slice(&(entries.len() as u16).to_be_bytes()); - current_offset += entries.len() * ENTRY_SIZE; + current_offset += entries.len() * entry_size; entry_counts.push(entries.len()); } // Write entry sections for entries in by_log_id.values() { for (filter_idx, min_ts, max_ts) in entries { - buf.push(*filter_idx); + match entry_size { + ENTRY_SIZE_V0 => buf.push(*filter_idx as u8), + _ => buf.extend_from_slice(&filter_idx.to_be_bytes()), + } buf.extend_from_slice(&min_ts.to_be_bytes()); buf.extend_from_slice(&max_ts.to_be_bytes()); }