From 423e7e19e6af256c6b3453e3f6d59a94ee5a51cb Mon Sep 17 00:00:00 2001 From: debugactiveprocess <49375302+debugactiveprocess@users.noreply.github.com> Date: Thu, 28 May 2026 23:20:58 -0300 Subject: [PATCH 1/2] perf: share compiled RegexSet cache across scanners --- lib/src/compiler/mod.rs | 1 + lib/src/compiler/rules.rs | 31 ++++++++++++++++++++++++++++++- lib/src/scanner/context.rs | 18 ++---------------- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/lib/src/compiler/mod.rs b/lib/src/compiler/mod.rs index 9f49a9c06..2e7c737f6 100644 --- a/lib/src/compiler/mod.rs +++ b/lib/src/compiler/mod.rs @@ -814,6 +814,7 @@ impl<'a> Compiler<'a> { warnings: self.warnings.into(), filesize_bounds: self.filesize_bounds, regex_sets: self.regex_sets, + compiled_regex_sets: Default::default(), fast_scan_patterns: self.fast_scan_patterns, }; diff --git a/lib/src/compiler/rules.rs b/lib/src/compiler/rules.rs index c532169a3..076db01f2 100644 --- a/lib/src/compiler/rules.rs +++ b/lib/src/compiler/rules.rs @@ -2,6 +2,7 @@ use std::fmt; use std::io::{BufWriter, Read, Write}; use std::ops::{Bound, RangeBounds}; use std::slice::Iter; +use std::sync::OnceLock; #[cfg(feature = "logging")] use std::time::Instant; @@ -165,6 +166,16 @@ pub struct Rules { /// set automata for single-pass evaluation. pub(in crate::compiler) regex_sets: FxHashMap>, + /// Lazily compiled `RegexSet` automata shared by all scanners using these + /// rules. + /// + /// The definitions in `regex_sets` are serialized with the rules, but the + /// compiled automata are runtime-only. Sharing this cache at the `Rules` + /// level avoids rebuilding the same `RegexSet` once per scanner/thread. + #[serde(skip)] + pub(in crate::compiler) compiled_regex_sets: + OnceLock>>, + /// BitVec where the N-th bit indicates whether the pattern with /// PatternId = N is a fast-scan pattern. /// @@ -375,7 +386,25 @@ impl Rules { pub(crate) fn get_regex_set( &self, set_id: RegexSetId, - ) -> regex::bytes::RegexSet { + ) -> ®ex::bytes::RegexSet { + let compiled_regex_sets = self.compiled_regex_sets.get_or_init(|| { + let mut regex_sets = FxHashMap::default(); + regex_sets.reserve(self.regex_sets.len()); + for &set_id in self.regex_sets.keys() { + regex_sets.insert(set_id, OnceLock::new()); + } + regex_sets + }); + + let regex_set = compiled_regex_sets + .get(&set_id) + .unwrap_or_else(|| panic!("invalid RegexSetId: {set_id:?}")); + + regex_set.get_or_init(|| self.build_regex_set(set_id)) + } + + /// Builds the `RegexSet` automaton for a given [`RegexSetId`]. + fn build_regex_set(&self, set_id: RegexSetId) -> regex::bytes::RegexSet { let re_ids = self.regex_sets.get(&set_id).unwrap(); let mut patterns = Vec::with_capacity(re_ids.len()); diff --git a/lib/src/scanner/context.rs b/lib/src/scanner/context.rs index 7e3786502..5ed29fc7c 100644 --- a/lib/src/scanner/context.rs +++ b/lib/src/scanner/context.rs @@ -24,7 +24,7 @@ use rustc_hash::{FxHashMap, FxHashSet}; use smallvec::SmallVec; use crate::compiler::{ - NamespaceId, PatternId, RegexId, RegexSetId, RuleId, Rules, SubPattern, + NamespaceId, PatternId, RegexId, RuleId, Rules, SubPattern, SubPatternAtom, SubPatternFlags, SubPatternId, }; use crate::errors::VariableError; @@ -153,15 +153,6 @@ pub struct ScanContext<'r, 'd> { /// is evaluated, it is compiled the first time and stored in this hash /// map. pub(crate) regex_cache: RefCell>, - /// Persistent cache for grouped `RegexSet` automata. - /// - /// Like individual regular expressions, compiling a multi-pattern - /// `RegexSet` is an expensive operation. This cache compiles the set - /// automata lazily upon the very first match request and preserves the - /// compiled automata across all subsequent scans performed by the - /// scanner instance. - pub(crate) regex_set_cache: - RefCell>, /// Engines for custom base64 alphabets used by base64 string modifiers. /// /// These engines are derived from rule literals and reused across scans. @@ -306,11 +297,7 @@ impl ScanContext<'_, '_> { set_id: crate::compiler::RegexSetId, haystack: &[u8], ) -> bool { - let mut automata_cache = self.regex_set_cache.borrow_mut(); - let regex_set = automata_cache - .entry(set_id) - .or_insert_with(|| self.compiled_rules.get_regex_set(set_id)); - regex_set.is_match(haystack) + self.compiled_rules.get_regex_set(set_id).is_match(haystack) } /// Returns the protobuf struct produced by a module. @@ -1919,7 +1906,6 @@ pub fn create_wasm_store_and_ctx<'r>( }, deadline: 0, regex_cache: RefCell::new(FxHashMap::default()), - regex_set_cache: RefCell::new(FxHashMap::default()), vm: VM { pike_vm: PikeVM::new(rules.re_code()), fast_vm: FastVM::new(rules.re_code()), From 9d60139c74dcfa98f7b20835cb3756dd9a665193 Mon Sep 17 00:00:00 2001 From: debugactiveprocess <49375302+debugactiveprocess@users.noreply.github.com> Date: Wed, 3 Jun 2026 09:10:12 -0300 Subject: [PATCH 2/2] refactor: simplify RegexSet cache initialization --- lib/src/compiler/mod.rs | 1 + lib/src/compiler/rules.rs | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/src/compiler/mod.rs b/lib/src/compiler/mod.rs index 2e7c737f6..a14014307 100644 --- a/lib/src/compiler/mod.rs +++ b/lib/src/compiler/mod.rs @@ -818,6 +818,7 @@ impl<'a> Compiler<'a> { fast_scan_patterns: self.fast_scan_patterns, }; + rules.init_regex_set_cache(); rules.build_ac_automaton(); rules } diff --git a/lib/src/compiler/rules.rs b/lib/src/compiler/rules.rs index 076db01f2..3f8f807f1 100644 --- a/lib/src/compiler/rules.rs +++ b/lib/src/compiler/rules.rs @@ -170,11 +170,12 @@ pub struct Rules { /// rules. /// /// The definitions in `regex_sets` are serialized with the rules, but the - /// compiled automata are runtime-only. Sharing this cache at the `Rules` - /// level avoids rebuilding the same `RegexSet` once per scanner/thread. + /// compiled automata are runtime-only. The vector is initialized when the + /// [`Rules`] are built or deserialized, while each individual `RegexSet` + /// is compiled lazily on first use. #[serde(skip)] pub(in crate::compiler) compiled_regex_sets: - OnceLock>>, + Vec>, /// BitVec where the N-th bit indicates whether the pattern with /// PatternId = N is a fast-scan pattern. @@ -279,6 +280,7 @@ impl Rules { info!("WASM build time: {:?}", Instant::elapsed(&start)); } + rules.init_regex_set_cache(); rules.build_ac_automaton(); Ok(rules) @@ -381,23 +383,21 @@ impl Rules { }) } + /// Initializes the runtime-only cache for compiled `RegexSet` automata. + pub(in crate::compiler) fn init_regex_set_cache(&mut self) { + self.compiled_regex_sets = + (0..self.regex_sets.len()).map(|_| OnceLock::new()).collect(); + } + /// Returns a compiled multi-pattern `RegexSet` for a given `RegexSetId`. #[inline] pub(crate) fn get_regex_set( &self, set_id: RegexSetId, ) -> ®ex::bytes::RegexSet { - let compiled_regex_sets = self.compiled_regex_sets.get_or_init(|| { - let mut regex_sets = FxHashMap::default(); - regex_sets.reserve(self.regex_sets.len()); - for &set_id in self.regex_sets.keys() { - regex_sets.insert(set_id, OnceLock::new()); - } - regex_sets - }); - - let regex_set = compiled_regex_sets - .get(&set_id) + let regex_set = self + .compiled_regex_sets + .get(usize::from(set_id)) .unwrap_or_else(|| panic!("invalid RegexSetId: {set_id:?}")); regex_set.get_or_init(|| self.build_regex_set(set_id))