diff --git a/Cargo.lock b/Cargo.lock index 5a62b9f7..e47fd1c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -889,6 +889,7 @@ dependencies = [ "serde_urlencoded", "sha2 0.11.0", "sqlx", + "structstruck", "subtle", "textris-pdf", "tokio", @@ -3037,6 +3038,17 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "structstruck" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f318209842d04696a2c139cd3d5d4d75c4a9e2af435b077aa5cbd391a79c34b3" +dependencies = [ + "proc-macro2", + "quote", + "venial", +] + [[package]] name = "subsetter" version = "0.2.6" @@ -3550,6 +3562,16 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "venial" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61584a325b16f97b5b25fcc852eb9550843a251057a5e3e5992d2376f3df4bb2" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index 81ec0da7..afad65db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,7 @@ aes-gcm = "0.11.0" # 118M downloads (RustCrypto) hkdf = "0.13.0" # 198M downloads (RustCrypto) postcard = { version = "1.1.3", default-features = false, features = ["alloc"] } # 42M downloads (James Munns) secrecy = "0.10.3" # 126M downloads (Tony Arcieri) +structstruck = "0.5.1" # 212K downloads (Julius Michaelis) urlencoding = "2.1.3" # 213M downloads (Kornel) zeroize = "1.9.0" # 359M downloads (RustCrypto) @@ -154,6 +155,7 @@ sqlx = { workspace = true, optional = true } subtle = { workspace = true } textris-pdf = { workspace = true } tokio = { workspace = true } +structstruck = { workspace = true } tokio-util = { workspace = true } tower-http = { workspace = true } tracing = { workspace = true } diff --git a/compose.yml b/compose.yml index 6090b250..8643682e 100644 --- a/compose.yml +++ b/compose.yml @@ -9,3 +9,12 @@ services: TZ: Europe/Amsterdam ports: ['127.0.0.1:5432:5432'] networks: [default] + + personen-mock: + container_name: brp-personen-mock + image: ghcr.io/brp-api/personen-mock:2.7.0-latest + environment: + - ASPNETCORE_ENVIRONMENT=Release + - ASPNETCORE_URLS=http://+:5010 + ports: + - "5010:5010" diff --git a/locales/en/audit_log.yml b/locales/en/audit_log.yml index 46a27554..11b3969b 100644 --- a/locales/en/audit_log.yml +++ b/locales/en/audit_log.yml @@ -40,6 +40,7 @@ documents: empty: No events have been recorded yet. event: add_candidate_to_list: Added candidate to list + brp_validation: Validated candidate against the BRP create_candidate_list: Created list of candidates create_name_authorisation: Created statutory name and authorised agent create_empty: Created empty political group @@ -60,6 +61,7 @@ event: import: Imported political group import_csv: Imported CSV remove_candidate_from_list: Removed candidate from list + set_brp_validation_state: Set BRP validation state set_finished: Set finished state update_candidate_list_districts: Updated electoral districts of list update_candidate_list_order: Updated order of list of candidates diff --git a/locales/nl/audit_log.yml b/locales/nl/audit_log.yml index 7d364213..6f73bf08 100644 --- a/locales/nl/audit_log.yml +++ b/locales/nl/audit_log.yml @@ -40,6 +40,7 @@ documents: empty: Er zijn nog geen gebeurtenissen vastgelegd. event: add_candidate_to_list: Kandidaat aan lijst toegevoegd + brp_validation: Kandidaat gevalideerd tegen de BRP create_candidate_list: Kandidatenlijst aangemaakt create_name_authorisation: Statutaire naam en gemachtigde aangemaakt create_empty: Lege politieke groepering aangemaakt @@ -60,6 +61,7 @@ event: import: Politieke groepering geïmporteerd import_csv: CSV geïmporteerd remove_candidate_from_list: Kandidaat van lijst verwijderd + set_brp_validation_state: BRP validatie status gezet set_finished: Afgerond gezet update_candidate_list_districts: Kieskringen van lijst bijgewerkt update_candidate_list_order: Volgorde van kandidatenlijst bijgewerkt diff --git a/src/core/config.rs b/src/core/config.rs index 90a2bec1..33c6cf8b 100644 --- a/src/core/config.rs +++ b/src/core/config.rs @@ -1,11 +1,14 @@ //! Loads runtime configuration from environment variables for AppState. //! Used by AppState::new to construct service URLs and storage settings. -use std::{env, path::PathBuf}; +use std::{env, path::PathBuf, time::Duration}; use secrecy::SecretString; -use crate::AppError; +use crate::{ + AppError, + structs::brp::{BRP_PERSONS_ENDPOINT, BRP_TIMEOUT}, +}; #[cfg(feature = "dev-features")] mod dev_defaults { @@ -20,11 +23,16 @@ mod dev_defaults { pub(super) const DEFAULT_MASTER_ENCRYPTION_KEY: &str = "eks-dev-master-encryption-key-not-for-production"; + pub(super) const BRP_API_KEY: &str = ""; + pub(super) const BRP_BASE_URL: &str = "http://localhost:5010"; + pub(super) fn lookup(name: &'static str) -> Result { std::collections::HashMap::from([ ("STORAGE_URL", STORAGE_URL), ("ID_DERIVATION_KEY", ID_DERIVATION_KEY), ("MASTER_ENCRYPTION_KEY", DEFAULT_MASTER_ENCRYPTION_KEY), + ("BRP_BASE_URL", BRP_BASE_URL), + ("BRP_API_KEY", BRP_API_KEY), ]) .get(name) .map(|value| (*value).to_string()) @@ -39,6 +47,15 @@ pub struct TlsConfig { pub key_path: PathBuf, } +/// TLS configuration for serving HTTPS via rustls. +#[derive(Debug, Clone)] +pub struct BrpConfig { + pub base_url: String, + pub api_key: String, + pub persons_endpoint: String, + pub timeout: Duration, +} + /// Runtime configuration loaded from environment variables. #[derive(Debug, Clone)] pub struct Config { @@ -61,6 +78,7 @@ pub struct Config { /// container. Set via `DISABLE_AUTH_SERVICE` (`1`, `true`, or `yes`, /// case-insensitive); anything else leaves the auth-service enabled. pub disable_auth_service: bool, + pub brp_client: BrpConfig, } fn get_env_with(name: &'static str, lookup: &mut F) -> Result @@ -117,6 +135,24 @@ impl Config { ) }); + let base_url = get_env_with("BRP_BASE_URL", &mut lookup)?; + let api_key = get_env_with("BRP_API_KEY", &mut lookup)?; + + let timeout: u64 = lookup("BRP_TIMEOUT") + .unwrap_or(BRP_TIMEOUT.to_string()) + .parse() + .map_err(|_| { + AppError::ConfigLoadError("Invalid BRP_TIMEOUT; please enter a number".to_string()) + })?; + + let brp_client = BrpConfig { + base_url, + api_key, + persons_endpoint: lookup("BRP_PERSONS_ENDPOINT") + .unwrap_or(BRP_PERSONS_ENDPOINT.to_string()), + timeout: Duration::from_secs(timeout), + }; + Ok(Self { storage_url: SecretString::from(storage_url), id_derivation_key: SecretString::from(id_derivation_key), @@ -125,6 +161,7 @@ impl Config { server_name, eks_key, disable_auth_service, + brp_client, }) } @@ -138,6 +175,12 @@ impl Config { server_name: None, eks_key: None, disable_auth_service: false, + brp_client: BrpConfig { + base_url: "http://localhost:5010".to_string(), + api_key: "".to_string(), + persons_endpoint: BRP_PERSONS_ENDPOINT.to_string(), + timeout: Duration::from_secs(BRP_TIMEOUT), + }, } } } diff --git a/src/csb/import/pages/import.rs b/src/csb/import/pages/import.rs index 14115224..23511327 100644 --- a/src/csb/import/pages/import.rs +++ b/src/csb/import/pages/import.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + use askama::Template; use axum::{ extract::State, @@ -6,15 +8,25 @@ use axum::{ use serde::Deserialize; use crate::{ - AppError, AppState, Context, CsbContext, CsbEvent, Form, HtmlTemplate, Locale, PgStoreData, - StreamId, - csb::examination::{CsbExaminationOverviewPath, CsbPoliticalGroupPath}, - filters, redirect_success, trans, + AppError, AppState, Context, CsbContext, CsbEvent, CsbStoreData, Form, HtmlTemplate, Locale, + PgStoreData, StreamId, + csb::{ + WithCorrections, + examination::{CsbExaminationOverviewPath, CsbPoliticalGroupPath}, + }, + filters, + persons::Person, + redirect_success, + store::Store, + structs::brp::{BrpClient, BrpStatus}, + trans, utils::parse_hash_prefix, }; use super::{CsbCreateEmptyPath, CsbImportPath}; +const BRP_COURTESY_TIMEOUT: Duration = Duration::from_secs(1); + #[derive(Template)] #[template(path = "csb/import/pages/import.html")] struct CsbImportTemplate { @@ -121,6 +133,8 @@ async fn do_import( }) .await?; + do_brp_verification(&csb_store, &state.brp_client).await?; + Ok(redirect_success(CsbPoliticalGroupPath { stream_id: csb_store.stream_id, })) @@ -141,13 +155,114 @@ pub async fn create_empty( })) } +/// Verifies every candidate against the BRP in a background task. Returns +/// immediately after spawning it instead of waiting for it to finish. +pub async fn do_brp_verification( + store: &Store, + brp_client: &BrpClient, +) -> Result<(), AppError> { + store + .update(CsbEvent::SetBrpStatus(BrpStatus::InProgress)) + .await?; + + // Spawned and intentionally not awaited here: verifying every candidate is + // slow, and this must not block the request that triggered it. + tokio::task::spawn(monitor_verification(store.clone(), brp_client.clone())); + + Ok(()) +} + +async fn monitor_verification(store: Store, brp_client: BrpClient) { + let outcome = tokio::task::spawn(verify_candidates(store.clone(), brp_client)).await; + + let error = match outcome { + Ok(Ok(())) => return, + Ok(Err(err)) => err.to_string(), + Err(join_err) => join_err.to_string(), + }; + + if let Err(err) = store + .update(CsbEvent::SetBrpStatus(BrpStatus::Aborted(error))) + .await + { + tracing::error!("failed to record aborted BRP status: {err}"); + } +} + +/// Check every not-yet-validated candidate on `store` against the BRP. +/// Candidates already present in `CsbStoreData::brp_validations` are skipped, +/// so a later call resumes instead of re-checking everyone. A single +/// candidate's failure is logged and does not stop the rest of the sweep; only +/// a failure to record the final status is propagated to the caller. +async fn verify_candidates( + store: Store, + brp_client: BrpClient, +) -> Result<(), AppError> { + let already_validated = store.get_brp_validations(); + let mut ticker = tokio::time::interval(BRP_COURTESY_TIMEOUT); + + for person in store.get_persons(WithCorrections::None) { + if already_validated.contains_key(&person.id) { + tracing::debug!("Person {} has already been validated", person.id); + continue; + } + + tracing::info!("Checking person {} against the brp", person.id); + ticker.tick().await; + + if let Err(err) = verify_candidate(&store, &brp_client, &person).await { + tracing::error!("BRP verification failed for {}: {err}", person.id); + } + } + + tracing::info!( + "Finished checking candidates on list {:?}", + store.data.read().imported_data.political_group + ); + + store + .update(CsbEvent::SetBrpStatus(BrpStatus::Finished)) + .await +} + +/// Verify this candidate against the BRP, creating omissions that contain the +/// lists this candidate is on. Finally, the store is updated with a +/// `BrpPersonValidated` event. +async fn verify_candidate( + store: &Store, + brp_client: &BrpClient, + person: &Person, +) -> Result<(), AppError> { + let candidate_lists = store + .get_candidate_lists(WithCorrections::None) + .iter() + .filter(|cl| cl.candidates.contains(&person.id)) + .map(|cl| cl.id) + .collect(); + + let omissions = brp_client.verify(person, candidate_lists).await?; + let valid = omissions.is_empty(); + + for omission in omissions { + omission.create(store).await?; + } + + store + .update(CsbEvent::BrpPersonValidated { + person: person.id, + valid, + }) + .await +} + #[cfg(test)] mod tests { use super::*; use axum::http::StatusCode; use crate::{ - AppState, CsbContext, ElectionConfig, PgEvent, test_utils::response_body_string, + AppState, CsbContext, ElectionConfig, PgEvent, + test_utils::{response_body_string, sample_person_from_brp}, utils::format_hash, }; @@ -164,6 +279,18 @@ mod tests { Ok((source_stream, format_hash(&hash, false))) } + /// Poll briefly for the background BRP check to finish, instead of + /// sleeping for the full courtesy timeout between candidates. + async fn wait_for_brp_status_finished(store: &Store) { + for _ in 0..100 { + if matches!(store.data.read().brp_validation_status, BrpStatus::Finished) { + return; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + panic!("BRP verification did not finish in time"); + } + #[tokio::test] async fn import_renders_placeholder_page() -> Result<(), AppError> { let response = import(CsbImportPath {}, CsbContext::new_test()) @@ -315,4 +442,90 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn brp_verification_records_an_omission_for_a_mismatched_person() -> Result<(), AppError> + { + let state = AppState::new_for_tests().await; + let csb_store = state + .csb_store_for_stream(StreamId::new(), ElectionConfig::EK27) + .await?; + + // This person matches a record in the local BRP mock exactly, except for + // the tampered field below, so the check should find one mismatch. + let mut person = sample_person_from_brp(); + person.address.house_number_addition = Some("nope".parse().unwrap()); + csb_store.add_person(person); + + let brp_client = BrpClient::new_for_test(); + do_brp_verification(&csb_store, &brp_client).await?; + + wait_for_brp_status_finished(&csb_store).await; + let omission = csb_store.get_omission_for_test(); + assert_eq!( + omission.description, + "De huisnummertoevoeging komt niet overeen met de BRP" + ); + + Ok(()) + } + + #[tokio::test] + async fn do_brp_verification_returns_without_waiting_for_the_brp_check() -> Result<(), AppError> + { + let state = AppState::new_for_tests().await; + let csb_store = state + .csb_store_for_stream(StreamId::new(), ElectionConfig::EK27) + .await?; + + // Two candidates requires one BRP_COURTESY_TIMEOUT (1s) tick. + // do_brp_verification should return well before that. + csb_store.add_person(sample_person_from_brp()); + csb_store.add_person(sample_person_from_brp()); + + let brp_client = BrpClient::new_for_test(); + + let start = tokio::time::Instant::now(); + do_brp_verification(&csb_store, &brp_client).await?; + let elapsed = start.elapsed(); + assert!( + elapsed < BRP_COURTESY_TIMEOUT, + "do_brp_verification should return immediately instead of waiting for the background check, took {elapsed:?}" + ); + + wait_for_brp_status_finished(&csb_store).await; + + Ok(()) + } + + #[tokio::test] + async fn do_brp_verification_skips_already_validated_persons_on_a_later_call() + -> Result<(), AppError> { + let state = AppState::new_for_tests().await; + let csb_store = state + .csb_store_for_stream(StreamId::new(), ElectionConfig::EK27) + .await?; + + let mut person = sample_person_from_brp(); + person.address.house_number_addition = Some("nope".parse().unwrap()); + csb_store.add_person(person); + + let brp_client = BrpClient::new_for_test(); + + do_brp_verification(&csb_store, &brp_client).await?; + wait_for_brp_status_finished(&csb_store).await; + assert_eq!(csb_store.data.read().omissions.len(), 1); + assert_eq!(csb_store.data.read().brp_validations.len(), 1); + + // Re-running verification should skip the already-validated candidate + // rather than re-checking them and recording a duplicate omission. With + // no candidate left to check, the background task has nothing to wait + // on, so a short fixed delay is enough instead of polling for + // `Finished` (which the first run has already left behind). + do_brp_verification(&csb_store, &brp_client).await?; + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!(csb_store.data.read().omissions.len(), 1); + + Ok(()) + } } diff --git a/src/csb/store_csb/event.rs b/src/csb/store_csb/event.rs index 8af2075a..bb63115d 100644 --- a/src/csb/store_csb/event.rs +++ b/src/csb/store_csb/event.rs @@ -2,7 +2,11 @@ use serde::{Deserialize, Serialize}; use crate::{ Event, PgEvent, PgStoreData, StreamId, - structs::csb::{Correction, Omission, OmissionId}, + persons::PersonId, + structs::{ + brp::BrpStatus, + csb::{Correction, Omission, OmissionId}, + }, trans, utils::format_hash, }; @@ -40,6 +44,11 @@ pub enum CsbEvent { omission_id: OmissionId, }, UpdateCorrection(Correction), + BrpPersonValidated { + person: PersonId, + valid: bool, + }, + SetBrpStatus(BrpStatus), } impl Event for CsbEvent { @@ -53,6 +62,7 @@ impl Event for CsbEvent { | CsbEvent::UpdateOmission(_) | CsbEvent::DeleteOmission { .. } => "omission", CsbEvent::UpdateCorrection(_) => "correction", + CsbEvent::BrpPersonValidated { .. } | CsbEvent::SetBrpStatus(_) => "brp_validation", } } @@ -66,6 +76,8 @@ impl Event for CsbEvent { CsbEvent::UpdateOmission(_) => "update_omission", CsbEvent::DeleteOmission { .. } => "delete_omission", CsbEvent::UpdateCorrection(_) => "update_correction", + CsbEvent::BrpPersonValidated { .. } => "brp_person_validated", + Self::SetBrpStatus(_) => "brp_validation", } } @@ -81,6 +93,10 @@ impl Event for CsbEvent { CsbEvent::UpdateCorrection { .. } => { trans!("audit_log.event.update_correction", locale) } + CsbEvent::BrpPersonValidated { .. } => trans!("audit_log.event.brp_validation", locale), + Self::SetBrpStatus(_) => { + trans!("audit_log.event.set_brp_validation_state", locale) + } } } @@ -102,6 +118,8 @@ impl Event for CsbEvent { CsbEvent::CreateOmission(o) | CsbEvent::UpdateOmission(o) => o.description.clone(), CsbEvent::DeleteOmission { omission_id } => omission_id.to_string(), CsbEvent::UpdateCorrection(_) => String::new(), + CsbEvent::BrpPersonValidated { person, .. } => person.to_string(), + CsbEvent::SetBrpStatus(value) => value.to_string(), } } diff --git a/src/csb/store_csb/getters.rs b/src/csb/store_csb/getters.rs index 1295e35f..39a8e44b 100644 --- a/src/csb/store_csb/getters.rs +++ b/src/csb/store_csb/getters.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use parking_lot::{ RawRwLock, lock_api::{MappedRwLockReadGuard, RwLockReadGuard}, @@ -227,6 +229,14 @@ impl CsbStore { .collect() } + pub fn get_persons(&self, corrections: WithCorrections) -> Vec { + self.read(corrections).persons.values().cloned().collect() + } + + pub fn get_brp_validations(&self) -> HashMap { + self.data.read().brp_validations.clone() + } + /// Return the single stored omission. Test-only helper for asserting on /// omissions whose category has no dedicated getter (e.g. candidate lists). #[cfg(test)] diff --git a/src/csb/store_csb/mod.rs b/src/csb/store_csb/mod.rs index ad7c1ad9..28c39b36 100644 --- a/src/csb/store_csb/mod.rs +++ b/src/csb/store_csb/mod.rs @@ -16,7 +16,10 @@ use crate::{ common::{DisplayName, UtcDateTime}, persons::{Person, PersonId}, store::{StoreData, StoreEvent}, - structs::csb::{Correction, Omission, OmissionId}, + structs::{ + brp::BrpStatus, + csb::{Correction, Omission, OmissionId}, + }, }; /// Event-sourced domain projection for a single (stream, election) pair on the @@ -30,6 +33,8 @@ pub struct CsbStoreData { pub(crate) omissions: HashMap, pub(crate) csb_corrected_persons: HashMap, pub(crate) csb_corrected_display_name: Option, + pub(crate) brp_validations: HashMap, + pub(crate) brp_validation_status: BrpStatus, } impl StoreData for CsbStoreData { @@ -108,6 +113,10 @@ impl StoreData for CsbStoreData { correction.apply(person); } }, + CsbEvent::BrpPersonValidated { person, valid } => { + self.brp_validations.insert(person, valid); + } + CsbEvent::SetBrpStatus(value) => self.brp_validation_status = value, } } diff --git a/src/csb/store_main/mod.rs b/src/csb/store_main/mod.rs index a6459899..59d5a173 100644 --- a/src/csb/store_main/mod.rs +++ b/src/csb/store_main/mod.rs @@ -2,11 +2,13 @@ mod event; mod extractor; pub use event::CsbMainEvent; +use std::collections::HashMap; use serde::{Deserialize, Serialize}; use crate::{ Scope, StreamId, + persons::PersonId, store::{StoreData, StoreEvent}, }; @@ -20,6 +22,7 @@ pub const CSB_MAIN_STREAM_ID: StreamId = StreamId(uuid::Uuid::from_u128( #[derive(Debug, Default, Serialize, Deserialize)] pub struct CsbMainStoreData { pub(crate) events: Vec>, + pub(crate) brp_verifications: HashMap, } impl StoreData for CsbMainStoreData { diff --git a/src/state.rs b/src/state.rs index 0c3588e1..5a200a28 100644 --- a/src/state.rs +++ b/src/state.rs @@ -11,6 +11,7 @@ use crate::{ crypto::MasterKey, csb::CSB_MAIN_STREAM_ID, store::{Store, StoreRegistry}, + structs::brp::BrpClient, }; #[cfg(feature = "fixtures")] @@ -34,6 +35,7 @@ pub struct AppState { pub id_deriver: IdDeriver, pub auth_service_state: AuthServiceState, pub db_health: DbHealth, + pub brp_client: BrpClient, } /// Contract the application's request extractors expect from the router @@ -80,6 +82,13 @@ impl AppState { AuthServiceState::new_from_env().await? }; + let brp_client = BrpClient::new( + &config.brp_client.base_url, + &config.brp_client.api_key, + &config.brp_client.persons_endpoint, + config.brp_client.timeout, + ); + Ok(Self { config: Box::leak(Box::new(config)), store_registry, @@ -90,6 +99,7 @@ impl AppState { id_deriver, auth_service_state, db_health: DbHealth::default(), + brp_client, }) } @@ -178,6 +188,13 @@ impl AppState { let csb_main_store_registry = StoreRegistry::with_persistence(store_registry.persistence().clone(), master); + let brp_client = BrpClient::new( + &config.brp_client.base_url, + &config.brp_client.api_key, + &config.brp_client.persons_endpoint, + config.brp_client.timeout, + ); + Self { store_registry, csb_store_registry, @@ -188,6 +205,7 @@ impl AppState { id_deriver, auth_service_state, db_health: DbHealth::default(), + brp_client, } } } diff --git a/src/structs/brp/client.rs b/src/structs/brp/client.rs new file mode 100644 index 00000000..0328f130 --- /dev/null +++ b/src/structs/brp/client.rs @@ -0,0 +1,385 @@ +use reqwest::Client; +use serde::{Deserialize, Serialize}; +use std::time::Duration; + +use super::{BrpField, BrpPerson}; +use crate::{ + AppError, + candidate_lists::CandidateListId, + common::{Bsn, BsnOrNoneConfirmed}, + persons::Person, + structs::csb::{Omission, OmissionCategory}, +}; + +pub const BRP_PERSONS_ENDPOINT: &str = "haalcentraal/api/brp/personen"; +pub const BRP_TIMEOUT: u64 = 30; + +#[derive(Clone)] +pub struct BrpClient { + http_client: Client, + base_url: String, + api_key: String, + persons_endpoint: String, + timeout: Duration, +} + +impl BrpClient { + pub fn new(base_url: &str, api_key: &str, persons_endpoint: &str, timeout: Duration) -> Self { + Self { + http_client: Client::new(), + base_url: base_url.to_string(), + api_key: api_key.to_string(), + persons_endpoint: persons_endpoint.to_string(), + timeout, + } + } + + #[cfg(test)] + pub fn new_for_test() -> Self { + BrpClient::new( + "http://localhost:5010", + "", + BRP_PERSONS_ENDPOINT, + Duration::from_secs(5), + ) + } + + pub async fn get_persons(&self, query: &BrpQuery) -> Result, AppError> { + let url = format!("{}/{}", self.base_url, self.persons_endpoint); + + let response = self + .http_client + .post(&url) + .header("Authorization", format!("Bearer {}", self.api_key)) + .json(query) + .timeout(self.timeout) + .send() + .await?; + + match response.json::().await? { + BrpResponse::ConsultWithBsn { persons } => Ok(persons), + } + } + + pub async fn verify( + &self, + person: &Person, + candidate_lists: Vec, + ) -> Result, AppError> { + let query = match person.personal_data.bsn { + Some(BsnOrNoneConfirmed::Bsn(ref bsn)) => BrpQuery::ConsultWithBsn { + bsn: vec![bsn.clone()], + fields: vec![ + BrpField::Bsn, + BrpField::DateOfBirth, + BrpField::Gender, + BrpField::Initials, + BrpField::LastNamePrefix, + BrpField::LastName, + BrpField::StreetName, + BrpField::HouseNumber, + BrpField::HouseNumberAddition, + BrpField::PostalCode, + BrpField::PlaceOfResidence, + ], + }, + Some(BsnOrNoneConfirmed::NoneConfirmed) => { + // TODO: This needs to be implemented + tracing::error!( + "Person {} has BSN none confirmed. Use BRP search with address? Or manual verification", + person.id + ); + return Err(AppError::GenericNotFound); + } + None => { + // TODO: This needs to be implemented + tracing::warn!("Person {} does not have a BSN filled in", person.id,); + return Err(AppError::GenericNotFound); + } + }; + + let mut omissions = vec![]; + let mut add_omission = |title: &str, description: &str, help_text: &str| { + omissions.push(Omission::new( + OmissionCategory::Candidate { + person: person.id, + lists: candidate_lists.clone(), + }, + // TODO: These should likely be user configurable and translatable + title.to_string(), + description.to_string(), + help_text.to_string(), + )); + }; + + let brp_persons = self.get_persons(&query).await?; + let brp_person = match brp_persons.as_slice() { + [] => { + add_omission( + "Burgerservicenummer onbekend", + "Er is geen persoon gevonden met dit burgerservicenummer", + "Controleer of er een fout is gemaakt bij het invoeren", + ); + return Ok(omissions); + } + [brp_person] => brp_person, + [..] => { + add_omission( + "Burgerservicenummer niet uniek", + "Er zijn meerder personen gevonden met dit burgerservicenummer", + "Controleer of er een fout is gemaakt bij het invoeren", + ); + return Ok(omissions); + } + }; + + match &brp_person.address { + Some(address) => { + // Check all, except `known_in_bag` + if person.address.street_name != address.street_name { + add_omission( + "Onjuiste straatnaam", + "De straatnaam komt niet overeen met de BRP", + "Controleer de straatnaam", + ); + } + if person.address.house_number != address.house_number { + add_omission( + "Onjuist huisnummer", + "Het huisnummer komt niet overeen met de BRP", + "Controleer het huisnummer", + ); + } + if person.address.house_number_addition != address.house_number_addition { + add_omission( + "Onjuiste huisnummertoevoeging", + "De huisnummertoevoeging komt niet overeen met de BRP", + "Controleer de huisnummertoevoeging", + ); + } + if person.address.locality != address.locality { + add_omission( + "Onjuiste woonplaats", + "De woonplaats komt niet overeen met de BRP", + "Controleer de woonplaats", + ); + } + if person.address.postal_code != address.postal_code { + add_omission( + "Onjuiste postcode", + "De postcode komt niet overeen met de BRP", + "Controleer de postcode", + ); + } + } + None => { + tracing::warn!( + "Not a Dutch Address or no address at all (because the field 'verblijfplaats' was not included)" + ); + } + }; + + // Don't check first name (roepnaam) + if person.name.last_name != brp_person.name.last_name { + add_omission( + "Onjuiste achternaam", + "De achternaam komt niet overeen met de BRP", + "Controleer de achternaam", + ); + } + if person.name.last_name_prefix != brp_person.name.last_name_prefix { + add_omission( + "Onjuist voorvoegsel", + "Het voorvoegsel komt niet overeen met de BRP", + "Controleer het voorvoegsel", + ); + } + if person.name.initials != brp_person.name.initials { + add_omission( + "Onjuiste voorletters", + "De voorletters komen niet overeen met de BRP", + "Controleer de voorletters", + ); + } + + // Check all fields of personal_data except country, check gender only when filled in + if brp_person.personal_data.bsn != person.personal_data.bsn { + add_omission( + "Onjuist burgerservicenummer", + "Het burgerservicenummer komt niet overeen met de BRP", + "Controleer het burgerservicenummer", + ); + } + if brp_person.personal_data.date_of_birth != person.personal_data.date_of_birth { + add_omission( + "Onjuiste geboortedatum", + "De geboortedatum komt niet overeen met de BRP", + "Controleer de geboortedatum", + ); + } + // Gender field is optional, but if it is filled in, we check it + if person.personal_data.gender.is_some() + && brp_person.personal_data.gender != person.personal_data.gender + { + add_omission( + "Onjuist geslacht", + "Het geslacht komt niet overeen met de BRP", + "Controleer het geslacht", + ); + } + + Ok(omissions) + } +} + +#[derive(Debug, Serialize)] +#[serde(tag = "type")] +pub enum BrpQuery { + #[serde(rename = "RaadpleegMetBurgerservicenummer")] + ConsultWithBsn { + #[serde(rename = "burgerservicenummer")] + bsn: Vec, + fields: Vec, + }, +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "type")] +pub enum BrpResponse { + #[serde(rename = "RaadpleegMetBurgerservicenummer")] + ConsultWithBsn { + #[serde(rename = "personen")] + persons: Vec, + }, +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use crate::{ + persons::PersonId, + test_utils::{sample_person, sample_person_from_brp}, + }; + + use super::*; + + #[tokio::test] + async fn brp_request() { + let brp_client = BrpClient::new_for_test(); + let query = BrpQuery::ConsultWithBsn { + bsn: vec!["100600505".parse().unwrap()], + fields: vec![BrpField::LastName], + }; + + let response = brp_client.get_persons(&query).await.unwrap(); + let expected = "Digid 1 100600505 geslachtsnaam".parse().unwrap(); + assert!(response.first().unwrap().name.last_name == expected); + } + + #[tokio::test] + async fn brp_verify() { + let brp_client = BrpClient::new_for_test(); + + let person = sample_person_from_brp(); + + match brp_client.verify(&person, Vec::new()).await { + Err(e) => panic!("brp verification error: {e}"), + Ok(omissions) if !omissions.is_empty() => panic!( + "person could not be verified: {}\nFollowing omissions were found: {:?}", + serde_json::to_string_pretty(&person).unwrap(), + omissions + ), + _ => {} + } + } + + #[tokio::test] + async fn brp_verify_returns_omissions() { + let brp_client = BrpClient::new_for_test(); + + let list_id = CandidateListId::new(); + let mut person = sample_person(PersonId::new()); + // Dit bsn voldoet aan de 11-proef maar staat niet in de mock brp + person.personal_data.bsn = Some("123456782".parse().unwrap()); + match brp_client.verify(&person, vec![list_id]).await { + Ok(omissions) => { + assert_eq!(omissions.len(), 1); + let omission = &omissions[0]; + let OmissionCategory::Candidate { lists, .. } = &omission.category else { + panic!("Unexpected omission category") + }; + assert_eq!(lists, &[list_id]); + assert_eq!( + omission.description, + "Er is geen persoon gevonden met dit burgerservicenummer", + ) + } + Err(e) => panic!("{e}"), + } + + let mut person = sample_person_from_brp(); + person.address.house_number_addition = Some("nope".parse().unwrap()); + match brp_client.verify(&person, Vec::new()).await { + Ok(omissions) => { + assert_eq!(omissions.len(), 1); + let omission = &omissions[0]; + assert!(matches!( + omission.category, + OmissionCategory::Candidate { .. } + )); + assert_eq!( + omission.description, + "De huisnummertoevoeging komt niet overeen met de BRP", + ) + } + Err(e) => panic!("{e}"), + } + + let mut person = sample_person(PersonId::new()); + // De gegevens in de brp voor dit bsn komen in zijn geheel niet overeen. Dit zou kunnen voorkomen + // als het verkeerde bsn is ingevuld. + person.personal_data.bsn = Some("999992806".parse().unwrap()); + + let expected_titles: HashSet = [ + "Onjuist huisnummer".to_string(), + "Onjuiste achternaam".to_string(), + "Onjuiste geboortedatum".to_string(), + "Onjuiste huisnummertoevoeging".to_string(), + "Onjuiste postcode".to_string(), + "Onjuiste straatnaam".to_string(), + "Onjuiste voorletters".to_string(), + "Onjuiste woonplaats".to_string(), + ] + .into(); + + match brp_client.verify(&person, Vec::new()).await { + Ok(omissions) => { + let actual_titles = HashSet::from_iter(omissions.into_iter().map(|o| o.title)); + assert_eq!( + expected_titles.symmetric_difference(&actual_titles).count(), + 0 + ) + } + Err(e) => panic!("{e}"), + } + } + + #[tokio::test] + async fn omission_includes_candidate_lists() { + let brp_client = BrpClient::new_for_test(); + + let person = sample_person_from_brp(); + let list_id = CandidateListId::new(); + + match brp_client.verify(&person, vec![list_id]).await { + Err(e) => panic!("brp verification error: {e}"), + Ok(omissions) if !omissions.is_empty() => panic!( + "person could not be verified: {}\nFollowing omissions were found: {:?}", + serde_json::to_string_pretty(&person).unwrap(), + omissions + ), + _ => {} + } + } +} diff --git a/src/structs/brp/field.rs b/src/structs/brp/field.rs new file mode 100644 index 00000000..9f40f19c --- /dev/null +++ b/src/structs/brp/field.rs @@ -0,0 +1,76 @@ +use serde::Serialize; + +// ontbreekt: Aanduiding bijzonder Nederlanderschap +// ontbreekt: Ingangsdatum geldigheid met betrekking tot de elementen van de categorie Nationaliteit +#[derive(Debug, Serialize)] +pub enum BrpField { + // Personen + #[serde(rename = "burgerservicenummer")] + Bsn, + #[serde(rename = "naam.voornamen")] + FirstNames, + #[serde(rename = "naam.voorletters")] + Initials, + #[serde(rename = "naam.adellijkeTitelPredicaat")] + TitleOfNobility, + #[serde(rename = "naam.voorvoegsel")] + LastNamePrefix, + #[serde(rename = "naam.geslachtsnaam")] + LastName, + #[serde(rename = "geboorte.datum")] + DateOfBirth, + #[serde(rename = "geslacht")] + Gender, + #[serde(rename = "naam.aanduidingNaamgebruik")] + DesignatedNameUsage, + + // Nationaliteit + #[serde(rename = "nationaliteiten.nationaliteit")] + Nationality, + + // Partners + #[serde(rename = "partners.naam.voorvoegsel")] + PartnerLastNamePrefix, + #[serde(rename = "partners.naam.geslachtsnaam")] + PartnerLastName, + #[serde(rename = "partners.aangaanHuwelijkPartnerschap.datum")] + DateOfMarriage, + #[serde(rename = "partners.ontbindingHuwelijkPartnerschap")] + DateOfDissolutionMarriage, + + // Date of death + #[serde(rename = "overlijden.datum")] + DateOfDeath, + + // Place of residence + // TODO: What to do with Registratie Niet Ingezetenen? + #[serde(rename = "gemeenteVanInschrijving")] + RegisteredMunicipality, + #[serde(rename = "datumInschrijvingInGemeente")] + DateMunicipalRegistration, + #[serde(rename = "verblijfplaats.verblijfadres.korteStraatnaam")] + StreetName, + #[serde(rename = "verblijfplaats.verblijfadres.huisnummer")] + HouseNumber, + #[serde(rename = "verblijfplaats.verblijfadres.huisletter")] + HouseLetter, + #[serde(rename = "verblijfplaats.verblijfadres.huisnummertoevoeging")] + HouseNumberAddition, + #[serde(rename = "verblijfplaats.verblijfadres.postcode")] + PostalCode, + #[serde(rename = "verblijfplaats.verblijfadres.woonplaats")] + PlaceOfResidence, + + // Not sure if these are correct. They should be specifically foreign, but they + // may also apply to interior addresses + #[serde(rename = "verblijfplaats.verblijfadres.land")] + CountryOfResidence, // Land adres buitenland + #[serde(rename = "verblijfplaats.datumVan")] + ResidenceDateFrom, // Datum aanvang adres buitenland + #[serde(rename = "verblijfplaats.verblijfadres.regel1")] + AddressLine1, // Regel 1 adres buitenland + #[serde(rename = "verblijfplaats.verblijfadres.regel2")] + AddressLine2, // Regel 2 adres buitenland + #[serde(rename = "verblijfplaats.verblijfadres.regel3")] + AddressLine3, // Regel 3 adres buitenland +} diff --git a/src/structs/brp/mod.rs b/src/structs/brp/mod.rs new file mode 100644 index 00000000..43800111 --- /dev/null +++ b/src/structs/brp/mod.rs @@ -0,0 +1,9 @@ +mod client; +mod field; +mod person; +mod status; + +pub use client::{BRP_PERSONS_ENDPOINT, BRP_TIMEOUT, BrpClient}; +pub use field::BrpField; +pub use person::BrpPerson; +pub use status::BrpStatus; diff --git a/src/structs/brp/person.rs b/src/structs/brp/person.rs new file mode 100644 index 00000000..d803f53b --- /dev/null +++ b/src/structs/brp/person.rs @@ -0,0 +1,155 @@ +use chrono::NaiveDate; +use serde::Deserialize; + +use crate::{ + common::{Bsn, BsnOrNoneConfirmed, DateOfBirth, DutchAddress, FullName}, + persons::PersonalData, +}; + +#[derive(Debug, Deserialize)] +#[serde(from = "BrpPersonRaw")] +pub struct BrpPerson { + pub name: FullName, + pub personal_data: PersonalData, + pub address: Option, +} + +structstruck::strike! { + #[structstruck::each[derive(Debug, Deserialize)]] + struct BrpPersonRaw { + #[serde(rename = "burgerservicenummer")] + bsn: Option, + #[serde(rename = "geslacht")] + gender: Option, + #[serde(rename = "naam")] + name: Option, + #[serde(rename = "voorvoegsel")] + last_name_prefix: Option, + #[serde(rename = "voorletters")] + initials: Option + }>, + #[serde(rename = "geboorte")] + birth: Option + }> + }>, + #[serde(rename = "verblijfplaats")] + place_of_residence: Option< + #[serde(tag = "type")] + enum BrpPlaceOfResidence { + #[serde(rename = "Adres")] + Address { + #[serde(rename = "verblijfadres")] + residence_address: struct BrpAddress { + #[serde(rename = "korteStraatnaam")] + street_name: Option, + #[serde(rename = "huisnummer")] + house_number: Option, + #[serde(rename = "huisnummertoevoeging")] + house_number_addition: Option, + #[serde(rename = "postcode")] + postal_code: Option, + #[serde(rename = "woonplaats")] + place_of_residence: Option, + } + }, + #[serde(other)] + NonDutchAddress + }> + } +} + +impl From for BrpPerson { + fn from(raw: BrpPersonRaw) -> Self { + let name = raw + .name + .map(|naam| FullName { + // First name isn't checked, because this does not necessarily need to be the same (roepnaam) + first_name: None, + last_name: naam + .last_name + .and_then(|s| s.parse().ok()) + .unwrap_or_default(), + last_name_prefix: naam.last_name_prefix.and_then(|s| s.parse().ok()), + initials: naam + .initials + .and_then(|s| s.parse().ok()) + .unwrap_or_default(), + }) + .unwrap_or_default(); + + let bsn = raw + .bsn + .and_then(|s| s.parse::().ok()) + .map(BsnOrNoneConfirmed::Bsn); + + let gender = raw.gender.and_then(|g| g.gender.parse().ok()); + + let date_of_birth = raw + .birth + .as_ref() + .and_then(|b| b.date.as_ref()) + .and_then(|d| d.date.as_ref()) + .and_then(|s| NaiveDate::parse_from_str(s, "%Y-%m-%d").ok()) + .map(DateOfBirth::from); + + let (address, place_of_residence) = match raw.place_of_residence { + Some(BrpPlaceOfResidence::Address { + residence_address: ra, + }) => { + let addr = Some(DutchAddress { + street_name: ra.street_name.and_then(|s| s.parse().ok()), + house_number: ra.house_number.and_then(|s| s.to_string().parse().ok()), + house_number_addition: ra.house_number_addition.and_then(|s| s.parse().ok()), + locality: ra + .place_of_residence + .as_deref() + .and_then(|s| s.parse().ok()), + postal_code: ra.postal_code.and_then(|s| s.parse().ok()), + // Known in BRP probably implies known in bag, I guess maybe this could be Some(true), but + // I don't think it matters + known_in_bag: None, + }); + + // TODO: Is place of residence really the same as locality (above). + // (though note that above is parsed as `Locality`, and below as `PlaceOfResidence`) + let por = ra.place_of_residence.and_then(|s| s.parse().ok()); + + (addr, por) + } + Some(BrpPlaceOfResidence::NonDutchAddress) => { + // TODO: How to handle this? Set the address to None and conduct an additional BRP check + // for the Authorised Person? + tracing::warn!("Person has an non-Dutch address"); + (None, None) + } + None => { + tracing::error!("Field 'verblijfplaats' not included"); + (None, None) + } + }; + + BrpPerson { + name, + personal_data: PersonalData { + gender, + bsn, + date_of_birth, + place_of_residence, + // TODO: Can country be None here? Because we check with the BRP whether the address is international. + // If it is, then `address` will be None (since we can't verify international addresses) and we know that + // instead, it is necesarry to verify the Authorised Person's address + country: None, + }, + address, + } + } +} diff --git a/src/structs/brp/status.rs b/src/structs/brp/status.rs new file mode 100644 index 00000000..a4ac5883 --- /dev/null +++ b/src/structs/brp/status.rs @@ -0,0 +1,22 @@ +use serde::{Deserialize, Serialize}; +use std::fmt::Display; + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub enum BrpStatus { + #[default] + NotStarted, + InProgress, + Aborted(String), + Finished, +} + +impl Display for BrpStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BrpStatus::NotStarted => write!(f, "not_started"), + BrpStatus::InProgress => write!(f, "in_progress"), + BrpStatus::Aborted(_) => write!(f, "aborted"), + BrpStatus::Finished => write!(f, "finished"), + } + } +} diff --git a/src/structs/mod.rs b/src/structs/mod.rs index a3370ad3..fcde1639 100644 --- a/src/structs/mod.rs +++ b/src/structs/mod.rs @@ -1,6 +1,7 @@ //! Shared domain model structs, used by both the political group section //! (`src/pg`) and the central voting bureau section (`src/csb`). pub mod audit_log; +pub mod brp; pub mod candidate_lists; pub mod candidates; pub mod common; diff --git a/src/utils/test_utils.rs b/src/utils/test_utils.rs index ea4da436..fbb0cbdd 100644 --- a/src/utils/test_utils.rs +++ b/src/utils/test_utils.rs @@ -158,6 +158,35 @@ pub fn sample_person(id: PersonId) -> Person { } } +pub fn sample_person_from_brp() -> Person { + Person { + id: PersonId::new(), + name: FullName { + first_name: Some("Tina-Antïna".parse().unwrap()), + last_name: "Bruin".parse().unwrap(), + last_name_prefix: Some("de".parse().unwrap()), + initials: "T.".parse().unwrap(), + }, + personal_data: PersonalData { + gender: Some(Gender::Female), + bsn: Some("900194054".parse().unwrap()), + date_of_birth: Some("11-12-1990".parse().unwrap()), + place_of_residence: Some("Utrecht".parse().unwrap()), + country: Some("NL".parse().unwrap()), + }, + address: DutchAddress { + street_name: Some("Croeselaan".parse().unwrap()), + house_number: Some("15".parse().unwrap()), + house_number_addition: None, + locality: Some("Utrecht".parse().unwrap()), + postal_code: Some("3521BJ".parse().unwrap()), + known_in_bag: None, + }, + representative: None, + updated_at: Default::default(), + } +} + pub fn sample_person_with_last_name(id: PersonId, last_name: &str) -> Person { sample_person_with(id, None, last_name, None, "H.A.H.A.") }