Added historical data in gcs bucket WorldBankDatasets

scripts/world_bank/datasets/process.py

@@ -2,6 +2,7 @@
 import os
 import csv
 import sys
+import pandas as pd
 from absl import logging, flags, app
 from collections import Counter
 
@@ -29,6 +30,10 @@
     "historical file path")
 flags.DEFINE_string("historical_file", "bq-results-20250423.csv",
                     "historical file name")
+flags.DEFINE_string(
+    'historical_gcs_path',
+    'https://storage.mtls.cloud.google.com/unresolved_mcf/world_bank/datasets/deleted_historical_data_06_2026.csv',
+    'GCS path to the deleted historical data CSV file')
 
 DCID_MAP = {}
 IGNORE_DCIDS = set()
@@ -190,6 +195,34 @@ def main(_):
             f"\nSuccessfully processed {len(input_files)} files. Combined output written to '{output_file_path}'"
         )
 
+        # Read deleted historical data from GCS if it exists
+        try:
+            logging.info(
+                f"Reading historical deleted data from GCS: {FLAGS.historical_gcs_path}"
+            )
+            final_df = pd.read_csv(output_file_path)
+            with file_util.FileIO(FLAGS.historical_gcs_path, 'r') as f:
+                deleted_df = pd.read_csv(f)
+
+            # Combine dataframes. final_df is placed first so its versions are preferred.
+            final_df = pd.concat([final_df, deleted_df], ignore_index=True)
+
+            # Deduplicate based on composite keys, keeping the first occurrence (from final_df)
+            composite_keys = [
+                'indicatorcode', 'statvar', 'measurementmethod',
+                'observationabout', 'observationdate', 'unit'
+            ]
+            final_df = final_df.drop_duplicates(subset=composite_keys,
+                                                keep='first')
+            final_df.to_csv(output_file_path, index=False)
+            logging.info(
+                "Successfully merged and de-duplicated deleted historical data."
+            )
+        except Exception as e:
+            logging.warning(
+                f"Could not read historical deleted data from GCS: {e}. Proceeding with fresh data only."
+            )
+
         file_util.file_copy(f'{FLAGS.gs_path}{FLAGS.historical_file}',
                             f'{output}/{FLAGS.historical_file}')