NOAA_GPCC_StandardardizedPrecipitation_fix

scripts/noaa/gpcc_spi/golden_data/golden_summary_report_drought_spi_9m_polygon_place_svobs.csv

@@ -0,0 +1,3 @@
+"ScalingFactors","Units","MinDate","observationPeriods","StatVar","MeasurementMethods","NumPlaces"
+"[]","[SquareKilometer]","2010","[P1Y, P1M]","Area_DroughtEvent","[]","14770"
+"[]","[]","2010","[P1Y, P1M]","Count_DroughtEvent","[]","14770"

scripts/noaa/gpcc_spi/golden_data/golden_summary_report_drought_spi_9m_polygon_svobs.csv

@@ -0,0 +1,3 @@
+"StatVar","ScalingFactors","observationPeriods","NumPlaces","Units","MinDate","MeasurementMethods"
+"Area_DroughtEvent","[]","[]","23907","[SquareKilometer]","2010-12","[dcAggregate/PearsonDistribution]"
+"standardizedPrecipitationIndex","[]","[]","23907","[]","2010-12","[dcAggregate/PearsonDistribution]"

scripts/noaa/gpcc_spi/golden_data/golden_summary_report_gpcc_spi.csv

@@ -0,0 +1,7 @@
+"StatVar","observationPeriods","MeasurementMethods","MinDate","ScalingFactors","NumPlaces","Units"
+"StandardizedPrecipitationIndex_Atmosphere_36MonthPeriod","[P36M]","[PearsonDistribution]","2010-12","[]","18967","[]"
+"StandardizedPrecipitationIndex_Atmosphere_1MonthPeriod","[P1M]","[PearsonDistribution]","2010-12","[]","18967","[]"
+"StandardizedPrecipitationIndex_Atmosphere_3MonthPeriod","[P3M]","[PearsonDistribution]","2010-12","[]","18967","[]"
+"StandardizedPrecipitationIndex_Atmosphere_6MonthPeriod","[P6M]","[PearsonDistribution]","2010-12","[]","18967","[]"
+"StandardizedPrecipitationIndex_Atmosphere_72MonthPeriod","[P72M]","[PearsonDistribution]","2010-12","[]","18967","[]"
+"StandardizedPrecipitationIndex_Atmosphere_9MonthPeriod","[P9M]","[PearsonDistribution]","2010-12","[]","18967","[]"

scripts/noaa/gpcc_spi/golden_data/golden_summary_report_gpcc_spi_aggregation.csv

@@ -0,0 +1,7 @@
+"StatVar","MinDate","MeasurementMethods","NumPlaces","Units","ScalingFactors","observationPeriods"
+"StandardizedPrecipitationIndex_Atmosphere_36MonthPeriod","2010-12","[dcAggregate/GridWeightedPearson]","5392","[]","[]","[P36M]"
+"StandardizedPrecipitationIndex_Atmosphere_1MonthPeriod","2010-12","[dcAggregate/GridWeightedPearson]","5392","[]","[]","[P1M]"
+"StandardizedPrecipitationIndex_Atmosphere_3MonthPeriod","2010-12","[dcAggregate/GridWeightedPearson]","5392","[]","[]","[P3M]"
+"StandardizedPrecipitationIndex_Atmosphere_6MonthPeriod","2010-12","[dcAggregate/GridWeightedPearson]","5392","[]","[]","[P6M]"
+"StandardizedPrecipitationIndex_Atmosphere_72MonthPeriod","2010-12","[dcAggregate/GridWeightedPearson]","5392","[]","[]","[P72M]"
+"StandardizedPrecipitationIndex_Atmosphere_9MonthPeriod","2010-12","[dcAggregate/GridWeightedPearson]","5392","[]","[]","[P9M]"

scripts/noaa/gpcc_spi/manifest.json

@@ -35,7 +35,13 @@
                     "cleaned_csv": "output_files/events_spi_9m_polygon/event_svobs/drought_spi_9m_polygon_svobs.csv"
                 }
             ],
-            "cron_schedule": "0 9 1,15 * *"
+            "cron_schedule": "0 9 1,15 * *",
+            "validation_config_file": "validation_config.json",
+            "resource_limits": {
+                "cpu": 64,
+                "memory": 256,
+                "disk": 4096
+            }
         }
     ]
 }

scripts/noaa/gpcc_spi/preprocess_gpcc_spi.py

@@ -49,15 +49,6 @@
                     'Dash separated start date. Defaults to the running date.')
 
 
-def to_one_degree_grid_place(latlon):
-    """Latlng data to grid format.
-
-  Change longitude from 0 ~ 360 scale to -180 ~ 180 scale.
-  Change coordinate from middle of the grid to north west point of the grid.
-  """
-    return f'grid_1/{math.floor(latlon[0])}_{math.floor((latlon[1]+180)%360 - 180)}'
-
-
 def nc_to_df(nc_path, period, spi_col, start_date, end_date):
     """Read a netcdf and parse to df."""
     logging.info(f"Read a netcdf file - {nc_path} and parse to df.")
@@ -79,7 +70,13 @@ def nc_to_df(nc_path, period, spi_col, start_date, end_date):
     df['period'] = f'"[{int(period)} dcs:Monthly]"'
     # Adding observation period in output csv
     df['observationPeriod'] = f'P{int(period)}M'
-    df['place'] = df[['lat', 'lon']].apply(to_one_degree_grid_place, axis=1)
+    # Convert latitude and longitude to 1-degree grid format.
+    # Longitude is converted from 0-360 scale to -180-180 scale.
+    # Coordinates are floored to map them to the grid's corner.
+    lat_floor = (df['lat'] // 1).astype(int)
+    lon_floor = (((df['lon'] + 180) % 360 - 180) // 1).astype(int)
+    df['place'] = ('grid_1/' + lat_floor.astype(str) + '_' +
+                   lon_floor.astype(str))
     df = df.drop('lat', axis=1)
     df = df.drop('lon', axis=1)
     return df

scripts/noaa/gpcc_spi/validation_config.json

@@ -0,0 +1,56 @@
+{
+  "schema_version": "1.0",
+  "rules": [
+    {
+      "rule_id": "check_deleted_records_percent",
+      "description": "Checks that the percentage of deleted records for the entire import is within threshold.",
+      "validator": "DELETED_RECORDS_PERCENT",
+      "params": { "threshold": 0.1}
+    },
+    {
+      "rule_id": "check_goldens_summary_gpcc_spi",
+      "description": "Validates gpcc_spi data against its golden summary report.",
+      "validator": "GOLDENS_CHECK",
+      "params": {
+        "golden_files": "../../../../golden_data/golden_summary_report_gpcc_spi.csv",
+        "input_files": "../../input0/genmcf/summary_report.csv"
+      }
+    },
+    {
+      "rule_id": "check_goldens_summary_gpcc_spi_aggregation",
+      "description": "Validates gpcc_spi_aggregation data against its golden summary report.",
+      "validator": "GOLDENS_CHECK",
+      "params": {
+        "golden_files": "../../../../golden_data/golden_summary_report_gpcc_spi_aggregation.csv",
+        "input_files": "../../input1/genmcf/summary_report.csv"
+      }
+    },
+    {
+      "rule_id": "check_goldens_summary_drought_spi_9m_polygon_place_svobs",
+      "description": "Validates drought_spi_9m_polygon_place_svobs data against its golden summary report.",
+      "validator": "GOLDENS_CHECK",
+      "params": {
+        "golden_files": "../../../../golden_data/golden_summary_report_drought_spi_9m_polygon_place_svobs.csv",
+        "input_files": "../../input2/genmcf/summary_report.csv"
+      }
+    },
+    {
+      "rule_id": "check_goldens_summary_drought_spi_9m_polygon_svobs",
+      "description": "Validates drought_spi_9m_polygon_svobs data against its golden summary report.",
+      "validator": "GOLDENS_CHECK",
+      "params": {
+        "golden_files": "../../../../golden_data/golden_summary_report_drought_spi_9m_polygon_svobs.csv",
+        "input_files": "../../input4/genmcf/summary_report.csv"
+      }
+    },
+    {
+      "rule_id": "check_goldens_output_csv_gpcc_spi_aggregation",
+      "description": "Verifies the generated gpcc_spi_aggregation output CSV data matches established critical golden records",
+      "validator": "GOLDENS_CHECK",
+      "params": {
+        "golden_files": "../../../../golden_data/golden_observations_gpcc_spi_aggregation.csv",
+        "input_files": "../../../../output_files/aggregations/*.csv"
+      }
+    }
+  ]
+}