Added goldens for USCensuspep imports

scripts/us_census/pep/monthly_population_estimate/golden_data/golden_observations.csv

@@ -0,0 +1,2 @@
+"Location"
+"country/USA"

scripts/us_census/pep/monthly_population_estimate/golden_data/golden_summary_report.csv

@@ -0,0 +1,7 @@
+"NumPlaces","MeasurementMethods","observationPeriods","Units","MinDate","ScalingFactors","StatVar"
+"1","[CensusPEPSurvey]","[P1M]","[]","1980-04","[]","Count_Person_USResident"
+"1","[CensusPEPSurvey]","[P1M]","[]","1990-04","[]","Count_Person_Civilian_NonInstitutionalized"
+"1","[dcAggregate/CensusPEPSurvey]","[P1M]","[]","1980-04","[]","Count_Person_InUSArmedForcesOverseas"
+"1","[CensusPEPSurvey]","[P1M]","[]","2010-04","[]","Count_Person_ResidesInHousehold"
+"1","[CensusPEPSurvey]","[P1M]","[]","1980-04","[]","Count_Person_Civilian"
+"1","[CensusPEPSurvey]","[P1M]","[]","1980-04","[]","Count_Person_USResidentOrInUSArmedForcesOverseas"

scripts/us_census/pep/monthly_population_estimate/manifest.json

@@ -19,7 +19,8 @@
                     "cleaned_csv": "output/USA_Population_Count.csv"
                 }
             ],
-            "cron_schedule": "0 7 * * 6"
+            "cron_schedule": "0 7 * * 6",
+            "validation_config_file": "validation_config.json"
         }
     ]
 }

scripts/us_census/pep/monthly_population_estimate/preprocess.py

@@ -585,10 +585,10 @@ def _concat_cols(col: pd.Series) -> pd.Series:
     Returns:
         res (Series) : Concatenated DataFrame Columns
     """
-    res = col[0]
-    if col[1] is None:
+    res = col.iloc[0]
+    if col.iloc[1] is None:
         return res
-    res = col[0] + ' ' + col[1]
+    res = col.iloc[0] + ' ' + col.iloc[1]
     return res
 
 
@@ -654,7 +654,7 @@ def download_files():
                     ]
                     df = pd.read_table(url,
                                        index_col=False,
-                                       delim_whitespace=True,
+                                       sep=r'\s+',
                                        engine='python',
                                        skiprows=17,
                                        names=cols)

scripts/us_census/pep/monthly_population_estimate/validation_config.json

@@ -0,0 +1,32 @@
+{
+    "schema_version": "1.0",
+    "rules": [
+        {
+            "rule_id": "check_deleted_records_percent",
+            "description": "Checks that the percentage of deleted points is within the threshold.",
+            "validator": "DELETED_RECORDS_PERCENT",
+            "params": {
+                "threshold": 0.1
+            }
+        },
+        {
+            "rule_id": "check_goldens_summary_report",
+            "validator": "GOLDENS_CHECK",
+            "params": {
+                "golden_files": "../../../../golden_data/golden_summary_report.csv"
+            }
+        },
+        {
+            "rule_id": "Check_goldens_output_csv",
+            "description": "Verifies the generated output CSV data matches established critical golden records",
+            "validator": "GOLDENS_CHECK",
+            "params": {
+                "golden_files": "../../../../golden_data/golden_observations.csv",
+                "input_files": "../../../../output/USA_Population_Count.csv"
+            }
+        }
+    ]
+}
+
+
+

scripts/us_census/pep/population_estimates_by_asr/county_1990_2000.py

@@ -42,7 +42,7 @@ def county1990(output_folder: str):
                 '/popest/tables/1990-2000/counties/asrh/casrh'+str(j)+'.txt'
             cols=['Year','geo_ID','Race',0,1,2,3,4,5,6,7\
                 ,8,9,10,11,12,13,14,15,16,17]
-            df = pd.read_table(url,index_col=False,delim_whitespace=True\
+            df = pd.read_table(url,index_col=False,sep=r'\s+'\
                 ,skiprows=16,skipfooter=14,engine='python',names=cols,\
                     encoding='ISO-8859-1')
             #Writing raw data to csv