From 9bb02aab694d03719329c3d657f17f94bd822211 Mon Sep 17 00:00:00 2001 From: gmguarino Date: Mon, 13 Apr 2026 16:16:01 +0200 Subject: [PATCH 1/4] add script to push to hub --- data/update_huggingface_dataset.py | 110 +++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 data/update_huggingface_dataset.py diff --git a/data/update_huggingface_dataset.py b/data/update_huggingface_dataset.py new file mode 100644 index 0000000..8e6d838 --- /dev/null +++ b/data/update_huggingface_dataset.py @@ -0,0 +1,110 @@ +from datetime import date, timedelta, datetime +import os +from datasets import load_dataset, Dataset, DatasetDict +from dotenv import load_dotenv +import pandas as pd +import psycopg +from typing import List + +import requests + +load_dotenv(os.path.join(os.path.dirname(__file__), ".env")) + + +def fetch_articles(columns: List[str]=None) -> pd.DataFrame: + conninfo = ( + f"host={os.getenv['PG_HOST']} port={os.getenv['PG_PORT']} " + f"dbname={os.getenv['PG_DATABASE']} user={os.getenv['PG_USER']} " + f"password={os.getenv['PG_PASSWORD']}" + ) + columns_str = ", ".join(columns) if columns else "*" + query = f""" + SELECT {columns_str} + FROM analytics.task_global_completion + """ + with psycopg.connect(conninfo) as conn: + with conn.cursor() as cur: + cur.execute(query) + rows = cur.fetchall() + columns = [desc.name for desc in cur.description] + + return pd.DataFrame(rows, columns=columns) + +def get_week_number(record): + format_str = "%Y-%m-%dT%H:%M:%S" + if len(record["data_item_start"]) > 19: + format_str = format_str + "%z" + week_number = ( + datetime.strptime(record["data_item_start"], format_str) + .isocalendar() + .week + ) + return week_number + + +def split_df_by_week(df): + df["week_number"] = df.apply(get_week_number, axis=1) + df_train = df.loc[df.week_number % 4] + df_test = df.loc[df.week_number % 4 == 0] + return df_train, df_test + + +def generate_hf_dataset(df_train, df_test): + df_train.data_item_start = df_train.data_item_start.dt.strftime("%Y-%m-%dT%H:%M:%S%z") + df_test.data_item_start = df_test.data_item_start.dt.strftime("%Y-%m-%dT%H:%M:%S%z") + dataset = DatasetDict( + { + "train": Dataset.from_pandas(df_train), + "test": Dataset.from_pandas(df_test), + } + ) + return dataset + + +DATASET_COLUMNS = [ + "task_completion_aggregate_id", + "task_aggregate_id", + "created_at", + "updated_at", + "is_labeled", + "project_id", + "country", + "data_item_id", + "data_item_channel", + "data_item_channel_name", + "data_item_channel_title", + "data_item_channel_program", + "data_item_channel_program_type", + "data_item_day", + "data_item_month", + "data_item_year", + "data_item_start", + "data_item_model_name", + "data_item_model_reason", + "data_item_model_result", + "data_item_plaintext", + "data_item_plaintext_whisper", + "data_item_url_mediatree", + "mesinfo_choice", + "locuteur_choice", + "mesinfo_correct", + "mesinfo_incorrect", + "speaker_journalist", + "speaker_commentator", + "speaker_guest", + "speaker_politician", + "speaker_audience", + "speaker_unknown", + "mesinfo_corrected", + "mesinfo_corrected_bool", + "debunk_references", + "claims", + "explanations", + "other_comments" +] + +if __name__=="__main__": + df = fetch_articles(DATASET_COLUMNS) + df_train, df_test = split_df_by_week(df) + dataset = generate_hf_dataset(df_train, df_test) + dataset.push_to_hub("DataForGood/climateguard-training", private=True, token=True) \ No newline at end of file From ca76c3ab2ceb889a1e13f7e593b0e61b9bf6a70c Mon Sep 17 00:00:00 2001 From: gmguarino Date: Mon, 13 Apr 2026 18:28:35 +0200 Subject: [PATCH 2/4] fix getenv call --- data/update_huggingface_dataset.py | 33 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/data/update_huggingface_dataset.py b/data/update_huggingface_dataset.py index 8e6d838..8270efc 100644 --- a/data/update_huggingface_dataset.py +++ b/data/update_huggingface_dataset.py @@ -1,21 +1,21 @@ -from datetime import date, timedelta, datetime import os -from datasets import load_dataset, Dataset, DatasetDict -from dotenv import load_dotenv -import pandas as pd -import psycopg +from datetime import date, datetime, timedelta from typing import List +import pandas as pd +import psycopg import requests +from datasets import Dataset, DatasetDict, load_dataset +from dotenv import load_dotenv load_dotenv(os.path.join(os.path.dirname(__file__), ".env")) -def fetch_articles(columns: List[str]=None) -> pd.DataFrame: +def fetch_articles(columns: List[str] = None) -> pd.DataFrame: conninfo = ( - f"host={os.getenv['PG_HOST']} port={os.getenv['PG_PORT']} " - f"dbname={os.getenv['PG_DATABASE']} user={os.getenv['PG_USER']} " - f"password={os.getenv['PG_PASSWORD']}" + f"host={os.getenv('PG_HOST', 'localhost')} port={os.getenv('PG_PORT', 5432)} " + f"dbname={os.getenv('PG_DATABASE', 'postgres')} user={os.getenv('PG_USER', 'user')} " + f"password={os.getenv('PG_PASSWORD', 'supersecret')}" ) columns_str = ", ".join(columns) if columns else "*" query = f""" @@ -30,14 +30,13 @@ def fetch_articles(columns: List[str]=None) -> pd.DataFrame: return pd.DataFrame(rows, columns=columns) + def get_week_number(record): format_str = "%Y-%m-%dT%H:%M:%S" if len(record["data_item_start"]) > 19: format_str = format_str + "%z" week_number = ( - datetime.strptime(record["data_item_start"], format_str) - .isocalendar() - .week + datetime.strptime(record["data_item_start"], format_str).isocalendar().week ) return week_number @@ -50,7 +49,9 @@ def split_df_by_week(df): def generate_hf_dataset(df_train, df_test): - df_train.data_item_start = df_train.data_item_start.dt.strftime("%Y-%m-%dT%H:%M:%S%z") + df_train.data_item_start = df_train.data_item_start.dt.strftime( + "%Y-%m-%dT%H:%M:%S%z" + ) df_test.data_item_start = df_test.data_item_start.dt.strftime("%Y-%m-%dT%H:%M:%S%z") dataset = DatasetDict( { @@ -100,11 +101,11 @@ def generate_hf_dataset(df_train, df_test): "debunk_references", "claims", "explanations", - "other_comments" + "other_comments", ] -if __name__=="__main__": +if __name__ == "__main__": df = fetch_articles(DATASET_COLUMNS) df_train, df_test = split_df_by_week(df) dataset = generate_hf_dataset(df_train, df_test) - dataset.push_to_hub("DataForGood/climateguard-training", private=True, token=True) \ No newline at end of file + dataset.push_to_hub("DataForGood/climateguard-training", private=True, token=True) From b6dc27582e35d057741bda2fbcef183175510a24 Mon Sep 17 00:00:00 2001 From: gmguarino Date: Tue, 14 Apr 2026 14:34:24 +0200 Subject: [PATCH 3/4] update the script to better separate datasets --- data/update_huggingface_dataset.py | 32 +++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/data/update_huggingface_dataset.py b/data/update_huggingface_dataset.py index 8270efc..a899322 100644 --- a/data/update_huggingface_dataset.py +++ b/data/update_huggingface_dataset.py @@ -2,6 +2,7 @@ from datetime import date, datetime, timedelta from typing import List +import numpy as np import pandas as pd import psycopg import requests @@ -21,6 +22,7 @@ def fetch_articles(columns: List[str] = None) -> pd.DataFrame: query = f""" SELECT {columns_str} FROM analytics.task_global_completion + where task_completion_aggregate_id is not null """ with psycopg.connect(conninfo) as conn: with conn.cursor() as cur: @@ -42,11 +44,34 @@ def get_week_number(record): def split_df_by_week(df): - df["week_number"] = df.apply(get_week_number, axis=1) - df_train = df.loc[df.week_number % 4] - df_test = df.loc[df.week_number % 4 == 0] + df["week_number"] = df.data_item_start.dt.isocalendar().week + df_train = df.loc[df.week_number % 5 > 0] + df_test = df.loc[df.week_number % 5 == 0] return df_train, df_test +def cast_to_int_with_nan(col: pd.Series, int_type: str='uint8'): + col = col.fillna(-1) + col = col.astype(int_type) + col[col==-1] = np.nan + return col + + +def format_dtypes(df: pd.DataFrame): + df.data_item_day = df.data_item_day.astype('f2') + df.data_item_month = df.data_item_month.astype('f2') + df.data_item_year = df.data_item_year.astype('f2') + df.data_item_model_result = df.data_item_model_result.astype('f2') + df.mesinfo_correct = df.mesinfo_correct.astype('f2') + df.mesinfo_incorrect = df.mesinfo_incorrect.astype('f2') + df.speaker_journalist = df.speaker_journalist.astype('f2') + df.speaker_commentator = df.speaker_commentator.astype('f2') + df.speaker_guest = df.speaker_guest.astype('f2') + df.speaker_politician = df.speaker_politician.astype('f2') + df.speaker_audience = df.speaker_audience.astype('f2') + df.speaker_unknown = df.speaker_unknown.astype('f2') + df.mesinfo_corrected_bool = df.mesinfo_corrected_bool.astype('f2') + df.mesinfo_corrected = df.mesinfo_corrected.fillna("") + return df def generate_hf_dataset(df_train, df_test): df_train.data_item_start = df_train.data_item_start.dt.strftime( @@ -106,6 +131,7 @@ def generate_hf_dataset(df_train, df_test): if __name__ == "__main__": df = fetch_articles(DATASET_COLUMNS) + df = format_dtypes(df) df_train, df_test = split_df_by_week(df) dataset = generate_hf_dataset(df_train, df_test) dataset.push_to_hub("DataForGood/climateguard-training", private=True, token=True) From f53119c1d3e5159f5b2f3aaa6d4e300a710be0f7 Mon Sep 17 00:00:00 2001 From: gmguarino Date: Tue, 14 Apr 2026 14:36:08 +0200 Subject: [PATCH 4/4] update gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2b51c6e..b87dd4a 100644 --- a/.gitignore +++ b/.gitignore @@ -109,4 +109,6 @@ climateguard/experiments/claim_extraction/climateguard_claim_extraction .aws/ germany_s3_bucket/ france_s3_bucket/ -.gcloud \ No newline at end of file +.gcloud +.claude +CLAUDE.md \ No newline at end of file