From cf4d2238c5605f3eab223326f004e2d9fb96a2ee Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Fri, 10 Nov 2023 10:43:01 -0500 Subject: [PATCH 1/6] Make analytics django image more generic Since we'd also like to use this Django app for error taxonomy upload (and potentially, any other analytics-related things other than build timings in the future), I refactored this docker image by - Renaming it to have a more generic name (`ci-analytics`) - Removing the invocation of the build timing upload script from the Docker file, and instead specifying it in the job template. --- .github/workflows/custom_docker_builds.yml | 2 +- analytics/Dockerfile | 2 -- images/build-timing-processor/job-template.yaml | 3 ++- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/custom_docker_builds.yml b/.github/workflows/custom_docker_builds.yml index 768e27e4e..0512ccd56 100644 --- a/.github/workflows/custom_docker_builds.yml +++ b/.github/workflows/custom_docker_builds.yml @@ -48,7 +48,7 @@ jobs: - docker-image: ./images/build-timing-processor image-tags: ghcr.io/spack/build-timing-processor:0.0.4 - docker-image: ./analytics - image-tags: ghcr.io/spack/upload-build-timings:0.0.4 + image-tags: ghcr.io/spack/ci-analytics:0.0.1 steps: - name: Checkout uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 diff --git a/analytics/Dockerfile b/analytics/Dockerfile index 89fbd7255..6cbc286d2 100644 --- a/analytics/Dockerfile +++ b/analytics/Dockerfile @@ -8,5 +8,3 @@ RUN pip install --upgrade pip RUN pip install -r requirements.txt COPY . . - -CMD [ "./manage.py", "upload_build_timings" ] diff --git a/images/build-timing-processor/job-template.yaml b/images/build-timing-processor/job-template.yaml index 3c20c9b7b..e88ba18d7 100644 --- a/images/build-timing-processor/job-template.yaml +++ b/images/build-timing-processor/job-template.yaml @@ -16,7 +16,8 @@ spec: restartPolicy: OnFailure containers: - name: build-timing-processing-job - image: ghcr.io/spack/upload-build-timings:0.0.4 + image: ghcr.io/spack/ci-analytics:0.0.1 + command: [ "./manage.py", "upload_build_timings" ] imagePullPolicy: Always env: - name: GITLAB_TOKEN From 269d5aa1a1afe722a942832bb14f6975b822d3de Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Fri, 10 Nov 2023 10:35:50 -0500 Subject: [PATCH 2/6] Add error taxonomy script to analytics app --- .../analytics/management/commands/__init__.py | 0 .../management/commands}/taxonomy.yaml | 0 .../commands/upload_error_taxonomy.py | 144 ++++++++ .../tests/test_upload_error_taxonomy.py | 339 ++++++++++++++++++ analytics/requirements.txt | 3 + 5 files changed, 486 insertions(+) create mode 100644 analytics/analytics/management/commands/__init__.py rename {images/upload-gitlab-failure-logs => analytics/analytics/management/commands}/taxonomy.yaml (100%) create mode 100644 analytics/analytics/management/commands/upload_error_taxonomy.py create mode 100644 analytics/analytics/tests/test_upload_error_taxonomy.py diff --git a/analytics/analytics/management/commands/__init__.py b/analytics/analytics/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/images/upload-gitlab-failure-logs/taxonomy.yaml b/analytics/analytics/management/commands/taxonomy.yaml similarity index 100% rename from images/upload-gitlab-failure-logs/taxonomy.yaml rename to analytics/analytics/management/commands/taxonomy.yaml diff --git a/analytics/analytics/management/commands/upload_error_taxonomy.py b/analytics/analytics/management/commands/upload_error_taxonomy.py new file mode 100644 index 000000000..3f0fbe9dd --- /dev/null +++ b/analytics/analytics/management/commands/upload_error_taxonomy.py @@ -0,0 +1,144 @@ +import json +import os +import re +from datetime import datetime, timedelta +from pathlib import Path +from time import sleep +from typing import Any + +import djclick as click +import gitlab +import psycopg2 +import yaml + +from analytics.models import ErrorTaxonomy, Job + +GITLAB_TOKEN = os.environ["GITLAB_TOKEN"] +GITLAB_POSTGRES_DB = os.environ["GITLAB_POSTGRES_DB"] +GITLAB_POSTGRES_USER = os.environ["GITLAB_POSTGRES_RO_USER"] +GITLAB_POSTGRES_PASSWORD = os.environ["GITLAB_POSTGRES_RO_PASSWORD"] +GITLAB_POSTGRES_HOST = os.environ["GITLAB_POSTGRES_HOST"] + + +def job_retry_data(job_id: str | int, job_name: str) -> tuple[int, bool]: + # Instantiate postgres connection + pg_conn = psycopg2.connect( + host=GITLAB_POSTGRES_HOST, + port="5432", + dbname=GITLAB_POSTGRES_DB, + user=GITLAB_POSTGRES_USER, + password=GITLAB_POSTGRES_PASSWORD, + ) + with pg_conn: + cur = pg_conn.cursor() + cur.execute( + """ + SELECT attempt_number, COALESCE(retried, FALSE) as retried FROM ( + SELECT ROW_NUMBER() OVER (ORDER BY id) as attempt_number, retried, id + FROM ci_builds + WHERE + ci_builds.name = %(job_name)s + and ci_builds.stage_id = ( + SELECT stage_id from ci_builds WHERE id = %(job_id)s LIMIT 1 + ) + and ci_builds.status = 'failed' + ) as build_attempts + WHERE build_attempts.id = %(job_id)s + ; + """, + {"job_id": job_id, "job_name": job_name}, + ) + result = cur.fetchone() + cur.close() + + return result + + +def assign_error_taxonomy( + job_input_data: dict[str, Any], job_trace: str +) -> tuple[str, str]: + # Read taxonomy file + with open(Path(__file__).parent / "taxonomy.yaml") as f: + taxonomy = yaml.safe_load(f)["taxonomy"] + + # Compile matching patterns from job trace + matching_patterns = set() + for error_class, lookups in taxonomy["error_classes"].items(): + if lookups: + for grep_expr in lookups.get("grep_for", []): + if re.compile(grep_expr).search(job_trace): + matching_patterns.add(error_class) + + # If the job logs matched any regexes, assign it the taxonomy + # with the highest priority in the "deconflict order". + # Otherwise, assign it a taxonomy of "other". + job_error_class: str | None = None + if len(matching_patterns): + for error_class in taxonomy["deconflict_order"]: + if error_class in matching_patterns: + job_error_class = error_class + break + else: + job_error_class = "other" + + # If this job timed out or failed to be scheduled by GitLab, + # label it as such. + if job_input_data["build_failure_reason"] in ( + "stuck_or_timeout_failure", + "scheduler_failure", + ): + job_error_class = job_input_data["build_failure_reason"] + + return job_error_class, taxonomy["version"] + + +def get_job_trace(gl_project_id: int, gl_build_id: int) -> str: + # Instantiate gitlab api wrapper + gl = gitlab.Gitlab("https://gitlab.spack.io", GITLAB_TOKEN) + + # Retrieve project and job from gitlab API + project = gl.projects.get(gl_project_id) + job = project.jobs.get(gl_build_id) + return job.trace().decode() # type: ignore + + +@click.command() +def main(): + # Read input data and extract params + job_input_data: dict[str, Any] = json.loads(os.environ["JOB_INPUT_DATA"]) + project_id: int = job_input_data["project_id"] + job_id: int = job_input_data["build_id"] + job_name: str = job_input_data["build_name"] + + # Annotate if job has been retried + attempt_number, retried = job_retry_data(job_id=job_id, job_name=job_name) + + job_trace = get_job_trace(project_id, job_id) + + # Assign any/all relevant errors + error_taxonomy, error_taxonomy_version = assign_error_taxonomy( + job_input_data, job_trace + ) + + # Give the build timing hook some time to create the Job record, if needed. + # If we're waiting longer than 10 minutes, stop the loop and let an + # exception get raised (which Sentry will catch) + start = datetime.now() + while ( + not Job.objects.filter(job_id=job_id).exists() + and start + timedelta(minutes=10) > datetime.now() + ): + sleep(2) + + ErrorTaxonomy.objects.create( + job_id=job_id, + attempt_number=attempt_number, + retried=retried, + error_taxonomy=error_taxonomy, + error_taxonomy_version=error_taxonomy_version, + webhook_payload=job_input_data, + ) + + +if __name__ == "__main__": + main() diff --git a/analytics/analytics/tests/test_upload_error_taxonomy.py b/analytics/analytics/tests/test_upload_error_taxonomy.py new file mode 100644 index 000000000..b00d17220 --- /dev/null +++ b/analytics/analytics/tests/test_upload_error_taxonomy.py @@ -0,0 +1,339 @@ +import json +import random +import threading +from datetime import datetime +from typing import Any + +import pytest + +from analytics.models import ErrorTaxonomy, Job + +JOB_TRACES = { + 8826259: """ + ERROR: No files to upload + Cleaning up project directory and file based variables + 00:00 + ERROR: Error cleaning up pod: pods "runner-eu8s7xzf-project-2-concurrent-0-mu17vi7t" not found + ERROR: Job failed (system failure): error sending request: Post "https://10.100.0.1:443/api/v1/namespaces/pipeline/pods/runner-eu8s7xzf-project-2-concurrent-0-mu17vi7t/attach?container=build&stdin=true": EOF + """, + 8825911: """ + To reproduce this build locally, run: + spack ci reproduce-build https://gitlab.spack.io/api/v4/projects/2/jobs/8825911/artifacts [--working-dir ] [--autostart] + If this project does not have public pipelines, you will need to first: + export GITLAB_PRIVATE_TOKEN= + ... then follow the printed instructions. + Running after_script + 00:01 + Running after script... + $ cat /proc/loadavg || true + 1.08 4.31 8.79 1/748 1261 + Uploading artifacts for failed job + 00:01 + Uploading artifacts... + jobs_scratch_dir/logs: found 1 matching artifact files and directories + jobs_scratch_dir/reproduction: found 8 matching artifact files and directories + jobs_scratch_dir/tests: found 2 matching artifact files and directories + jobs_scratch_dir/user_data: found 3 matching artifact files and directories + Uploading artifacts as "archive" to coordinator... 201 Created id=8825911 responseStatus=201 Created token=64_Cxyrv + Cleaning up project directory and file based variables + 00:01 + ERROR: Job failed: command terminated with exit code 1 + """, + 9043674: """ + RuntimeError: concretization failed for the following reasons: + 1. acts: '%gcc@:7' conflicts with '@0.23:' + 2. acts: '%gcc@:7' conflicts with '@0.23:' + required because conflict applies to spec @0.23: + required because acts+analysis+binaries+dd4hep+edm4hep+examples+fatras+geant4+hepmc3+identification+podio+pythia8+python+tgeo requested from CLI + required because conflict is triggered when %gcc@:7 + required because acts+analysis+binaries+dd4hep+edm4hep+examples+fatras+geant4+hepmc3+identification+podio+pythia8+python+tgeo requested from CLI + Running after_script + 00:00 + Running after script... + $ cat /proc/loadavg || true + 13.58 10.67 7.56 1/1785 12 + Cleaning up project directory and file based variables + 00:00 + ERROR: Job failed: exit code 1 + """, +} + + +@pytest.fixture() +def mock_gitlab(monkeypatch: pytest.MonkeyPatch): + # Mock env vars + IGNORED_ENV_VARS = ( + "GITLAB_TOKEN", + "GITLAB_POSTGRES_DB", + "GITLAB_POSTGRES_RO_USER", + "GITLAB_POSTGRES_RO_PASSWORD", + "GITLAB_POSTGRES_HOST", + ) + for env_var in IGNORED_ENV_VARS: + monkeypatch.setenv(env_var, "foobar") + + # Now that env vars are properly mocked, we can import this module + from analytics.management.commands import upload_error_taxonomy + + # Mock calls to gitlab API and database + monkeypatch.setattr( + upload_error_taxonomy, + "get_job_trace", + lambda project_id, job_id: JOB_TRACES[job_id], + ) + monkeypatch.setattr( + upload_error_taxonomy, "job_retry_data", lambda *args, **kwargs: (1, 0) + ) + + +@pytest.mark.parametrize( + ["job_webhook_payload", "expected_taxonomy"], + [ + ( + { + "object_kind": "build", + "ref": "develop", + "tag": False, + "before_sha": "a6466b9dddf59cc185800ac428bd4ba535b96c2e", + "sha": "a452e8379e12bd46925df30b99cf4b30edf80457", + "retries_count": 1, + "build_id": 8825911, + "build_name": "visit@3.3.3 /jbgulft %gcc@11.1.0 arch=linux-ubuntu20.04-x86_64_v3 Data and Vis SDK", + "build_stage": "stage-20", + "build_status": "failed", + "build_created_at": "2023-10-23 15:09:56 UTC", + "build_started_at": "2023-10-23 17:55:04 UTC", + "build_finished_at": "2023-10-23 17:58:32 UTC", + "build_duration": 208.425831, + "build_queued_duration": 10.771205, + "build_allow_failure": False, + "build_failure_reason": "script_failure", + "pipeline_id": 529203, + "runner": { + "id": 19462, + "description": "runner-x86-v4-prot-gitlab-runner-79b97d7974-2hj5n", + "runner_type": "instance_type", + "active": True, + "is_shared": True, + "tags": [ + "x86_64", + "x86_64_v2", + "x86_64_v3", + "x86_64_v4", + "avx", + "avx2", + "avx512", + "small", + "medium", + "large", + "huge", + "protected", + "aws", + "spack", + ], + }, + "project_id": 2, + "project_name": "spack / spack", + "user": { + "id": 3, + "name": "Spack Bot", + "username": "spackbot", + "avatar_url": "https://gitlab.spack.io/uploads/-/system/user/avatar/3/57643519.png", + "email": "[REDACTED]", + }, + "commit": { + "id": 529203, + "name": None, + "sha": "a452e8379e12bd46925df30b99cf4b30edf80457", + "message": "nghttp2: add v1.57.0 (#40652)\n\n", + "author_name": "Harmen Stoppels", + "author_email": "[REDACTED]", + "author_url": "mailto:me@harmenstoppels.nl", + "status": "running", + "duration": None, + "started_at": "2023-10-23 15:10:25 UTC", + "finished_at": None, + }, + "repository": { + "name": "spack", + "url": "git@ssh.gitlab.spack.io:spack/spack.git", + "description": "", + "homepage": "https://gitlab.spack.io/spack/spack", + "git_http_url": "https://gitlab.spack.io/spack/spack.git", + "git_ssh_url": "git@ssh.gitlab.spack.io:spack/spack.git", + "visibility_level": 20, + }, + "project": { + "id": 2, + "name": "spack", + "description": "", + "web_url": "https://gitlab.spack.io/spack/spack", + "avatar_url": None, + "git_ssh_url": "git@ssh.gitlab.spack.io:spack/spack.git", + "git_http_url": "https://gitlab.spack.io/spack/spack.git", + "namespace": "spack", + "visibility_level": 20, + "path_with_namespace": "spack/spack", + "default_branch": "develop", + "ci_config_path": "share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml", + }, + "environment": None, + "source_pipeline": { + "project": { + "id": 2, + "web_url": "https://gitlab.spack.io/spack/spack", + "path_with_namespace": "spack/spack", + }, + "job_id": 8825079, + "pipeline_id": 529171, + }, + }, + "spack_error", + ), + ( + { + "object_kind": "build", + "ref": "pr40932_hep-cloud-pipeline", + "tag": False, + "before_sha": "0000000000000000000000000000000000000000", + "sha": "a0d0305f2ff9a9f375010aaa79336d8241b9ab0f", + "retries_count": 1, + "build_id": 9043674, + "build_name": "hep-generate", + "build_stage": "generate", + "build_status": "failed", + "build_created_at": "2023-11-10 14:43:57 UTC", + "build_started_at": "2023-11-10 14:44:08 UTC", + "build_finished_at": "2023-11-10 14:47:28 UTC", + "build_duration": 200.08593, + "build_queued_duration": 0.261321, + "build_allow_failure": False, + "build_failure_reason": "script_failure", + "pipeline_id": 547935, + "runner": { + "id": 16964, + "description": "uo-voltar-cpu-small-medium-1", + "runner_type": "instance_type", + "active": True, + "is_shared": True, + "tags": [ + "avx", + "public", + "spack", + "medium", + "avx2", + "avx512", + "small", + "uo", + "x86_64", + "voltar", + "x86_64_v3", + "x86_64_v4", + "x86_64_v2", + "docker", + "cpu", + "service", + ], + }, + "project_id": 2, + "project_name": "spack / spack", + "user": { + "id": 3, + "name": "Spack Bot", + "username": "spackbot", + "avatar_url": "https://gitlab.spack.io/uploads/-/system/user/avatar/3/57643519.png", + "email": "[REDACTED]", + }, + "commit": { + "id": 547935, + "name": None, + "sha": "a0d0305f2ff9a9f375010aaa79336d8241b9ab0f", + "message": "Merge 78c2eb5afb669bd28c52ae68d13ef49f95432227 into 4027a2139b053251dafc2de38d24eac4d69d42a0\n", + "author_name": "spackbot", + "author_email": "[REDACTED]", + "author_url": "mailto:noreply@spack.io", + "status": "running", + "duration": None, + "started_at": "2023-11-10 14:44:09 UTC", + "finished_at": None, + }, + "repository": { + "name": "spack", + "url": "git@ssh.gitlab.spack.io:spack/spack.git", + "description": "", + "homepage": "https://gitlab.spack.io/spack/spack", + "git_http_url": "https://gitlab.spack.io/spack/spack.git", + "git_ssh_url": "git@ssh.gitlab.spack.io:spack/spack.git", + "visibility_level": 20, + }, + "project": { + "id": 2, + "name": "spack", + "description": "", + "web_url": "https://gitlab.spack.io/spack/spack", + "avatar_url": None, + "git_ssh_url": "git@ssh.gitlab.spack.io:spack/spack.git", + "git_http_url": "https://gitlab.spack.io/spack/spack.git", + "namespace": "spack", + "visibility_level": 20, + "path_with_namespace": "spack/spack", + "default_branch": "develop", + "ci_config_path": "share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml", + }, + "environment": None, + }, + "concretization_error", + ), + # TODO: this test case fails. + # ( + # { + # "before_sha": "a6466b9dddf59cc185800ac428bd4ba535b96c2e", + # "build_id": 8826259, + # "build_name": "py-scipy@1.5.4 /mscpuhz %gcc@11.3.0 arch=linux-ubuntu22.04-x86_64_v3 Machine Learning", + # "build_stage": "stage-13", + # "build_status": "failed", + # "object_kind": "build", + # "ref": "develop", + # "retries_count": 1, + # "sha": "a452e8379e12bd46925df30b99cf4b30edf80457", + # "tag": False, + # }, + # "other", + # ), + ], +) +@pytest.mark.django_db() +def test_upload_gitlab_failure_logs( + job_webhook_payload: dict[str, Any], + expected_taxonomy: str, + monkeypatch: pytest.MonkeyPatch, + mock_gitlab, +): + from analytics.management.commands import upload_error_taxonomy + + monkeypatch.setenv("JOB_INPUT_DATA", json.dumps(job_webhook_payload)) + + def _create_job(): + return Job.objects.create( + job_id=job_webhook_payload["build_id"], + project_id=1, + name="test", + started_at=datetime.now(), + duration=200, + ref="develop", + tags=["test"], + package_name="test", + ) + + # Intentionally delay creation of a Job record until after the call to + # the error taxonomy script. This tests a potential race condition where + # the error processing job executes before the build timing job. + threading.Timer(random.randint(5, 10), _create_job).start() + + upload_error_taxonomy.main() + + error_taxonomy_record = ErrorTaxonomy.objects.get( + job_id=job_webhook_payload["build_id"] + ) + + assert error_taxonomy_record.error_taxonomy == expected_taxonomy diff --git a/analytics/requirements.txt b/analytics/requirements.txt index bdde77838..d77236552 100644 --- a/analytics/requirements.txt +++ b/analytics/requirements.txt @@ -3,5 +3,8 @@ django-click==2.3.0 django-extensions==3.2.3 kubernetes==26.1.0 python-gitlab==3.11.0 +PyYAML==6.0 psycopg2-binary==2.9.5 +pytest +pytest-django sentry-sdk[django] From 42844d30705ae6fb32a5449c156cbfa1a5f515fc Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Fri, 10 Nov 2023 10:46:06 -0500 Subject: [PATCH 3/6] Add new job template for analytics db error processor I kept the old one there for now instead of deleting it. Once we confirm that the new analytics DB-based workflow works, I'll remove it. --- images/gitlab-error-processor/app.py | 49 +++++++------- .../job-template-old.yaml | 64 +++++++++++++++++++ .../gitlab-error-processor/job-template.yaml | 20 +----- 3 files changed, 91 insertions(+), 42 deletions(-) create mode 100644 images/gitlab-error-processor/job-template-old.yaml diff --git a/images/gitlab-error-processor/app.py b/images/gitlab-error-processor/app.py index f5219fc11..52129b7e9 100644 --- a/images/gitlab-error-processor/app.py +++ b/images/gitlab-error-processor/app.py @@ -32,30 +32,31 @@ async def gitlab_webhook_consumer(request: Request): if job_input_data["build_status"] != "failed": return Response("Not a failed job, no action needed.", status_code=200) - with open(Path(__file__).parent / "job-template.yaml") as f: - job_template = yaml.safe_load(f) - - for container in job_template["spec"]["template"]["spec"]["containers"]: - container.setdefault("env", []).extend( - [dict(name="JOB_INPUT_DATA", value=json.dumps(job_input_data))] + for template in ("job-template.yaml", "job-template-old.yaml"): + with open(Path(__file__).parent / template) as f: + job_template = yaml.safe_load(f) + + for container in job_template["spec"]["template"]["spec"]["containers"]: + container.setdefault("env", []).extend( + [dict(name="JOB_INPUT_DATA", value=json.dumps(job_input_data))] + ) + + job_build_id = str(job_input_data["build_id"]) + job_pipeline_id = str(job_input_data["pipeline_id"]) + + job_template["metadata"][ + "name" + ] = f"gitlab-error-processing-job-{job_build_id}-{job_pipeline_id}" + + # Add labels to make finding the job that proccessed the error log easier. + job_template["metadata"]["labels"] = { + "spack.io/gitlab-build-id": job_build_id, + "spack.io/gitlab-pipeline-id": job_pipeline_id, + } + + batch.create_namespaced_job( + "custom", + job_template, ) - job_build_id = str(job_input_data["build_id"]) - job_pipeline_id = str(job_input_data["pipeline_id"]) - - job_template["metadata"][ - "name" - ] = f"gitlab-error-processing-job-{job_build_id}-{job_pipeline_id}" - - # Add labels to make finding the job that proccessed the error log easier. - job_template["metadata"]["labels"] = { - "spack.io/gitlab-build-id": job_build_id, - "spack.io/gitlab-pipeline-id": job_pipeline_id, - } - - batch.create_namespaced_job( - "custom", - job_template, - ) - return Response("Upload job dispatched.", status_code=202) diff --git a/images/gitlab-error-processor/job-template-old.yaml b/images/gitlab-error-processor/job-template-old.yaml new file mode 100644 index 000000000..f46fc8906 --- /dev/null +++ b/images/gitlab-error-processor/job-template-old.yaml @@ -0,0 +1,64 @@ +--- +# TODO: remove this job when the analytics DB is used. +apiVersion: batch/v1 +kind: Job +metadata: + name: gitlab-error-processing-job-opensearch + labels: + app: gitlab-error-processing-job-opensearch +spec: + ttlSecondsAfterFinished: 7200 + template: + metadata: + labels: + app: gitlab-error-processing-job + spec: + restartPolicy: OnFailure + containers: + - name: gitlab-error-processing-job + image: ghcr.io/spack/upload-gitlab-failure-logs:0.0.3 + imagePullPolicy: Always + env: + - name: GITLAB_TOKEN + valueFrom: + secretKeyRef: + name: gitlab-error-processor + key: gitlab-token + + # postgres credentials + - name: GITLAB_POSTGRES_HOST + valueFrom: + secretKeyRef: + name: gitlab-error-processor + key: postgresql-gitlab-host + - name: GITLAB_POSTGRES_DB + value: gitlabhq_production + - name: GITLAB_POSTGRES_RO_USER + value: gitlab_ro_user + - name: GITLAB_POSTGRES_RO_PASSWORD + valueFrom: + secretKeyRef: + name: gitlab-error-processor + key: postgresql-gitlab-ro-user-password + + # opensearch credentials + - name: OPENSEARCH_ENDPOINT + valueFrom: + secretKeyRef: + name: gitlab-error-processor + key: opensearch-endpoint + - name: OPENSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: gitlab-error-processor + key: opensearch-username + - name: OPENSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: gitlab-error-processor + key: opensearch-password + envFrom: + - configMapRef: + name: python-scripts-sentry-config + nodeSelector: + spack.io/node-pool: base diff --git a/images/gitlab-error-processor/job-template.yaml b/images/gitlab-error-processor/job-template.yaml index f4bfad573..4124eef75 100644 --- a/images/gitlab-error-processor/job-template.yaml +++ b/images/gitlab-error-processor/job-template.yaml @@ -15,7 +15,8 @@ spec: restartPolicy: OnFailure containers: - name: gitlab-error-processing-job - image: ghcr.io/spack/upload-gitlab-failure-logs:0.0.3 + image: ghcr.io/spack/ci-analytics:0.0.1 + command: [ "./manage.py", "upload_error_taxonomy" ] imagePullPolicy: Always env: - name: GITLAB_TOKEN @@ -39,23 +40,6 @@ spec: secretKeyRef: name: gitlab-error-processor key: postgresql-gitlab-ro-user-password - - # opensearch credentials - - name: OPENSEARCH_ENDPOINT - valueFrom: - secretKeyRef: - name: gitlab-error-processor - key: opensearch-endpoint - - name: OPENSEARCH_USERNAME - valueFrom: - secretKeyRef: - name: gitlab-error-processor - key: opensearch-username - - name: OPENSEARCH_PASSWORD - valueFrom: - secretKeyRef: - name: gitlab-error-processor - key: opensearch-password envFrom: - configMapRef: name: python-scripts-sentry-config From faf8b3d063c26340438972cb445cdf6d3527ec66 Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Fri, 10 Nov 2023 10:47:56 -0500 Subject: [PATCH 4/6] Bump job processor images Both of their job-template.yaml files got updated, so we need to bump these as well. --- .github/workflows/custom_docker_builds.yml | 4 ++-- k8s/production/custom/build-timing-processor/deployments.yaml | 2 +- k8s/production/custom/gitlab-error-processor/deployments.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/custom_docker_builds.yml b/.github/workflows/custom_docker_builds.yml index 0512ccd56..fcd9444a4 100644 --- a/.github/workflows/custom_docker_builds.yml +++ b/.github/workflows/custom_docker_builds.yml @@ -38,7 +38,7 @@ jobs: - docker-image: ./images/python-aws-bash image-tags: ghcr.io/spack/python-aws-bash:0.0.2 - docker-image: ./images/gitlab-error-processor - image-tags: ghcr.io/spack/gitlab-error-processor:0.0.3 + image-tags: ghcr.io/spack/gitlab-error-processor:0.0.4 - docker-image: ./images/upload-gitlab-failure-logs image-tags: ghcr.io/spack/upload-gitlab-failure-logs:0.0.3 - docker-image: ./images/snapshot-release-tags @@ -46,7 +46,7 @@ jobs: - docker-image: ./images/cache-indexer image-tags: ghcr.io/spack/cache-indexer:0.0.3 - docker-image: ./images/build-timing-processor - image-tags: ghcr.io/spack/build-timing-processor:0.0.4 + image-tags: ghcr.io/spack/build-timing-processor:0.0.5 - docker-image: ./analytics image-tags: ghcr.io/spack/ci-analytics:0.0.1 steps: diff --git a/k8s/production/custom/build-timing-processor/deployments.yaml b/k8s/production/custom/build-timing-processor/deployments.yaml index dff00184d..97be89bcb 100644 --- a/k8s/production/custom/build-timing-processor/deployments.yaml +++ b/k8s/production/custom/build-timing-processor/deployments.yaml @@ -23,7 +23,7 @@ spec: serviceAccountName: build-timing-processor containers: - name: build-timing-processor - image: ghcr.io/spack/build-timing-processor:0.0.4 + image: ghcr.io/spack/build-timing-processor:0.0.5 imagePullPolicy: Always resources: requests: diff --git a/k8s/production/custom/gitlab-error-processor/deployments.yaml b/k8s/production/custom/gitlab-error-processor/deployments.yaml index ed0baa5fe..d2a68e783 100644 --- a/k8s/production/custom/gitlab-error-processor/deployments.yaml +++ b/k8s/production/custom/gitlab-error-processor/deployments.yaml @@ -23,7 +23,7 @@ spec: serviceAccountName: gitlab-error-processor containers: - name: gitlab-error-processor - image: ghcr.io/spack/gitlab-error-processor:0.0.3 + image: ghcr.io/spack/gitlab-error-processor:0.0.4 imagePullPolicy: Always envFrom: - configMapRef: From 156f91ef79840050b2bcbde51805359ddf54a2a7 Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Tue, 14 Nov 2023 11:11:09 -0500 Subject: [PATCH 5/6] Run analytics tests in github actions --- .github/workflows/analytics_ci.yml | 35 ++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/analytics_ci.yml diff --git a/.github/workflows/analytics_ci.yml b/.github/workflows/analytics_ci.yml new file mode 100644 index 000000000..97f15cbc8 --- /dev/null +++ b/.github/workflows/analytics_ci.yml @@ -0,0 +1,35 @@ +name: Analytics CI + +on: + pull_request: + paths: + - "analytics/**" + +jobs: + test: + runs-on: ubuntu-latest + defaults: + run: + working-directory: analytics/ + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Start background services + run: docker compose up -d + + - name: Install requirements + run: pip install -r requirements.txt + + - name: Provide default env vars for django + run: cat dev/.env.docker-compose >> $GITHUB_ENV + + - name: Run tests + run: pytest analytics/tests + env: + DJANGO_SETTINGS_MODULE: settings From e42e1f6b03c106cadafdd4278e4632c6e1142992 Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Wed, 15 Nov 2023 16:02:48 -0500 Subject: [PATCH 6/6] Make job id an int instead of foreign key Failed jobs don't have a `Job` record saved. --- .../migrations/0004_errortaxonomy.py | 20 +++---------------- analytics/analytics/models.py | 5 ++--- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/analytics/analytics/migrations/0004_errortaxonomy.py b/analytics/analytics/migrations/0004_errortaxonomy.py index 3d6ec6269..a9ebd90c9 100644 --- a/analytics/analytics/migrations/0004_errortaxonomy.py +++ b/analytics/analytics/migrations/0004_errortaxonomy.py @@ -1,6 +1,5 @@ -# Generated by Django 4.2.4 on 2023-11-09 21:07 +# Generated by Django 4.2.4 on 2023-11-15 21:02 -import django.db.models.deletion from django.db import migrations, models @@ -14,13 +13,8 @@ class Migration(migrations.Migration): name="ErrorTaxonomy", fields=[ ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), + "job_id", + models.PositiveBigIntegerField(primary_key=True, serialize=False), ), ("created", models.DateTimeField(auto_now_add=True)), ("attempt_number", models.PositiveSmallIntegerField()), @@ -33,14 +27,6 @@ class Migration(migrations.Migration): help_text="The JSON payload received from the GitLab job webhook." ), ), - ( - "job", - models.OneToOneField( - on_delete=django.db.models.deletion.CASCADE, - related_name="error_taxonomy", - to="analytics.job", - ), - ), ], ), ] diff --git a/analytics/analytics/models.py b/analytics/analytics/models.py index b46f6dff2..425971966 100644 --- a/analytics/analytics/models.py +++ b/analytics/analytics/models.py @@ -69,10 +69,9 @@ class Meta: class ErrorTaxonomy(models.Model): + job_id = models.PositiveBigIntegerField(primary_key=True) + created = models.DateTimeField(auto_now_add=True) - job = models.OneToOneField( - Job, related_name="error_taxonomy", on_delete=models.CASCADE - ) attempt_number = models.PositiveSmallIntegerField() retried = models.BooleanField()