diff --git a/.github/workflows/custom_docker_builds.yml b/.github/workflows/custom_docker_builds.yml index 38d9f7b24..bead7c7af 100644 --- a/.github/workflows/custom_docker_builds.yml +++ b/.github/workflows/custom_docker_builds.yml @@ -35,16 +35,12 @@ jobs: image-tags: ghcr.io/spack/notary:latest - docker-image: ./images/python-aws-bash image-tags: ghcr.io/spack/python-aws-bash:0.0.2 - - docker-image: ./images/gitlab-error-processor - image-tags: ghcr.io/spack/gitlab-error-processor:0.0.6 - - docker-image: ./images/upload-gitlab-failure-logs - image-tags: ghcr.io/spack/upload-gitlab-failure-logs:0.0.6 - docker-image: ./images/snapshot-release-tags image-tags: ghcr.io/spack/snapshot-release-tags:0.0.4 - docker-image: ./images/cache-indexer image-tags: ghcr.io/spack/cache-indexer:0.0.3 - docker-image: ./analytics - image-tags: ghcr.io/spack/django:0.1.3 + image-tags: ghcr.io/spack/django:0.1.4 steps: - name: Checkout uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 diff --git a/images/upload-gitlab-failure-logs/upload_gitlab_failure_logs.py b/analytics/analytics/core/job_failure_classifier/__init__.py similarity index 61% rename from images/upload-gitlab-failure-logs/upload_gitlab_failure_logs.py rename to analytics/analytics/core/job_failure_classifier/__init__.py index 8a5562338..01950fb94 100644 --- a/images/upload-gitlab-failure-logs/upload_gitlab_failure_logs.py +++ b/analytics/analytics/core/job_failure_classifier/__init__.py @@ -1,33 +1,19 @@ -import json -import os -import re from datetime import datetime +import json from pathlib import Path -from time import sleep +import re from typing import Any +from celery import shared_task +from django.conf import settings +from django.db import connections import gitlab -import psycopg2 -import sentry_sdk +import opensearch_dsl import yaml -from kubernetes import client, config -from kubernetes.client.exceptions import ApiException -from kubernetes.client.models.v1_pod import V1Pod -from kubernetes.client.models.v1_pod_status import V1PodStatus -from opensearch_dsl import Date, Document, connections - -sentry_sdk.init( - # Sample only 1% of jobs - traces_sample_rate=0.01, -) -config.load_config() -v1_client = client.CoreV1Api() - - -class JobPayload(Document): - timestamp = Date() +class JobPayload(opensearch_dsl.Document): + timestamp = opensearch_dsl.Date() class Index: name = "gitlab-job-failures-*" @@ -42,34 +28,9 @@ def save(self, **kwargs): return super().save(**kwargs) -GITLAB_TOKEN = os.environ["GITLAB_TOKEN"] -GITLAB_POSTGRES_DB = os.environ["GITLAB_POSTGRES_DB"] -GITLAB_POSTGRES_USER = os.environ["GITLAB_POSTGRES_RO_USER"] -GITLAB_POSTGRES_PASSWORD = os.environ["GITLAB_POSTGRES_RO_PASSWORD"] -GITLAB_POSTGRES_HOST = os.environ["GITLAB_POSTGRES_HOST"] - -OPENSEARCH_ENDPOINT = os.environ["OPENSEARCH_ENDPOINT"] -OPENSEARCH_USERNAME = os.environ["OPENSEARCH_USERNAME"] -OPENSEARCH_PASSWORD = os.environ["OPENSEARCH_PASSWORD"] - - -# Instantiate gitlab api wrapper -gl = gitlab.Gitlab("https://gitlab.spack.io", GITLAB_TOKEN, retry_transient_errors=True) - -# Instantiate postgres connection -pg_conn = psycopg2.connect( - host=GITLAB_POSTGRES_HOST, - port="5432", - dbname=GITLAB_POSTGRES_DB, - user=GITLAB_POSTGRES_USER, - password=GITLAB_POSTGRES_PASSWORD, -) - - -def job_retry_data(job_id: str | int, job_name: str) -> tuple[int, bool]: - with pg_conn: - cur = pg_conn.cursor() - cur.execute( +def _job_retry_data(job_id: str | int, job_name: str) -> tuple[int, bool]: + with connections["gitlab"].cursor() as cursor: + cursor.execute( """ SELECT attempt_number, COALESCE(retried, FALSE) as retried FROM ( SELECT ROW_NUMBER() OVER (ORDER BY id) as attempt_number, retried, id @@ -86,13 +47,13 @@ def job_retry_data(job_id: str | int, job_name: str) -> tuple[int, bool]: """, {"job_id": job_id, "job_name": job_name}, ) - result = cur.fetchone() - cur.close() + result = cursor.fetchone() + cursor.close() return result -def assign_error_taxonomy(job_input_data: dict[str, Any], job_trace: str): +def _assign_error_taxonomy(job_input_data: dict[str, Any], job_trace: str): # Read taxonomy file with open(Path(__file__).parent / "taxonomy.yaml") as f: taxonomy = yaml.safe_load(f)["taxonomy"] @@ -130,7 +91,7 @@ def assign_error_taxonomy(job_input_data: dict[str, Any], job_trace: str): return -def collect_pod_status(job_input_data: dict[str, Any], job_trace: str): +def _collect_pod_status(job_input_data: dict[str, Any], job_trace: str): """Collect k8s info about this job and store it in the OpenSearch record""" # Record whether this job was run on a kubernetes pod or via some other # means (a UO runner, for example) @@ -156,37 +117,18 @@ def collect_pod_status(job_input_data: dict[str, Any], job_trace: str): job_input_data["pod_status"] = None return - pod_name = runner_name_matches[0] - - pod: V1Pod | None = None - while True: - # Try to fetch pod with kube - try: - pod = v1_client.read_namespaced_pod(name=pod_name, namespace="pipeline") - except ApiException: - # If it doesn't work, that means the pod has already been cleaned up. - # In that case, we break out of the loop and return. - break - - # Check if the pod is still running. If so, keep re-fetching it until it's complete - status: V1PodStatus = pod.status - if status.phase != "Running": - break - - sleep(1) - - if pod: - job_input_data["pod_status"] = pod.status.to_dict() +@shared_task(name="upload_job_failure_classification", soft_time_limit=60) +def upload_job_failure_classification(job_input_data_json: str) -> None: + gl = gitlab.Gitlab(settings.GITLAB_ENDPOINT, settings.GITLAB_TOKEN, retry_transient_errors=True) -def main(): # Read input data and extract params - job_input_data = json.loads(os.environ["JOB_INPUT_DATA"]) + job_input_data = json.loads(job_input_data_json) job_id = job_input_data["build_id"] job_name = job_input_data["build_name"] # Annotate if job has been retried - attempt_number, retried = job_retry_data(job_id=job_id, job_name=job_name) + attempt_number, retried = _job_retry_data(job_id=job_id, job_name=job_name) job_input_data["attempt_number"] = attempt_number job_input_data["retried"] = retried @@ -204,22 +146,17 @@ def main(): job_trace: str = job.trace().decode() # type: ignore # Get info about the k8s pod this job ran on - collect_pod_status(job_input_data, job_trace) + _collect_pod_status(job_input_data, job_trace) # Assign any/all relevant errors - assign_error_taxonomy(job_input_data, job_trace) + _assign_error_taxonomy(job_input_data, job_trace) - # Upload to OpenSearch - connections.create_connection( - hosts=[OPENSEARCH_ENDPOINT], + opensearch_dsl.connections.create_connection( + hosts=[settings.OPENSEARCH_ENDPOINT], http_auth=( - OPENSEARCH_USERNAME, - OPENSEARCH_PASSWORD, + settings.OPENSEARCH_USERNAME, + settings.OPENSEARCH_PASSWORD, ), ) doc = JobPayload(**job_input_data) doc.save() - - -if __name__ == "__main__": - main() diff --git a/images/upload-gitlab-failure-logs/taxonomy.yaml b/analytics/analytics/core/job_failure_classifier/taxonomy.yaml similarity index 100% rename from images/upload-gitlab-failure-logs/taxonomy.yaml rename to analytics/analytics/core/job_failure_classifier/taxonomy.yaml diff --git a/analytics/analytics/core/views.py b/analytics/analytics/core/views.py index a852554a4..76743c330 100644 --- a/analytics/analytics/core/views.py +++ b/analytics/analytics/core/views.py @@ -7,6 +7,7 @@ from django.views.decorators.http import require_http_methods import sentry_sdk +from analytics.core.job_failure_classifier import upload_job_failure_classification from analytics.core.job_log_uploader import upload_job_log from analytics.job_processor import process_job @@ -25,6 +26,9 @@ def webhook_handler(request: HttpRequest) -> HttpResponse: if job_input_data["build_status"] in ["success", "failed"]: upload_job_log.delay(request.body) + if job_input_data["build_status"] == "failed": + upload_job_failure_classification.delay(request.body) + if ( re.match(BUILD_STAGE_REGEX, job_input_data["build_stage"]) and job_input_data["build_status"] == "success" diff --git a/analytics/analytics/settings/base.py b/analytics/analytics/settings/base.py index 71163d240..3866f48e0 100644 --- a/analytics/analytics/settings/base.py +++ b/analytics/analytics/settings/base.py @@ -33,6 +33,18 @@ "django.middleware.clickjacking.XFrameOptionsMiddleware", ] +# Databases +DATABASES |= { + "gitlab": { + "ENGINE": "django.db.backends.postgresql", + "USER": os.environ["GITLAB_DB_USER"], + "HOST": os.environ["GITLAB_DB_HOST"], + "NAME": os.environ["GITLAB_DB_NAME"], + "PASSWORD": os.environ["GITLAB_DB_PASS"], + "PORT": os.environ["GITLAB_DB_PORT"], + }, +} + # django-extensions RUNSERVER_PLUS_PRINT_SQL_TRUNCATE = None SHELL_PLUS_PRINT_SQL = True diff --git a/analytics/dev/.env.docker-compose b/analytics/dev/.env.docker-compose index 6d92933d7..c14ff7ef4 100644 --- a/analytics/dev/.env.docker-compose +++ b/analytics/dev/.env.docker-compose @@ -10,4 +10,9 @@ OPENSEARCH_PASSWORD=elastic SECRET_KEY=deadbeef GITLAB_ENDPOINT="http://fakeurl" GITLAB_TOKEN="bar" +GITLAB_DB_USER=gitlab +GITLAB_DB_HOST=gitlab-db +GITLAB_DB_PORT=5432 +GITLAB_DB_NAME=gitlabhq_production +GITLAB_DB_PASS=gitlab PROMETHEUS_URL=http://prometheus:9090 diff --git a/analytics/dev/.env.docker-compose-native b/analytics/dev/.env.docker-compose-native index 179ec6e4b..29aea260f 100644 --- a/analytics/dev/.env.docker-compose-native +++ b/analytics/dev/.env.docker-compose-native @@ -9,4 +9,9 @@ OPENSEARCH_USERNAME=elastic OPENSEARCH_PASSWORD=elastic GITLAB_ENDPOINT=http://fakegitlab GITLAB_TOKEN=glpat-fakegitlab +GITLAB_DB_USER=gitlab +GITLAB_DB_HOST=localhost +GITLAB_DB_PORT=5433 +GITLAB_DB_NAME=gitlabhq_production +GITLAB_DB_PASS=gitlab PROMETHEUS_URL=http://localhost:9090 diff --git a/analytics/docker-compose.yml b/analytics/docker-compose.yml index 07138c0b4..ddb93934c 100644 --- a/analytics/docker-compose.yml +++ b/analytics/docker-compose.yml @@ -31,6 +31,17 @@ services: ports: - ${DOCKER_PROMETHEUS_PORT-9090}:9090 + gitlab-db: + image: postgres:latest + ports: + - "5433:5432" + volumes: + - gitlab-db:/var/lib/postgresql/data + environment: + POSTGRES_DB: gitlabhq_production + POSTGRES_USER: gitlab + POSTGRES_PASSWORD: gitlab + volumes: postgres: - + gitlab-db: diff --git a/images/gitlab-error-processor/Dockerfile b/images/gitlab-error-processor/Dockerfile deleted file mode 100644 index 4b4b7c06c..000000000 --- a/images/gitlab-error-processor/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM python:3.11-slim - -WORKDIR /app - -COPY requirements.txt /app/requirements.txt - -RUN pip install --upgrade pip -RUN pip install -r requirements.txt - -COPY . . - -EXPOSE 8080 -CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/images/gitlab-error-processor/app.py b/images/gitlab-error-processor/app.py deleted file mode 100644 index f5219fc11..000000000 --- a/images/gitlab-error-processor/app.py +++ /dev/null @@ -1,61 +0,0 @@ -import json -from pathlib import Path - -import sentry_sdk -import yaml -from fastapi import FastAPI, HTTPException, Request, Response -from kubernetes import client, config - -sentry_sdk.init( - # Sample only 1% of jobs - traces_sample_rate=0.01, -) - -config.load_incluster_config() - -batch = client.BatchV1Api() - -app = FastAPI() - - -@app.post("/") -async def gitlab_webhook_consumer(request: Request): - """ - This endpoint receives the gitlab webhook for failed jobs and creates - a k8s job to parse the logs and upload them to opensearch. - """ - job_input_data = await request.json() - - if job_input_data.get("object_kind", "") != "build": - raise HTTPException(status_code=400, detail="Invalid request") - - if job_input_data["build_status"] != "failed": - return Response("Not a failed job, no action needed.", status_code=200) - - with open(Path(__file__).parent / "job-template.yaml") as f: - job_template = yaml.safe_load(f) - - for container in job_template["spec"]["template"]["spec"]["containers"]: - container.setdefault("env", []).extend( - [dict(name="JOB_INPUT_DATA", value=json.dumps(job_input_data))] - ) - - job_build_id = str(job_input_data["build_id"]) - job_pipeline_id = str(job_input_data["pipeline_id"]) - - job_template["metadata"][ - "name" - ] = f"gitlab-error-processing-job-{job_build_id}-{job_pipeline_id}" - - # Add labels to make finding the job that proccessed the error log easier. - job_template["metadata"]["labels"] = { - "spack.io/gitlab-build-id": job_build_id, - "spack.io/gitlab-pipeline-id": job_pipeline_id, - } - - batch.create_namespaced_job( - "custom", - job_template, - ) - - return Response("Upload job dispatched.", status_code=202) diff --git a/images/gitlab-error-processor/job-template.yaml b/images/gitlab-error-processor/job-template.yaml deleted file mode 100644 index f3bea879e..000000000 --- a/images/gitlab-error-processor/job-template.yaml +++ /dev/null @@ -1,63 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: gitlab-error-processing-job - labels: - app: gitlab-error-processing-job -spec: - ttlSecondsAfterFinished: 7200 - template: - metadata: - labels: - app: gitlab-error-processing-job - spec: - restartPolicy: OnFailure - containers: - - name: gitlab-error-processing-job - image: ghcr.io/spack/upload-gitlab-failure-logs:0.0.6 - imagePullPolicy: Always - env: - - name: GITLAB_TOKEN - valueFrom: - secretKeyRef: - name: gitlab-error-processor - key: gitlab-token - - # postgres credentials - - name: GITLAB_POSTGRES_HOST - valueFrom: - secretKeyRef: - name: gitlab-error-processor - key: postgresql-gitlab-host - - name: GITLAB_POSTGRES_DB - value: gitlabhq_production - - name: GITLAB_POSTGRES_RO_USER - value: gitlab_ro_user - - name: GITLAB_POSTGRES_RO_PASSWORD - valueFrom: - secretKeyRef: - name: gitlab-error-processor - key: postgresql-gitlab-ro-user-password - - # opensearch credentials - - name: OPENSEARCH_ENDPOINT - valueFrom: - secretKeyRef: - name: gitlab-error-processor - key: opensearch-endpoint - - name: OPENSEARCH_USERNAME - valueFrom: - secretKeyRef: - name: gitlab-error-processor - key: opensearch-username - - name: OPENSEARCH_PASSWORD - valueFrom: - secretKeyRef: - name: gitlab-error-processor - key: opensearch-password - envFrom: - - configMapRef: - name: python-scripts-sentry-config - nodeSelector: - spack.io/node-pool: base diff --git a/images/gitlab-error-processor/requirements.txt b/images/gitlab-error-processor/requirements.txt deleted file mode 100644 index b90cb9126..000000000 --- a/images/gitlab-error-processor/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -fastapi==0.85.1 -kubernetes==25.3.0 -PyYAML==6.0 -sentry-sdk[fastapi] -uvicorn==0.19.0 diff --git a/images/upload-gitlab-failure-logs/Dockerfile b/images/upload-gitlab-failure-logs/Dockerfile deleted file mode 100644 index 447581034..000000000 --- a/images/upload-gitlab-failure-logs/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM python:3.11-slim - -WORKDIR /app - -COPY requirements.txt /app/requirements.txt - -RUN pip install --upgrade pip -RUN pip install -r requirements.txt - -COPY . . - -CMD [ "python", "./upload_gitlab_failure_logs.py" ] diff --git a/images/upload-gitlab-failure-logs/requirements.txt b/images/upload-gitlab-failure-logs/requirements.txt deleted file mode 100644 index 95892e37e..000000000 --- a/images/upload-gitlab-failure-logs/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -kubernetes==26.1.0 -opensearch-dsl==2.0.1 -python-gitlab==3.11.0 -PyYAML==6.0 -psycopg2-binary==2.9.5 -sentry-sdk diff --git a/k8s/production/custom/gitlab-error-processor/deployments.yaml b/k8s/production/custom/gitlab-error-processor/deployments.yaml deleted file mode 100644 index 99493b3dc..000000000 --- a/k8s/production/custom/gitlab-error-processor/deployments.yaml +++ /dev/null @@ -1,41 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: gitlab-error-processor - namespace: custom - labels: - app: gitlab-error-processor - svc: web -spec: - selector: - matchLabels: - app: gitlab-error-processor - svc: web - replicas: 2 - template: - metadata: - labels: - app: gitlab-error-processor - svc: web - spec: - restartPolicy: Always - serviceAccountName: gitlab-error-processor - containers: - - name: gitlab-error-processor - image: ghcr.io/spack/gitlab-error-processor:0.0.5 - imagePullPolicy: Always - envFrom: - - configMapRef: - name: python-scripts-sentry-config - resources: - requests: - cpu: 350m - memory: 1G - limits: - cpu: 1600m - memory: 1G - ports: - - containerPort: 8080 - nodeSelector: - spack.io/node-pool: base diff --git a/k8s/production/custom/gitlab-error-processor/sealed-secrets.yaml b/k8s/production/custom/gitlab-error-processor/sealed-secrets.yaml deleted file mode 100644 index 88c5f2fca..000000000 --- a/k8s/production/custom/gitlab-error-processor/sealed-secrets.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: bitnami.com/v1alpha1 -kind: SealedSecret -metadata: - name: gitlab-error-processor - namespace: custom -spec: - encryptedData: - # GitLab personal access token with read access to spack/spack repo - gitlab-token: AgAoSAnNd5m/Nr8bWWHgvDDS1Kfr0LFXv1q3tvVtCMwpQw0ZtRd4toTiC7gKYZqJq1YHq/7ajpSXPXnb503fO1dnm/N+zsKpNNCAFijvg0RA0WYQQxIGADVl9GeE4k0MCRewLSbIyjgLPWMhizHJjSNBHFygSyqW/OGn23GseYByt20wJ/jd284gCqjgP51CB6/kk6LD5zeQPmswL4HaPoi2+/oEoJuzDKwsNouAwCl2XsTEqAL/aduXvdxo9GnbOtlUpszSDAGbsLRbrRMMtTIPmZZmLEEDw8zrI6qunQJUuCCZnGMQkSJuEvh+uJQM/z7dZ5Cv6PH7nVEN2uNAIEhLDb6UVRj5R/SUXo9GciwVZqUkZImtSedVDSkCgsxIo07ae+0Z+Y0KsC0I2q9XuzwjwOHx8nQhPz2SOqjnIMTuyuG0jwoM5+mBSql1jv+wOAlr+NjGE2YyndS9NC1fb2M59a3Na6kqOzx8yfz450btPJGdc7zRA/OPwA9LZ4NE/y/sMiz2pFR7u35KuQlxt8g1jcxjJF64c9a4euyHWlyF7c0QNZwb3kMWr4hKMjezgL8eCvoPjzx3obxtzpIblLOp0UXN88ZFQZjjndFT0T4FB5xMGfrBv8Ai0U5ohYimZxrPuFspFhnJIStbtjcWbl5DKAqtJ3atwHxc9lS05Zq6lFMlBvsxT9Q6aYfK5EyZBiaZ+0821NM/dEnhuee/6CGHFXddMgFWJ0qJfg== - # The endpoint of the opensearch API. Should be the same as the "host" from k8s/logging/secrets.yaml - opensearch-endpoint: AgBfmzIyNATaOQNTY3lwA8dJoPXDb5jKJOJRf8vvPLKGGGmCzBNXngP2r3KBya3zpKpd0pcMfmQALR/GbbcdqdwhuaqrJmKTnBNWtBEhndxglaLotoYGeFg4+DGYGvXF7j8DJXEyl/Efmm+90ubk/XIobGr8m2AxXwAK4T2d5L3zuZXLfR2ol8NTXejzoCyQa9DwxGWJeYn0hJLy4kKwW450ucYndhJvSkfvOsQSKO3vrnZRYN5qUMuURv/yX4lKwXG7RwDbFbw+39knQ/hRn4Ts/IxAiRKlRyYzKfY0ZrziP49OT9+PaKJ5A7XSF63LBE9Pkx55KrAChVBwZb6dOK90pDdJ4nmT0nvm3ggBxkRqHQl+Abdf0/q3H2HZ//cprwh5tUEIxU6H1PLJeY3gVd/xqviG1rrOyqrCggmol93RJN6GCji9dRJP3mhT5f6YFyeHOLOXZ8Gv7pTEMGFohnquTJ9hQ87l4VOY+zs5Oe3EZ9KxUXgdFBMlSncmkYWsqcs/2ckctbKDh4C6B9dbdoqx6ZAzZsQG/i5cexpHp6C36ECUyatBCAXP41myC84KHImQS3IJonMNt54DmiVLUyVDbhdfPw4XgFIW93S3e1xbRbh3XOPzLggy1Sn5rU3Hx1IWgfZn7ePq5YX1apo2nvxQdrsZFWPVi5opO1m/2JlpEZyNpUjPhmJ/3RewqV06uVLfgLiNQInm7uPdScV2F56rla+wIIbbj4gMaYltXNu7XzQ7Erihu/n+sBFAiGb78QYUT27mUu3UQLm4tsRXjyXKNZJqiw5zyTDYEg== - # Password for the admin account of the OpenSearch cluster. Set using the AWS console. - opensearch-password: AgA2I8GIFNDKJuFwTa/kLnMYRJMUsTdOyGui7xzZIswxsTRuPYZJeqxIN7Zq2uOENa6H/diAV0QF8aQhieopMI7JLFTzky3nrjaI86Wsj8JbfbdPA1VSaiovHXQDKDioru21gR2oUI70j+o8gp9kdmL7DYhATZlU5Ix1T1BZXG4f5ZmZBUcL2V731+Se0UFqVoQNLMf7LOLuZwMNpfIVyZ0q7d+HHxoYvpxKI94tj3V8v1r+Takj4Rc9aLffwIym98duhXseVx3fkkcPvrZIxj0AADgfAuMtsSOVnVuR0wqm0MIrUYDo261jwsvmPn167bD3VLMF5qaNBD5RFYZ+JwbRKlytn+NpBN+/Nzk7oEjTB5VX9VIOEDsllhTFrCraIGTxQZD7TONPoxGhGzJvxsN/5zYtKGmsZg6tYhh76nD6RYGF23ixgu5hJ7wYx0Q101mUL/FSJrM/207MgNbrRt1HCw40fJZRyrFkEQxzJXFhVUVVjRslH46lEnNtxA9BNMxdJa1+vAW8r73UJgFSSRICjgSCNYORx8m/+iNOShjrKx4eEM+dWs6/gNnvYN1doj574ZxKULjRI7lGJWQEFI+gXs+zDNy8N0GU2rXeqrlGENyo0c+lBg94x6CGrerk8CLnMJ+0yn7XM9UgdN1bfvkm2VkZsudvS/uLV9cspgeEgo26fALl+QelBxJhbTcgoFtO4UXV87TaqDUGBfSjUhAbzxQVOnuf1m4qdtZl54KOW/O/cJhRwSNLgneUz/fckwRKPysWq/zpfpMSq+0OOZC4 - # Admin user name for the OpenSearch cluster. Set using the AWS console. - opensearch-username: AgAF1yT2IDAVwzdxDQlTwkAx7P42NTNpyHOQYiXpxNhcPVEz+49zU9lOUTZY4W/X3NO1wgiQXKbntOX7cD2+EabkBYN3Oj/uujZTLgnnJoOYYQ0I//OgHgnx37SjfMV7+7pbdx7HTS85U2yQ7xfS9x1jsxjzLOjGTLCF5ymFVf4RxWEmqR0z7LnhQcRh6esNEUrjMbs+S1uQ/ZWvFJ3WMkuRntMKt2MI4cRKvZMRrXXwnOoCUn1vUWYU1sLQTFQFHvO8FCWfm1sfuIRv144odXf7K+ADoQe1pnlLkG8tUfau/qgKEU610W+vQMFRrLLiUg33NNVO2ZX47xxzRrVUwHzgkmDgefqDtxhOW8MkTny3YjeOteeM/yVQseW5S1/exGzJE1vqxfziyegivU+Wzkfp6obTFN3PL4urj33oSTPXvFDXmq6Nf3twvNMqsI0ccu1SiMUvBjJXTLRp6ReWm+vKJDiNU185WNiyjX8jQxM7sVNLzVLVyQW0ZOFEFJxIZXfi8icmzgSvpsS9pKdsnBC/PD+944LUXWG9zeJOiPFSIOvSDw1a7VRALVcZ4sPrXilJLZ4++8UMqDIy56VABg7/c19DxD0xNPdnbhfSHeN7JeNiF2fXcAdqPHeNDbh8SgBctgVD32i7cs2ZgorY10Y9zJSc/Jlcp1mox/XksTtclW6FpH8xk7uBA71ckVri0PesR0MAyg== - # The host for the gitlab RDS instance - postgresql-gitlab-host: AgBHW6PSaP33FshUWCq7WHb5Ynd/rUdjgx38Zn1FJy+LHICWBW/l0NOTJ2JwIazuzZ9UEUT4mrvXack5SD2AHKyV1cdT+dYcfmLjMEJSNycf5FsSukqHAnPI5d8UJQUPsZqDasAQY64KG7LvfXEH/YjVwvGX94GeJiuQh8oM0eISD1iI0T2aW88Oro4b7YeHMphW3yVG841vXgkU4XUADx7LYX6JDO6VeowU8xlNz0TYztNAoKM40bZXw9DZslXX0/e5wFygbJgAfbeqoCZ240Q2SBwg5dxAtKzvEkuOZetsU4mq/kVKIQsBa/jg9tiA3mvvXz7qKDpZmgMYXl3hGUb0qcYxYf+hiRA29YPHB2ltrWiIPLWh4DJZ42RcTyJy6cWuzQhdvLJ9z16aBgqlKtWEpY6UbPKoyQ2jElHKfacicFT+aTeLlBd9dydq0Ar4UXecI/koRd7B54gxJpAbgixcVR6PE2KZYdp+2j4bRpB5i5ligVXsKeGjTCVxVO6wc3i3Y2teqeAnVh55vY8sRSbN3YwlwF7sjIl/k+stB/8jvjbLRAKdTat+CH4npE0ThU+0msRWfW7Ju5v2JSPQuvU6c4q4OC9i2F7mKoRb4b+mGnS8oPRlOe+u98ntPOhbfeNJvhU7Qe6zSTRMmHqxRuhyOzWUalPioZHBNw6EbnsB1L3D9Sh9BxO8ImUpAaWwaQ6x6f92RJ6/7/NCMPcZqlbf7NJ5T7NlFFiHZj08wLYW3IVScdbiVFgErT0Cw+d+Pn0+Gnm4 - # The password for the read-only user on the gitlab RDS instance - postgresql-gitlab-ro-user-password: AgBcFyWYPG8JyXorn4UbW/InmyT91o6ao2PSwUPEQdhFZVfq0c5gEeUnrgk7pBIwsPOjLgNVWUo/CoM6OV6Ud6XN7lhBwq+13rFRsp8KUoTHEgEeXDzwa4qFwqlrvI9Sec98vIRcXSmh3QhpVFoy13UhwM1G2n4Q8lgz0Miqe0BAmlclZqvhDkl3NKuQ/bFHeztNw8Q8xEObZ8QWfCCHW3p5QRIvE0Rmi/U4nMoJnUnINO1YWJ/yc4nuiqt4sIWnes8FUorcigjYABVOMc6RaJFvqUlajgSkLIdjJ63kv30h8iATsEhOJwW8aHTqc31Di3l3nfl5p65Y7QlrZMdHKVouzv3be5VtDuynNXiFBuD9KpMzQoWeemVH323f/O3QNuXVN83RktQcwChxQPIPY8RGl58rQLrijN/mOBYW9RYMqzm8G78Xf5FW4x4KOVAoObVG9oCNq6MmSsHbJb0ddKloCyfXMDS6qCAypBZCb/GNSpcdsLyKyv3JJhSrq7fLjjlL43YE8wNTwpgias/l32BmzqLYjCwst/MGF2S7aO40Fsbynaqz1NjlDNXACCkkL1nbRpMvxWqDT5AjN/E4j/49yQssGKflLvYw4dY2jbd0rNxPgA38bfHVZ7Laxd207l5vKlNi5AMnZ+Phve/3fINjcaclaz/TW1du2biI2QsJGeSvDb9ndQrYYLMJog87LQywJIpwRakDsbRb7qf3hnL1UyaErg== - template: - metadata: - annotations: - kustomize.toolkit.fluxcd.io/reconcile: disabled - sealedsecrets.bitnami.com/managed: "true" diff --git a/k8s/production/custom/gitlab-error-processor/service-accounts.yaml b/k8s/production/custom/gitlab-error-processor/service-accounts.yaml deleted file mode 100644 index d43023917..000000000 --- a/k8s/production/custom/gitlab-error-processor/service-accounts.yaml +++ /dev/null @@ -1,32 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: gitlab-error-processor - namespace: custom - ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: gitlab-error-processor - namespace: custom -rules: - - apiGroups: ["batch"] - resources: ["jobs"] - verbs: ["create"] - ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: gitlab-error-processor - namespace: custom -subjects: - - kind: ServiceAccount - name: gitlab-error-processor - namespace: custom -roleRef: - kind: Role - name: gitlab-error-processor - apiGroup: rbac.authorization.k8s.io diff --git a/k8s/production/custom/gitlab-error-processor/services.yaml b/k8s/production/custom/gitlab-error-processor/services.yaml deleted file mode 100644 index 0fd48553b..000000000 --- a/k8s/production/custom/gitlab-error-processor/services.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: gitlab-error-processor - namespace: custom - labels: - app: gitlab-error-processor - svc: web -spec: - type: ClusterIP - ports: - - name: web - port: 80 - targetPort: 8080 - selector: - app: gitlab-error-processor - svc: web diff --git a/k8s/production/custom/webhook-handler/deployments.yaml b/k8s/production/custom/webhook-handler/deployments.yaml index 1baf8b3b5..db1c6ff38 100644 --- a/k8s/production/custom/webhook-handler/deployments.yaml +++ b/k8s/production/custom/webhook-handler/deployments.yaml @@ -23,7 +23,7 @@ spec: serviceAccountName: webhook-handler containers: - name: webhook-handler - image: ghcr.io/spack/django:0.1.3 + image: ghcr.io/spack/django:0.1.4 imagePullPolicy: Always resources: requests: @@ -121,7 +121,7 @@ spec: serviceAccountName: webhook-handler containers: - name: webhook-handler-worker - image: ghcr.io/spack/django:0.1.3 + image: ghcr.io/spack/django:0.1.4 command: ["celery", "-A", "analytics.celery", "worker", "-l", "info", "-Q", "celery"] imagePullPolicy: Always resources: @@ -168,6 +168,31 @@ spec: secretKeyRef: name: webhook-handler key: analytics-postgresql-password + - name: GITLAB_DB_USER + valueFrom: + secretKeyRef: + name: webhook-handler + key: gitlab-db-user + - name: GITLAB_DB_HOST + valueFrom: + secretKeyRef: + name: webhook-handler + key: gitlab-db-host + - name: GITLAB_DB_NAME + valueFrom: + secretKeyRef: + name: webhook-handler + key: gitlab-db-name + - name: GITLAB_DB_PASS + valueFrom: + secretKeyRef: + name: webhook-handler + key: gitlab-db-password + - name: GITLAB_DB_PORT + valueFrom: + secretKeyRef: + name: webhook-handler + key: gitlab-db-port - name: OPENSEARCH_ENDPOINT valueFrom: secretKeyRef: diff --git a/terraform/modules/spack/gitlab_webhooks.tf b/terraform/modules/spack/gitlab_webhooks.tf index 5cc1d1a74..9c5a4a96d 100644 --- a/terraform/modules/spack/gitlab_webhooks.tf +++ b/terraform/modules/spack/gitlab_webhooks.tf @@ -36,6 +36,10 @@ resource "random_password" "webhook_handler" { special = false } +data "aws_secretsmanager_secret_version" "gitlab_db_ro_credentials" { + secret_id = "gitlab-${var.deployment_name}-readonly-credentials" +} + # Note: the /1 is important to ensure that the broker for the webhook handler isn't using the same # database as the broker for spackbot. resource "kubectl_manifest" "webhook_secrets" { @@ -48,6 +52,11 @@ resource "kubectl_manifest" "webhook_secrets" { data: gitlab-endpoint: ${base64encode("${var.gitlab_url}")} gitlab-token: ${base64encode("${gitlab_personal_access_token.webhook_handler.token}")} + gitlab-db-user: ${base64encode("${jsondecode(data.aws_secretsmanager_secret_version.gitlab_db_ro_credentials.secret_string)["username"]}")} + gitlab-db-host: ${base64encode("${jsondecode(data.aws_secretsmanager_secret_version.gitlab_db_ro_credentials.secret_string)["host"]}")} + gitlab-db-port: ${base64encode("${jsondecode(data.aws_secretsmanager_secret_version.gitlab_db_ro_credentials.secret_string)["port"]}")} + gitlab-db-name: ${base64encode("${jsondecode(data.aws_secretsmanager_secret_version.gitlab_db_ro_credentials.secret_string)["dbname"]}")} + gitlab-db-password: ${base64encode("${jsondecode(data.aws_secretsmanager_secret_version.gitlab_db_ro_credentials.secret_string)["password"]}")} sentry-dsn: ${base64encode("${data.sentry_key.webhook_handler.dsn_public}")} secret-key: ${base64encode("${random_password.webhook_handler.result}")} celery-broker-url: ${base64encode("redis://${aws_elasticache_replication_group.pr_binary_graduation_task_queue.primary_endpoint_address}:6379/1")}