Bug 2006684 - add builddecisionscript

.taskcluster.yml

@@ -79,6 +79,7 @@ tasks:
               - beetmoverscript
               - bitrisescript
               - bouncerscript
+              - builddecisionscript
               - githubscript
               # - iscript (iscript has special release via ronin_puppet repo)
               - landoscript

builddecisionscript/docker.d/init_worker.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -o errexit -o pipefail
+
+test_var_set() {
+  local varname=$1
+
+  if [[ -z "${!varname}" ]]; then
+    echo "error: ${varname} is not set"
+    exit 1
+  fi
+}
+
+test_var_set 'TASKCLUSTER_ROOT_URL'
+
+export VERIFY_CHAIN_OF_TRUST=false
+
+export WORKER_TYPE="${PROJECT_NAME}${WORKER_SUFFIX}"
+export WORKER_GROUP=${WORKER_TYPE}
+export WORKER_ID_PREFIX="${WORKER_TYPE}-"

builddecisionscript/docker.d/worker.yml

@@ -0,0 +1,3 @@
+work_dir: { "$eval": "WORK_DIR" }
+artifact_dir: { "$eval": "ARTIFACTS_DIR" }
+verbose: { "$eval": "VERBOSE == 'true'" }

builddecisionscript/pyproject.toml

@@ -0,0 +1,59 @@
+[project]
+name = "builddecisionscript"
+version = "1.0.0"
+description = "Scriptworker script to create build decision tasks for hg-push and cron triggers"
+url = "https://github.com/mozilla-releng/scriptworker-scripts/"
+license = "MPL-2.0"
+readme = "README.md"
+authors = [
+    { name = "Mozilla Release Engineering", email = "release+python@mozilla.com" }
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+]
+dependencies = [
+    "attrs",
+    "json-e",
+    "jsonschema>4.18",
+    "pyyaml",
+    "redo",
+    "referencing",
+    "requests",
+    "scriptworker-client",
+    "slugid",
+    "taskcluster",
+    "taskcluster-taskgraph",
+]
+
+[dependency-groups]
+dev = [
+    "tox",
+    "tox-uv",
+    "coverage>=4.2",
+    "pytest",
+    "pytest-asyncio<1.0",
+    "pytest-cov",
+    "pytest-mock",
+    "pytest-scriptworker-client",
+    "responses",
+]
+
+[tool.uv.sources]
+scriptworker-client = { workspace = true }
+pytest-scriptworker-client = { workspace = true }
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build]
+include = [
+    "src",
+]
+
+[tool.hatch.build.targets.wheel.sources]
+"src/" = ""
+
+[project.scripts]
+builddecisionscript = "builddecisionscript.script:main"

builddecisionscript/src/builddecisionscript/__init__.py

@@ -0,0 +1,3 @@
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at http://mozilla.org/MPL/2.0/.

builddecisionscript/src/builddecisionscript/cron/__init__.py

@@ -0,0 +1,137 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+import traceback
+from pathlib import Path
+
+from requests.exceptions import HTTPError
+from taskgraph.util.keyed_by import evaluate_keyed_by
+
+from ..repository import NoPushesError
+from ..util.schema import Schema
+from . import action, decision
+from .util import calculate_time, match_utc
+
+# Functions to handle each `job.type` in `.cron.yml`.  These are called with
+# the contents of the `job` property from `.cron.yml` and should return a
+# sequence of (taskId, task) tuples which will subsequently be fed to
+# createTask.
+JOB_TYPES = {
+    "decision-task": decision.run_decision_task,
+    "trigger-action": action.run_trigger_action,
+}
+
+logger = logging.getLogger(__name__)
+
+_cron_yml_schema = Schema.from_file(Path(__file__).with_name("schema.yml"))
+
+
+def load_jobs(repository, revision):
+    try:
+        cron_yml = repository.get_file(".cron.yml", revision=revision)
+    except HTTPError as e:
+        if e.response.status_code == 404:
+            return {}
+        raise
+    _cron_yml_schema.validate(cron_yml)
+
+    # resolve keyed_by fields in each job
+    jobs = cron_yml["jobs"]
+    return {j["name"]: j for j in jobs}
+
+
+def should_run(job, *, time, project):
+    if "run-on-projects" in job:
+        if project not in job["run-on-projects"]:
+            return False
+    # Resolve when key here, so we don't require it before we know that we
+    # actually want to run on this branch.
+    when = evaluate_keyed_by(
+        job.get("when", []),
+        "Cron job " + job["name"],
+        {"project": project},
+    )
+    if not any(match_utc(time=time, sched=sched) for sched in when):
+        return False
+    return True
+
+
+def run_job(job_name, job, *, repository, push_info, cron_input=None, dry_run=False):
+    job_type = job["job"]["type"]
+    if job_type in JOB_TYPES:
+        JOB_TYPES[job_type](
+            job_name,
+            job["job"],
+            repository=repository,
+            push_info=push_info,
+            cron_input=cron_input or {},
+            dry_run=dry_run,
+        )
+    else:
+        raise Exception(f"job type {job_type} not recognized")
+
+
+def run(*, repository, branch, force_run, cron_input=None, dry_run):
+    time = calculate_time()
+
+    try:
+        push_info = repository.get_push_info(branch=branch)
+    except NoPushesError:
+        # A common cause for this is a new repository being set-up that hasn't
+        # had pushes made to it yet. Avoiding an exception for this allows the
+        # repository to be added to projects.yml before pushes are made to it.
+        logger.info("No pushes found; doing nothing.")
+        return
+
+    jobs = load_jobs(repository, revision=push_info["revision"])
+
+    if force_run:
+        job_name = force_run
+        logger.info(f'force-running cron job "{job_name}"')
+        run_job(
+            job_name,
+            jobs[job_name],
+            repository=repository,
+            push_info=push_info,
+            cron_input=cron_input,
+            dry_run=dry_run,
+        )
+        return
+
+    failed_jobs = []
+    for job_name, job in sorted(jobs.items()):
+        if should_run(job, time=time, project=repository.project):
+            logger.info(f'running cron job "{job_name}"')
+            try:
+                run_job(
+                    job_name,
+                    job,
+                    repository=repository,
+                    push_info=push_info,
+                    cron_input=cron_input,
+                    dry_run=dry_run,
+                )
+            except Exception as exc:
+                # report the exception, but don't fail the whole cron task, as that
+                # would leave other jobs un-run.
+                failed_jobs.append((job_name, exc))
+                traceback.print_exc()
+                logger.error(f'cron job "{job_name}" run failed; continuing to next job')
+
+        else:
+            logger.info(f'not running cron job "{job_name}"')
+
+    _format_and_raise_error_if_any(failed_jobs)
+
+
+def _format_and_raise_error_if_any(failed_jobs):
+    if failed_jobs:
+        failed_job_names = [job_name for job_name, _ in failed_jobs]
+        failed_job_names_with_exceptions = (f'"{job_name}": "{exc}"' for job_name, exc in failed_jobs)
+        raise RuntimeError(
+            "Cron jobs {} couldn't be triggered properly. Reason(s):\n * {}\nSee logs above for details.".format(
+                failed_job_names, "\n * ".join(failed_job_names_with_exceptions)
+            )
+        )

builddecisionscript/src/builddecisionscript/cron/action.py

@@ -0,0 +1,47 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+
+import taskcluster
+
+from ..util.http import SESSION
+from ..util.taskcluster import get_taskcluster_options
+from ..util.trigger_action import render_action
+
+logger = logging.getLogger(__name__)
+
+
+def find_decision_task(repository, revision):
+    """Given repository and revision, find the taskId of the decision task."""
+    index = taskcluster.Index(get_taskcluster_options(), session=SESSION)
+    decision_index = f"{repository.trust_domain}.v2.{repository.project}.revision.{revision}.taskgraph.decision"
+    logger.info("Looking for index: %s", decision_index)
+    task_id = index.findTask(decision_index)["taskId"]
+    logger.info("Found decision task: %s", task_id)
+    return task_id
+
+
+def run_trigger_action(job_name, job, *, repository, push_info, cron_input=None, dry_run):
+    action_name = job["action-name"]
+    decision_task_id = find_decision_task(repository, push_info["revision"])
+
+    action_input = {}
+
+    if job.get("include-cron-input") and cron_input:
+        action_input.update(cron_input)
+
+    if job.get("extra-input"):
+        action_input.update(job["extra-input"])
+
+    hook = render_action(
+        action_name=action_name,
+        task_id=None,
+        decision_task_id=decision_task_id,
+        action_input=action_input,
+    )
+
+    hook.display()
+    if not dry_run:
+        hook.submit()

builddecisionscript/src/builddecisionscript/cron/decision.py

@@ -0,0 +1,66 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import copy
+import logging
+import os
+import shlex
+
+from ..decision import render_tc_yml
+
+logger = logging.getLogger(__name__)
+
+
+def make_arguments(job):
+    arguments = []
+    if "target-tasks-method" in job:
+        arguments.append("--target-tasks-method={}".format(job["target-tasks-method"]))
+    if job.get("optimize-target-tasks") is not None:
+        arguments.append(
+            "--optimize-target-tasks={}".format(
+                str(job["optimize-target-tasks"]).lower(),
+            )
+        )
+    if "include-push-tasks" in job:
+        arguments.append("--include-push-tasks")
+    if "rebuild-kinds" in job:
+        for kind in job["rebuild-kinds"]:
+            arguments.append(f"--rebuild-kind={kind}")
+    return arguments
+
+
+def run_decision_task(job_name, job, *, repository, push_info, cron_input=None, dry_run):
+    """Generate a basic decision task, based on the root .taskcluster.yml"""
+    push_info = copy.deepcopy(push_info)
+    push_info["owner"] = "cron"
+
+    taskcluster_yml = repository.get_file(".taskcluster.yml", revision=push_info["revision"])
+
+    arguments = make_arguments(job)
+
+    effective_cron_input = {}
+    if job.get("include-cron-input") and cron_input:
+        effective_cron_input.update(cron_input)
+
+    cron_info = {
+        "task_id": os.environ.get("TASK_ID", "<cron task id>"),
+        "job_name": job_name,
+        "job_symbol": job["treeherder-symbol"],
+        # args are shell-quoted since they are given to `bash -c`
+        "quoted_args": " ".join(shlex.quote(a) for a in arguments),
+        "input": effective_cron_input,
+    }
+
+    task = render_tc_yml(
+        taskcluster_yml,
+        taskcluster_root_url=os.environ["TASKCLUSTER_ROOT_URL"],
+        tasks_for="cron",
+        repository=repository.to_json(),
+        push=push_info,
+        cron=cron_info,
+    )
+
+    task.display()
+    if not dry_run:
+        task.submit()