From 43ec33097887e6796330525c864494607d1a5e12 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Thu, 2 Jul 2026 17:00:25 +1000 Subject: [PATCH 1/5] Shellcheck bin/clicksnap-setup --- bin/clicksnap-setup | 81 +++++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 35 deletions(-) diff --git a/bin/clicksnap-setup b/bin/clicksnap-setup index 1b138d9..8d9efe1 100755 --- a/bin/clicksnap-setup +++ b/bin/clicksnap-setup @@ -8,13 +8,13 @@ # Free Software Foundation; either version 3 of the License, or (at your # option) any later version. -fatal() { echo "FATAL [$(basename $0)]: $@" 1>&2; exit 1; } -warn() { echo -e "WARNING [$(basename $0)]: $@"; } -info() { echo "INFO [$(basename $0)]: $@"; } +fatal() { echo "FATAL [$(basename "$0")]: $*" 1>&2; exit 1; } +warn() { echo -e "WARNING [$(basename "$0")]: $*"; } +info() { echo "INFO [$(basename "$0")]: $*"; } usage() { cat<&2 + echo "Error: $*" >&2 exit 1 fi exit @@ -38,40 +38,49 @@ EOF [ -n "$BT_DEBUG" ] && set -x -export BT=$(dirname $(dirname $(readlink -f $0))) +BT=$(dirname "$(dirname "$(readlink -f "$0")")") +export BT export BT_CONFIG=$BT/config [[ -n "$BASE_DIR" ]] || BASE_DIR=/turnkey/public -mkdir -p $BASE_DIR -GH_URL=https://github.com/turnkeylinux +mkdir -p "$BASE_DIR" +GIT_REMOTE_URL="${GIT_REMOTE_URL:-https://github.com/turnkeylinux}" +GIT_REMOTE_NAME="${GIT_REMOTE_NAME:-origin}" unset app -while [ "$1" != "" ]; do +while [[ "$1" != "" ]]; do case $1 in - --help|-h ) usage;; - *) if [[ -z "$app" ]]; then - app="$1" - else - usage "Unknown option/multiple app names given: '$1'" - fi;; + --help|-h) + usage + ;; + *) + + if [[ -z "$app" ]]; then + app="$1" + else + usage "Unknown option/multiple app names given: '$1'" + fi + ;; esac shift done install() { - info "Updating apt cache and installing deps:" $@ - info "installing $@" + info "Updating apt cache and installing deps: $*" + info "installing $*" apt-get -qq update - DEBIAN_FRONTEND=noninteractive apt-get -y install $@ + DEBIAN_FRONTEND=noninteractive apt-get -y install "$@" } git_pull() { local dir=$1 local app=$2 - cd $dir - local remote=$(sed -En "s|(^[a-zA-Z0-9_-]*)[[:space:]].*|\1|p" \ - <<<$(git remote -v | grep -m1 "turnkeylinux/$app")) - git pull $remote master + cd "$dir" + git fetch "$GIT_REMOTE_NAME" + if [[ -n "$(git status --porcelain)" ]]; then + fatal "git repo $dir is dirty; please fix and retry" + fi + git pull "$GIT_REMOTE_NAME" master } # dl & check for screenshot code first so we can bail as early as possible @@ -86,25 +95,26 @@ for dl in tkldev-docker clicksnap; do fi else info "Downloading $dl source" - git clone --depth=1 $GH_URL/$dl $BASE_DIR/$dl + git clone --depth=1 --origin "$GIT_REMOTE_NAME" \ + "$GIT_REMOTE_URL/$dl" "$BASE_DIR/$dl" fi done -app=$(sed "s|-|_|g" <<<$app) +app="${app//-/_}" if [[ -z "$app" ]]; then warn "App name not given - continuing, but may fail later" -elif ! ls $BASE_DIR/clicksnap/src/apps/ | grep -q -w "$app" \ +elif [[ ! -f "$BASE_DIR/clicksnap/src/apps/$app" ]] \ && [[ -z "$no_screens" ]] ; then fatal "Clicksnap code for $app not found (checked in $BASE_DIR/clicksnap/src/apps/)" fi # sed, fab & deck should be installed, but just in case deps="podman sed fab deck" -missing='' +missing=() for dep in $deps; do - which $dep >/dev/null || missing="$missing $dep" + which "$dep" >/dev/null || missing+=("$dep") done -[[ -z "$missing" ]] || install $missing +[[ -z "${missing[*]}" ]] || install "${missing[@]}" case "$(which cargo || echo 'fail')" in "$HOME/.cargo/bin/cargo") @@ -116,16 +126,17 @@ case "$(which cargo || echo 'fail')" in fail) info "Installing rust via rustup" curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - echo "export PATH=\"\$HOME/.cargo/bin:\$PATH\"" > $HOME/.bashrc.d/rust - chmod +x $HOME/.bashrc.d/rust - source $HOME/.bashrc.d/rust;; + echo "export PATH=\"\$HOME/.cargo/bin:\$PATH\"" > "$HOME/.bashrc.d/rust" + chmod +x "$HOME/.bashrc.d/rust" + # shellcheck source=/dev/null + source "$HOME/.bashrc.d/rust";; *) fatal "Unexpected cargo path: '$1'";; esac -cd $BASE_DIR/clicksnap +cd "$BASE_DIR/clicksnap" info "Building & installing clicksnap" cargo build -ln -sf $PWD/target/debug/clicksnap /usr/local/bin/clicksnap -ln -sf $BASE_DIR/tkldev-docker/dockerize.sh /usr/local/bin/dockerize -ln -sf $BASE_DIR/tkldev-docker/wait-ready.sh /usr/local/bin/tkl-docker-wait-ready +ln -sf "$PWD/target/debug/clicksnap" /usr/local/bin/clicksnap +ln -sf "$BASE_DIR/tkldev-docker/dockerize.sh" /usr/local/bin/dockerize +ln -sf "$BASE_DIR/tkldev-docker/wait-ready.sh" /usr/local/bin/tkl-docker-wait-ready From ecef60f5b76a77d56e855d34d1ba55666374146b Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Thu, 2 Jul 2026 17:01:37 +1000 Subject: [PATCH 2/5] Extend config to support customizable git cloning --- config.example/common.cfg | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config.example/common.cfg b/config.example/common.cfg index 038f9f3..102444a 100644 --- a/config.example/common.cfg +++ b/config.example/common.cfg @@ -12,9 +12,16 @@ export BT_BUILDS=/mnt/builds export BT_SCREENS=/mnt/screens export BT_PROFILES=/turnkey/tklbam-profiles export BT_PRODUCTS=/turnkey/fab/products + # Some appliances derive latest version to install via GitHub API. To avoid # possible unauthenticated API rate limiting, provide GitHub credentials. # To generate a new GITHUB_USER_TOKEN, please browse to: # https://github.com/settings/tokens export GITHUB_USER=your-github-username export GITHUB_USER_TOKEN=your-github-user-token + +export BASE_DIR=/turnkey/public + +# Default git remote when cloning any needed repos +export GIT_REMOTE_URL="${GIT_REMOTE_URL:-https://github.com/turnkeylinux}" +export GIT_REMOTE_NAME="${GIT_REMOTE_NAME:-origin}" From 2a1d0b9a8ab646fa0947de967dfd3c11f941f044 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Fri, 3 Jul 2026 20:40:01 +1000 Subject: [PATCH 3/5] Update GITHUB bits in config --- config.example/common.cfg | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config.example/common.cfg b/config.example/common.cfg index 102444a..b86940c 100644 --- a/config.example/common.cfg +++ b/config.example/common.cfg @@ -13,12 +13,12 @@ export BT_SCREENS=/mnt/screens export BT_PROFILES=/turnkey/tklbam-profiles export BT_PRODUCTS=/turnkey/fab/products -# Some appliances derive latest version to install via GitHub API. To avoid -# possible unauthenticated API rate limiting, provide GitHub credentials. -# To generate a new GITHUB_USER_TOKEN, please browse to: -# https://github.com/settings/tokens -export GITHUB_USER=your-github-username -export GITHUB_USER_TOKEN=your-github-user-token +# Setting these variables when building appliances is highly recommended. +# See ~/.bashrc.d/github (on TKLDev) for more info and set there to always have +# these in your environment. Alternatively, hard code specific values here for +# use with buildtasks. +export GITHUB_USER="${GITHUB_USER}" +export GITHUB_USER_TOKEN="${GITHUB_USER_TOKEN}" export BASE_DIR=/turnkey/public From fa3c1905ac52934d25598e30527e9b5e075d9cc7 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Fri, 3 Jul 2026 20:40:45 +1000 Subject: [PATCH 4/5] Update tklbam profile generation for v19.x --- bin/generate-tklbam-profile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/generate-tklbam-profile b/bin/generate-tklbam-profile index 0912571..03eb28c 100755 --- a/bin/generate-tklbam-profile +++ b/bin/generate-tklbam-profile @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/lib/tklbam-pypy2/bin/pypy # # Copyright (c) 2010-2011 Liraz Siri # @@ -34,16 +34,16 @@ import re import shutil from StringIO import StringIO -import executil TKLBAM_LIB_PATH = os.environ.get('TKLBAM_LIB_PATH', '/usr/lib/tklbam') sys.path.insert(0, TKLBAM_LIB_PATH) -try: - from temp import TempDir, TempFile - from backup import ProfilePaths +try: # custom TKL imports + from pylib import executil + from pylib.temp import TempDir, TempFile import dirindex + from backup import ProfilePaths except ImportError, e: print >> sys.stderr, \ "Can't find TKLBAM modules in TKLBAM_LIB_PATH=%s\n" % TKLBAM_LIB_PATH From 3d2b0e82b81c5f3370cd77f92a0c10d49bde7a9b Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Fri, 3 Jul 2026 20:49:09 +1000 Subject: [PATCH 5/5] Linting, tidying and reorganising (now does what it says and says what it does...) --- bt-iso | 132 +++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 85 insertions(+), 47 deletions(-) diff --git a/bt-iso b/bt-iso index 27405f3..f1ceff8 100755 --- a/bt-iso +++ b/bt-iso @@ -26,9 +26,12 @@ Arguments:: Options:: -n|--no-screens - skip taking screenshots - -u|--use-existing - use an existing iso, rather than rebuilding, will - fail if iso does not exist + -u|--use-existing - use existing appname/build/product.iso, rather than + rebuilding, will fail if product.iso does not exist + (implies --no-screens) -s|--skip-setup - skip the inital setup step + -a|--arch ARCH - set architecture to build - amd64 or arm64 + (default: host arch) --publish - publish iso, release files and tklbam profile, incompatible with -u|--use-existing and -s|--skip-setup @@ -43,7 +46,7 @@ Environment:: - time (in seconds) to wait between webdriver connection attempts (for screenshots) DEFAULT: $WEBDRIVER_CONNECT_TIMEOUT - BT_DEBUG - turn on debugging + BT_DEBUG - verbose output & do not remove appname/build dir EOF if [[ "$#" -ne 0 ]]; then @@ -63,7 +66,7 @@ clone_or_pull() { if [ ! -e "$dir" ]; then info "Attempting to clone repo $repo to $dir." cd "$(dirname "$dir")" - git clone "https://github.com/$repo" "$dir" + git clone -o "$GIT_REMOTE_NAME" "$GIT_REMOTE_URL/$repo" "$dir" cd "$dir" else info "Repo $repo found, attempting to update" @@ -100,14 +103,12 @@ git_unshallow() { get_codename() { local tkl_ver=$1 case $tkl_ver in - 16*) - TKL_CODENAME=buster;; - 17*) - TKL_CODENAME=bullseye;; 18*) TKL_CODENAME=bookworm;; 19*) TKL_CODENAME=trixie;; + 20*) + TKL_CODENAME=forky;; *) fatal "Unrecognised TKL version: $tkl_ver";; esac @@ -129,54 +130,77 @@ check_debug() { export WEBDRIVER_CONNECT_ATTEMPTS=${WEBDRIVER_CONNECT_ATTEMPTS:-200} export WEBDRIVER_CONNECT_TIMEOUT=${WEBDRIVER_CONNECT_TIMEOUT:-5} ARGS="$*" +arch=$(dpkg --print-architecture) unset no_screens use_existing appname skip_setup TKL_CODENAME while [[ "$1" != "" ]]; do case $1 in - --help|-h) usage;; - --publish) publish="yes";; - -n|--no-screens) no_screens="true";; - -s|--skip-setup) skip_setup="true";; - -u|--use-existing) use_existing="true" - shift - iso="$1";; - *) if [[ -n "$appname" ]]; then - usage "Accepts only one appliance name" - else - appname=$1 - fi;; + --help|-h) + usage + ;; + --publish) + publish="yes";; + -n|--no-screens) + no_screens="true";; + -s|--skip-setup) + skip_setup="true";; + -u|--use-existing) + no_screens="true" + info "Screenshots disabled (using existing product.iso)" + use_existing="true";; + -a|--arch) + shift + arch=$1 + ;; + -*) + fatal "Unrecognized option: $1";; + *) + if [[ -n "$appname" ]]; then + usage "Accepts only one appliance name" + else + appname=$1 + fi;; esac shift done [[ -n "$appname" ]] || usage "Must give one app name" +ISO="$FAB_PATH/$appname/build/product.iso" +if [[ -n "$use_existing" ]] && [[ ! -f "$ISO" ]]; then + fatal "file not found: $ISO" +fi # don't try screens for tkldev & extend MAX_COUNT for gitlab & canvas -if [[ -z "$no_screens" ]]; then - case $appname in - tkldev|core) - info "$appname build - disabling screenshots as no web UI" - no_screens="true";; - gitlab|canvas) - # GitLab & Canvas take ages to start... - export MAX_COUNT=70 # possibly overkill - ;; - esac -fi +case $appname in + tkldev|core) + info "$appname build - disabling screenshots as no web UI" + no_screens="true";; + gitlab|canvas) + # GitLab & Canvas take ages to start... + export MAX_COUNT=70;; # possibly overkill +esac link=$(readlink -f "$0") BT=$(dirname "$link") export BT export BT_CONFIG="$BT/config" -. "$BT_CONFIG/common.cfg" - -INIT_CONF=/turnkey/public/tkldev-docker/inithooks.conf -if [[ ! -f "$INIT_CONF" ]]; then - check_debug "Inithooks preseed file ($INIT_CONF) not found" -else - . "$INIT_CONF" +BT_COMMON_CONF="$BT_CONFIG/common.cfg" +if [[ ! -d "$BT_CONFIG" ]]; then + if [[ -z "$publish" ]]; then + warning "$BT_CONFIG dir does not exist; attempting to use default conf" + cp -R "$BT/config.example" "$BT_CONFIG" + else + fatal "$BT_CONFIG dir does not exist" + fi +elif [[ ! -f "$BT_COMMON_CONF" ]]; then + fatal "$BT_COMMON_CONF file does not exist - see $BT/config.example" fi -if [[ -z "$APP_DOMAIN" ]]; then - check_debug "APP_DOMAIN not set" +# shellcheck source=config.example/common.cfg +source "$BT_CONFIG/common.cfg" + +if [[ -z "$BT_PROFILES" ]]; then + check_debug "BT_PROFILES not set" +elif [[ ! -d "$BT_PROFILES" ]]; then + check_debug "file not found: $BT_PROFILES" fi HOST_IP=$(hostname -I | awk '{print $1}') @@ -186,11 +210,12 @@ if [[ -z "$HOST_IP" ]]; then fi if [[ "$publish" == "yes" ]]; then - unset msg [[ -n "$BT_PUBLISH_IMGS" ]] || fatal "BT_PUBLISH_IMGS not set" [[ -n "$BT_PUBLISH_META" ]] || fatal "BT_PUBLISH_META not set" [[ -n "$BT_PUBLISH_SCREENS" ]] || fatal "BT_PUBLISH_SCREENS not set" [[ -n "$BT_PUBLISH_PROFILES" ]] || fatal "BT_PUBLISH_PROFILES not set" + [[ -n "$GITHUB_USER" ]] || warning "GITHUB_USER not set" + [[ -n "$GITHUB_USER_TOKEN" ]] || warning "GITHUB_USER_TOKEN not set" [[ -z "$use_existing" ]] || usage "conflicting options: --publish / -u|--use-existing" [[ -z "$skip_setup" ]] || usage "conflicting options: --publish / -s|--skip-setup" which aws >/dev/null || "$BT"/bin/aws-setup --awscli @@ -204,6 +229,7 @@ COMMIT_ID=$(git rev-parse --short HEAD) if [[ -z "$skip_setup" ]]; then # unless -n|--no-screens set up clicksnap and check for clicksnap file if [[ -z "$no_screens" ]]; then + which podman >/dev/null || apt-get install -y podman if [[ ! -f /root/clicksnap-setup.done ]]; then info "Setting up clicksnap and dockerize" "$BT"/bin/clicksnap-setup "$appname" @@ -220,7 +246,7 @@ if [[ -z "$skip_setup" ]]; then touch /root/clicksnap-setup.done fi fi - if ! grep -q $appname <<<$(clicksnap list); then + if ! grep -q "$appname" <<<"$(clicksnap list)"; then fatal "Clicksnap code for $appname not found" fi fi @@ -250,11 +276,9 @@ if [[ -z "$skip_setup" ]]; then fi cd - git_unshallow "$FAB_PATH/common" "$TKL_BUILD_VER.x" - # TODO KEY unused!? - KEY=$FAB_PATH/common/keys/tkl-$TKL_CODENAME-images.asc IMAGES="http://mirror.turnkeylinux.org/turnkeylinux/images" - BOOTSTRAP_NAME="bootstrap-$TKL_CODENAME-$(dpkg --print-architecture)" - BOOTSTRAP_PATH="$FAB_PATH/bootstraps/$TKL_CODENAME" + BOOTSTRAP_NAME="bootstrap-$TKL_CODENAME-$arch" + BOOTSTRAP_PATH="$FAB_PATH/bootstraps/$TKL_CODENAME-$arch" if [[ ! -d "$BOOTSTRAP_PATH" ]]; then info "Attempting to download $BOOTSTRAP_NAME" @@ -294,6 +318,20 @@ if [[ -z "$skip_setup" ]]; then fi fi +if [[ -z "$no_screens" ]]; then + init_conf=/turnkey/public/tkldev-docker/inithooks.conf + if [[ ! -f "$init_conf" ]]; then + check_debug "Inithooks preseed file ($init_conf) not found" + else + # shellcheck source=/turnkey/public/tkldev-docker/inithooks.conf + source "$init_conf" + fi + if [[ -z "$APP_DOMAIN" ]]; then + check_debug "APP_DOMAIN not set" + fi +fi + + cd "$BT_PRODUCTS/$appname" if [[ -n "$use_existing" ]]; then ISO=build/product.iso @@ -371,7 +409,7 @@ if [[ -z "$no_screens" ]]; then sleep 1 done info "container running - waiting for connection" - for ping in {1..20}; do + for _ping in {1..20}; do if ! curl localhost:7900 >/dev/null 2>&1; then sleep 1 fi