#!/usr/bin/env bash
# Trim the ecd-runner ~/.cache volume (uv + pip caches) on hzdev.
#
# The cache volume holds the deduped wheel/download set and is mostly stable,
# but it can creep over time. Trim it only once it exceeds a threshold so
# installs normally stay warm. Invoked by ecd-runner-cleanup.timer; safe to run
# by hand.
set -uo pipefail

# Cache is only trimmed once it exceeds this size (GB), to keep installs warm.
CACHE_MAX_GB="${CACHE_MAX_GB:-5}"
VOLUME_DATA="/var/lib/docker/volumes/ecd-runner-cache/_data"
# Same flag the supervisors watch: while it exists they launch no new containers.
PAUSE_FLAG="${PAUSE_FLAG:-/run/ecd-runner-paused}"
ENV_FILE="${ENV_FILE:-/etc/ecd-runner/runner.env}"

echo "[cleanup] $(date -u +%FT%TZ) starting"

if [[ ! -d "${VOLUME_DATA}" ]]; then
  echo "[cleanup] no cache volume; done"
  exit 0
fi

size_gb=$(du -s --block-size=1G "${VOLUME_DATA}" 2>/dev/null | cut -f1)
size_gb=${size_gb:-0}
if (( size_gb < CACHE_MAX_GB )); then
  echo "[cleanup] cache ${size_gb}G < ${CACHE_MAX_GB}G threshold; leaving warm"
  exit 0
fi

# Destructive phase. The wipe must never race a job, and any "check for a job,
# then act" scheme is a TOCTOU race — an idle slot can accept work between the
# check and the act. The only atomic primitive available is GitHub's: DELETE on
# a runner registration FAILS if that runner is busy. So (same protocol as the
# api-runner family):
#   a. touch the pause flag — supervisors stop launching/registering;
#   b. deregister every ecd-hzdev runner via the API — a busy runner makes the
#      DELETE fail, and we abort the whole phase without having killed anything
#      (deregistered idle runners simply re-register on resume);
#   c. only when zero runners remain registered (nothing can hold or receive a
#      job) wipe the volume;
#   d. the EXIT trap removes the flag and the supervisors resume.
source "${ENV_FILE}" || { echo "[cleanup] cannot read ${ENV_FILE}; skipping destructive phase" >&2; exit 1; }
RUNNERS_API="https://api.github.com/repos/${REPO}/actions/runners"

# Authorization via stdin (-H @-), never argv, so the PAT is not exposed in
# /proc/*/cmdline to other users on the host.
gh_api() {
  printf 'Authorization: Bearer %s\n' "${GITHUB_PAT}" | curl -fsSL \
    --connect-timeout 10 --max-time 30 \
    -H @- \
    -H "Accept: application/vnd.github+json" \
    -H "X-GitHub-Api-Version: 2022-11-28" "$@"
}

# Prints "id busy name" per registered ecd-hzdev runner. python3 (host) instead
# of jq: hzdev's jq is the snap build, whose AppArmor exec transition misbehaves
# under systemd sandboxing (same reason the supervisor avoids it).
list_runners() {
  gh_api "${RUNNERS_API}?per_page=100" | python3 -c '
import json, sys
for r in json.load(sys.stdin).get("runners", []):
    if r["name"].startswith("ecd-hzdev-"):
        print(r["id"], str(r["busy"]).lower(), r["name"])
'
}

# Fail fast if the flag can't be created: without it the supervisors would keep
# launching fresh listeners during the drain/wipe window.
touch "${PAUSE_FLAG}" || { echo "[cleanup] cannot create ${PAUSE_FLAG}; aborting" >&2; exit 1; }
trap 'rm -f "${PAUSE_FLAG}"' EXIT
echo "[cleanup] paused supervisors via ${PAUSE_FLAG}"
# Let any slot that was mid-launch when the flag appeared finish registering, so
# the listing below sees it too.
sleep 5

# Deregister until none remain. Loops because a launch in flight at flag time
# can register one more runner after our first listing; converges after.
deregistered_all=false
for _ in 1 2 3; do
  runners="$(list_runners)"
  if [[ $? -ne 0 ]]; then
    echo "[cleanup] failed to list runners; skipping destructive phase this run" >&2
    exit 0
  fi
  if [[ -z "${runners}" ]]; then
    deregistered_all=true
    break
  fi
  while read -r id busy name; do
    if [[ "${busy}" == "true" ]]; then
      echo "[cleanup] ${name} is running a job; skipping destructive phase this run"
      exit 0
    fi
    if gh_api -X DELETE "${RUNNERS_API}/${id}" >/dev/null; then
      echo "[cleanup] deregistered idle runner ${name}"
    else
      # 422 = became busy between listing and delete. Nothing was harmed; the
      # already-deregistered runners re-register once the flag is removed.
      echo "[cleanup] could not deregister ${name} (likely just went busy); skipping destructive phase"
      exit 0
    fi
  done <<< "${runners}"
  sleep 3
done
if [[ "${deregistered_all}" != "true" ]]; then
  echo "[cleanup] runners still registered after 3 rounds; skipping destructive phase" >&2
  exit 0
fi

# Nothing is registered, so no container can hold or receive a job. Deregistered
# listeners exit on their own (docker run --rm cleans up); force-remove
# stragglers so the cache volume is unmounted before the wipe.
for _ in $(seq 1 30); do
  docker ps --format '{{.Names}}' | grep -q '^ecd-runner-[0-9]' || break
  sleep 2
done
leftover=$(docker ps --format '{{.Names}}' | grep '^ecd-runner-[0-9]' || true)
if [[ -n "${leftover}" ]]; then
  echo "[cleanup] removing deregistered leftover containers: ${leftover}"
  echo "${leftover}" | xargs -r docker rm -f >/dev/null 2>&1 || true
fi

# Final mount-safety check: refuse to wipe if docker ps itself fails (the wait
# loop above would have treated that as "no containers") or a runner container
# is somehow still present holding the volume mount.
if ! ps_out=$(docker ps --format '{{.Names}}'); then
  echo "[cleanup] docker ps failed; skipping wipe this run" >&2
  exit 0
fi
if grep -q '^ecd-runner-[0-9]' <<<"${ps_out}"; then
  echo "[cleanup] runner containers still present; skipping wipe this run" >&2
  exit 0
fi

find "${VOLUME_DATA:?}" -mindepth 1 -delete \
  && echo "[cleanup] cache emptied (was ${size_gb}G)" \
  || echo "[cleanup] failed to empty cache" >&2

echo "[cleanup] resuming supervisors (removing ${PAUSE_FLAG})"
# Trap also handles this on every earlier exit path; explicit here for the log.
rm -f "${PAUSE_FLAG}"
trap - EXIT

echo "[cleanup] done"
