#!/usr/bin/env bash
#
# The `diagnostic` machine action: everything the morning of the 31st needed,
# in one file, from one button.
#
# That morning cost a keyboard carried to the television, a monitor unplugged
# from something else, and an SSH session - to read three files. The kiosk has
# no keyboard and no console by design, so the moment something is wrong the
# only way to see anything at all is to undo the design. This is the way back
# in that does not: the console page asks for it, the root side gathers it, and
# it lands in the journals list the page already shows, as plain text a browser
# opens and a phone can read.
#
# **What it is allowed to gather.** Everything below is either world-readable
# already or a systemd journal, and the file it writes is served by
# `/api/logs/<name>` to anyone who can reach the page - the same posture the
# server's own log has had since issue #5. So two rules hold here:
#
#   * nothing is read *because* root can read it. Every source below is one an
#     administrator would look at, not a private key or a shadow file;
#   * anything that could carry the wireless passphrase is put through
#     `redact` on the way out. hostapd prints its configuration into the
#     journal when it starts, and a diagnostic that publishes the WiFi
#     password to every phone in the room would be a worse morning than the
#     one it is here to prevent. Since the forge section prints a **JSON**
#     document, `redact` also knows the quoted form: `"password": "x"` walked
#     straight past rules written for `password = x`. Nothing in
#     `settings.json` carries a secret today - a forge URL with credentials in
#     it is refused where the file is parsed, and the machine half is one word
#     per key - so that rule is there for the keys this file has not got yet.
#
# **It never fails.** Every source is best-effort: a machine without a game
# log, without pacman, without a camera still produces a file, with the gaps
# named. An action that dies halfway leaves the page with a failure and the
# person in front of it with nothing, which is exactly the position this
# exists to end.

set -euo pipefail

# shellcheck source=/dev/null
source "${ARCHSTAR_ACTIONS_LIB:-/usr/lib/archstar-actions/lib/common.sh}"

# Where the page lists journals from. The server reads the same default out of
# ARCHSTAR_LOGS_DIR (config.py); the unit sets it on machines that moved it.
JOURNALS="${ARCHSTAR_LOGS_DIR:-/srv/karaoke/logs}"

# `python3 -m archstar_web.forge_report` needs the package on the path. The
# actions unit sets PYTHONPATH and this script is its child, so it is inherited
# - named here anyway, because a default that only holds by inheritance is one
# that breaks the day somebody runs this by hand.
export PYTHONPATH="${PYTHONPATH:-/usr/lib/archstar-web}"

# How many of these to keep. The server's own pruning only knows the
# `<source>-<kind>-` families a phone uploads, so this one prunes itself -
# otherwise a button somebody presses while waiting fills the partition.
KEEP=10

# The whole file, capped. Big enough for two boots of kiosk journal, small
# enough to open on a telephone over the hotspot.
MAX_BYTES=$((2 * 1024 * 1024))

# Lines of pacman.log. The question it answers is "what changed just before
# this started", and that is always the last few days.
PACMAN_LINES=300

# Lines per journal. Two boots of a kiosk that restarts on failure is the
# shape of an evening going wrong.
JOURNAL_LINES=800

# --- Gathering -----------------------------------------------------------------

# A heading, so the file can be read by somebody scrolling rather than
# searching.
section() { printf '\n\n===== %s =====\n' "$*"; }

# Run something and let it fail. `|| true` at every call site would be the same
# rule written thirty times, and the thirtieth would be the one forgotten.
try() {
    if ! "$@" 2>&1; then
        printf '(%s: nothing, or not available on this machine)\n' "$1"
    fi
}

# Print a file, or say it is not there. Not `cat` alone: a missing game log is
# itself an answer - it means the game never got far enough to write one.
show_file() {
    local path="$1" lines="${2:-0}"
    if [[ ! -r "${path}" ]]; then
        printf '(%s: not present)\n' "${path}"
        return 0
    fi
    if (( lines > 0 )); then
        tail -n "${lines}" "${path}" || true
    else
        cat "${path}" || true
    fi
}

# The one filter between this file and the network. Anything that looks like a
# secret being assigned keeps its name and loses its value, so the diagnostic
# still says "there is a passphrase configured" without saying what it is.
redact() {
    sed -E \
        -e 's/((passphrase|password|passwd|psk|secret|wpa_passphrase|pre-shared-key)[[:space:]]*[=:][[:space:]]*).*/\1[redacted]/I' \
        -e 's/(WIFI:[^[:space:]]*;P:)[^;[:space:]]*/\1[redacted]/g' \
        -e 's/("[A-Za-z0-9_.-]*(passphrase|password|passwd|psk|secret|token|api_?key)[A-Za-z0-9_.-]*"[[:space:]]*:[[:space:]]*)"[^"]*"/\1"[redacted]"/Ig'
}

gather() {
    printf 'ArchStar diagnostic\n'
    printf 'taken: %s\n' "$(date -u '+%Y-%m-%d %H:%M:%S UTC')"
    printf 'host:  %s\n' "$(uname -sr)"
    printf 'up:    %s\n' "$(uptime -p 2>/dev/null || true)"

    section "what is installed"
    show_file /etc/archstar-release
    printf '\n'
    # Ours first, then the four that decide whether anything is drawn at all.
    try pacman -Q archstar-web archstar-theme archstar-clamav-db ultrastardx
    printf '\n'
    try pacman -Q linux linux-firmware mesa cage wlroots sdl2 sdl2-compat

    section "what changed recently (pacman.log, last ${PACMAN_LINES} lines)"
    show_file /var/log/pacman.log "${PACMAN_LINES}"

    # The suspect of the 31st, and the machine's answer to it (B1). A
    # `[Graphics]` section that disagrees with game-graphics.conf means the
    # pre-start hook did not run, which is the first thing to know.
    section "the game's graphics settings"
    printf -- '--- /etc/archstar/game-graphics.conf (the machine decides) ---\n'
    show_file /etc/archstar/game-graphics.conf
    local home
    home="$(getent passwd "$(kiosk_user)" | cut -d: -f6)"
    printf -- '\n--- %s/.ultrastardx/config.ini, [Graphics] only ---\n' "${home}"
    if [[ -r "${home}/.ultrastardx/config.ini" ]]; then
        awk '/^\[/{inside = ($0 == "[Graphics]")} inside' \
            "${home}/.ultrastardx/config.ini" || true
    else
        printf '(no config.ini: the game has never run as this account)\n'
    fi

    # What the screen says about itself. Every one of these was typed by hand
    # during the incident, at a keyboard that had to be found first.
    section "the screen"
    local card
    for card in /sys/class/drm/card*-*; do
        [[ -d "${card}" ]] || continue
        printf '%s: status=%s enabled=%s\n' \
            "${card##*/}" \
            "$(cat "${card}/status" 2>/dev/null || echo '?')" \
            "$(cat "${card}/enabled" 2>/dev/null || echo '?')"
        if [[ -r "${card}/modes" ]]; then
            printf '  modes: %s\n' "$(head -n 3 "${card}/modes" | tr '\n' ' ')"
        fi
    done
    printf '\n'
    try journalctl -b -k --no-pager -g 'drm|amdgpu|i915|nouveau' -n 60

    # The camera, and the two halves of being allowed to open it. None of this
    # was gathered until a P2 with a USB camera plugged into it answered "the
    # camera gave nothing" to every press of the back office's photo button:
    # the node was there, the account could read it, and the service's own
    # device policy - which names the classes it may open, and named only ALSA
    # - was refusing it. A dump that showed the journal and not the policy
    # could say that the camera failed and nothing about why.
    section "the camera"
    if compgen -G '/dev/video*' >/dev/null; then
        try ls -l /dev/video*
    else
        printf '(no /dev/video*: nothing is plugged in, or no driver bound to it)\n'
    fi
    printf '\n'
    # The class name `DeviceAllow=char-video4linux` is resolved through, at the
    # moment the unit starts. No line here means that rule allowed nothing.
    printf -- '--- /proc/devices, the two classes the service names ---\n'
    grep -E '[[:space:]](video4linux|alsa)$' /proc/devices \
        || printf '(neither class is registered: no camera driver, no sound driver)\n'
    printf '\n'
    printf -- '--- what the service is allowed to open, as it is running ---\n'
    try systemctl show archstar-web \
        -p User -p DevicePolicy -p DeviceAllow -p SupplementaryGroups
    printf '\n'
    printf -- '--- and what groups that account is actually in ---\n'
    try id "$(kiosk_user)"
    printf '\n'
    # Best-effort, both of them: v4l2-ctl belongs to a package this appliance
    # does not install, and a machine with no lsusb is still a machine.
    try v4l2-ctl --list-devices
    printf '\n'
    try lsusb

    # The feature that is not on this machine at all, and therefore the one a
    # dump cannot show by reading this machine's own journals. A generation
    # that failed on 12/09 could only be explained by finding four of our
    # lines between two hundred of the game's, and everything that would have
    # answered it in one glance - the document the operator edits, the
    # verdicts, the job table, what the download left on the disk - was
    # gathered nowhere. `forge_report` reads all of it off the disk and asks
    # the two machines directly, so it still answers when the service is the
    # thing that is wrong.
    section "the forge"
    try python3 -m archstar_web.forge_report

    section "the forge in the journal, this boot"
    # Ours only. The service's journal is mostly the roster it offers the game
    # every thirty seconds, and the forge's own lines are a handful in the
    # middle of it - which is exactly why they were missed.
    try journalctl -b -u archstar-web --no-pager -n "${JOURNAL_LINES}" \
        -g 'forge|settings|Traceback|NameError|Exception in thread'

    section "units"
    try systemctl --failed --no-pager --no-legend
    printf '\n'
    local unit
    for unit in "archstar-kiosk@$(kiosk_tty)" archstar-web archstar-actions \
                archstar-hotspot archstar-ftp archstar-gamepad; do
        printf -- '--- %s ---\n' "${unit}"
        try systemctl status --no-pager --lines=0 "${unit}"
        printf '\n'
    done

    section "the kiosk, this boot (last ${JOURNAL_LINES} lines)"
    try journalctl -b -u "archstar-kiosk@$(kiosk_tty)" --no-pager -n "${JOURNAL_LINES}"

    section "the kiosk, the boot before (last ${JOURNAL_LINES} lines)"
    # The one that matters when the machine has been restarted since: the
    # evening it broke is the boot before the one somebody is looking from.
    try journalctl -b -1 -u "archstar-kiosk@$(kiosk_tty)" --no-pager -n "${JOURNAL_LINES}"

    section "the service, this boot (last ${JOURNAL_LINES} lines)"
    try journalctl -b -u archstar-web --no-pager -n "${JOURNAL_LINES}"

    section "the game's own log"
    local log found=0
    for log in "${home}/.ultrastardx"/*.log "${home}/.ultrastardx/logs"/*.log; do
        [[ -r "${log}" ]] || continue
        found=1
        printf -- '--- %s ---\n' "${log}"
        tail -n 200 "${log}" || true
        printf '\n'
    done
    (( found )) || printf '(the game has written no log)\n'

    section "disk"
    try df -h /srv/karaoke / /var
    printf '\n'
    try findmnt -no SOURCE,TARGET,FSTYPE,OPTIONS /srv/karaoke
}

# --- Writing it down -------------------------------------------------------------

main() {
    local stamp name path
    stamp="$(date -u '+%Y%m%d-%H%M%S')"
    name="diagnostic-${stamp}.log"
    path="${JOURNALS}/${name}"

    mkdir -p "${JOURNALS}"

    # Written under a leading dot and renamed: `LogStore.catalogue` skips
    # dotfiles precisely so a half-written journal is never offered, and this
    # one takes seconds to gather.
    local temporary="${JOURNALS}/.${name}"
    gather 2>&1 | redact | head -c "${MAX_BYTES}" > "${temporary}"
    # The page serves this to whoever asks; it is readable, not private, and
    # saying so here is better than inheriting root's umask by accident.
    chmod 0644 "${temporary}"
    mv "${temporary}" "${path}"

    # Keep the newest few. The server prunes the families a phone uploads and
    # knows nothing about this one.
    local stale
    while IFS= read -r stale; do
        rm -f "${stale}"
    done < <(find "${JOURNALS}" -maxdepth 1 -name 'diagnostic-*.log' -type f \
             -printf '%T@ %p\n' 2>/dev/null | sort -rn | tail -n +$((KEEP + 1)) \
             | cut -d' ' -f2-)

    # The last line is what the page shows as the verdict, and it is the file
    # name on purpose: the journals panel is right underneath, and that is what
    # to click.
    printf 'gathered %s (%s)\n' "${name}" \
        "$(du -h "${path}" 2>/dev/null | cut -f1 || echo '?')"
}

main "$@"
