{
# Install one release of the fleet API on its box. Run by deploy/deploy-from-pc.sh, which sends, in front of this file,
# the target's settings (deploy/targets/<target>.sh) and SHA, SUM, DEPLOYER, CUTOVER; the archive (git archive of the
# commit) is already in /tmp.
#
# Layout: $BASE/releases/<sha>/ one folder per commit; $BASE/shared/.env the only .env; every release links
# storage -> $STORAGE, .env -> $BASE/shared/.env, public/storage -> $STORAGE/app/public; $APP (the path Apache, the cron,
# supervisor and the scripts use) is a link to the live release. No git on the box.
#
# Steps: checks; unpack; links; vendor (copied from the live release when composer.lock did not change, then composer
# install); the new folder must boot; migrations if pending (dump first, maintenance during migrate + switch; a
# pending migration holding the word ONE_WAY stops the deploy); switch the link; reload php-fpm, queue:restart, reverb
# (RELOAD=no: queue:restart only); probes; if a probe fails, the previous release is put back. Never config:cache,
# never config:clear.
#
# CUTOVER=yes (once per box): $APP is still the old git tree; it becomes $BASE/releases/legacy-<date> and $APP the link.
# Way back from a cutover: link $APP to that legacy folder (the line printed at the end), it runs from there as before.
set -euo pipefail
: "${SHA:?}" "${SUM:?}" "${TARGET:?}" "${APP:?}" "${BASE:?}" "${STORAGE:?}" "${OWNER:?}" "${PHP:?}" "${FPM:?}" "${URL:?}"
: "${PROBES:?}" "${EXPECT:?}" "${QRESTART_USER:?}"
CUTOVER=${CUTOVER:-no}
REVERB=${REVERB:-}
DUMP=${DUMP:-no}
BACKUPS=${BACKUPS:-/var/backups/fleet_backups}
KEEP=${KEEP:-5}
ENV_MODE=${ENV_MODE:-640}
COMPOSER=${COMPOSER:-composer}
TGZ=${TGZ:-/tmp/fleet-$SHA.tgz}
RELOAD=${RELOAD:-yes}
# Without a php-fpm reload its workers keep the old folder until their realpath cache expires (120 s): probe after it.
if [ "$RELOAD" = no ]; then SLEEP_PROBE=${SLEEP_PROBE:-125}; else SLEEP_PROBE=${SLEEP_PROBE:-3}; fi
SLEEP_ERRORS=${SLEEP_ERRORS:-20}
REL=$BASE/releases/$SHA
STAMP=$(date -u +%Y%m%d_%H%M%S)
say() { printf '\n=== %s\n' "$*"; }
stop() { printf 'STOP: %s\n' "$*"; exit 1; }
art() { (cd "$1" && shift && sudo -u www-data "$PHP" artisan "$@"); }
code() { curl -s -m 20 -o /dev/null -w '%{http_code}' -H 'Accept: application/json' "$URL$1"; }
# A probe passes when it answers $EXPECT, or what it answered before the deploy (a route missing before and after
# is a wrong probe, not a broken release); at least one probe must answer $EXPECT.
declare -A BEFORE_CODE
probes_before() {
    local p c
    for p in $PROBES; do
        c=$(code "$p"); BEFORE_CODE[$p]=$c; echo "  $p -> $c"
    done
}
probes_ok() {
    local ok=0 up=no p c b
    for p in $PROBES; do
        c=$(code "$p"); b=${BEFORE_CODE[$p]:-}
        if [ "$c" = "$EXPECT" ]; then up=yes; echo "  $p -> $c"
        elif [ "$c" = "$b" ]; then echo "  $p -> $c (as before the deploy)"
        else echo "  $p -> $c (before: ${b:-?}, expected $EXPECT): FAILED"; ok=1
        fi
    done
    [ "$up" = yes ] || { echo "  no probe answered $EXPECT"; ok=1; }
    return $ok
}
routes() { art "$1" route:list --json 2>/dev/null | "${PYTHON:-python3}" -c 'import json,sys; print(len(json.load(sys.stdin)))' 2>/dev/null || echo 0; }

say "1. checks ($TARGET, $(hostname), $(date -u '+%F %T') UTC)"
[[ "$SHA" =~ ^[0-9a-f]{40}$ ]] || stop "SHA is not a commit id: $SHA"
[[ "$SUM" =~ ^[0-9a-f]{64}$ ]] || stop "SUM is not a sha256: $SUM"
[ -f "$TGZ" ] || stop "no $TGZ"
echo "$SUM  $TGZ" | sha256sum -c - >/dev/null || stop "the archive's sum differs, nothing installed"
if [ "$CUTOVER" = yes ] && [ -d "$APP/storage" ] && [ ! -L "$APP/storage" ] && [ ! -e "$STORAGE" ]; then
    # A tree that still keeps storage inside it (as prod did until 2026-08-12): moved out once, linked back.
    sudo mv -T "$APP/storage" "$STORAGE" && sudo ln -s "$STORAGE" "$APP/storage" || stop "cutover: could not move $APP/storage to $STORAGE"
    echo "cutover: $APP/storage moved to $STORAGE and linked back"
fi
[ -d "$STORAGE/logs" ] || stop "$STORAGE/logs missing"
if [ "$CUTOVER" = yes ]; then
    [ -d "$APP" ] && [ ! -L "$APP" ] || stop "cutover: $APP must be the old folder, it is not"
    PREV=$APP
    [ "$(readlink -e "$APP/storage")" = "$(readlink -e "$STORAGE")" ] || stop "cutover: $APP/storage is not $STORAGE"
    sudo test -f "$APP/.env" || stop "cutover: no $APP/.env"
else
    [ -L "$APP" ] || stop "$APP is not a link: run the cutover first"
    PREV=$(readlink -e "$APP" || true)
    case "$PREV" in "$BASE/releases/"?*) ;; *) stop "$APP points outside $BASE/releases: ${PREV:-nothing}" ;; esac
    [ "$PREV" != "$REL" ] || stop "$SHA is already the live release"
    sudo test -f "$BASE/shared/.env" || stop "no $BASE/shared/.env"
fi
echo "live: $PREV"
echo "new:  $REL"
[ -e "$PREV/bootstrap/cache/config.php" ] && stop "config is cached on the live release: ask Mhamed, never clear it"
pgrep -af 'mysqldump|db_backup|storage_backup' >/dev/null && stop "a backup is running"
FREE=$(df --output=avail -BM "$(dirname "$BASE")" | tail -1 | tr -dc 0-9)
[ "${FREE:-0}" -ge 2048 ] || stop "less than 2 GB free"
LL=$STORAGE/logs/laravel-$(date -u +%F).log
E0=$(sudo grep -c '\.ERROR:' "$LL" 2>/dev/null || true); E0=${E0:-0}
echo "probes before:"; probes_before

say "2. unpack"
sudo install -d -o "$OWNER" -g www-data -m 755 "$BASE" "$BASE/releases" "$BASE/shared"
sudo rm -rf "$REL.tmp"
[ -e "$REL" ] && [ "$REL" != "$PREV" ] && sudo rm -rf "$REL"
sudo install -d -o "$OWNER" -g www-data -m 755 "$REL.tmp"
sudo -u "$OWNER" tar -xzf "$TGZ" -C "$REL.tmp" --no-same-owner
[ -f "$REL.tmp/artisan" ] && [ -f "$REL.tmp/composer.lock" ] && [ -f "$REL.tmp/bootstrap/app.php" ] || stop "archive incomplete"

say "3. shared .env, storage, public/storage"
if [ "$CUTOVER" = yes ] && ! sudo test -f "$BASE/shared/.env"; then
    sudo cp -a "$APP/.env" "$BASE/shared/.env"
    echo ".env copied from $APP to $BASE/shared/.env"
fi
sudo chown "$OWNER:www-data" "$BASE/shared/.env"
sudo chmod "$ENV_MODE" "$BASE/shared/.env"
sudo stat -c '  shared .env: %U:%G %a, %s bytes' "$BASE/shared/.env"
sudo -u "$OWNER" rm -rf "$REL.tmp/storage" "$REL.tmp/public/storage"
sudo -u "$OWNER" ln -s "$STORAGE" "$REL.tmp/storage"
sudo -u "$OWNER" ln -s "$BASE/shared/.env" "$REL.tmp/.env"
sudo -u "$OWNER" ln -s "$STORAGE/app/public" "$REL.tmp/public/storage"
sudo install -d -o "$OWNER" -g www-data -m 2775 "$REL.tmp/bootstrap/cache"
sudo rm -f "$REL.tmp/bootstrap/cache/"*.php

say "4. vendor"
HAVE_COMPOSER=no
sudo -u "$OWNER" -H "$COMPOSER" --version >/dev/null 2>&1 && HAVE_COMPOSER=yes
# Read with sudo: the live tree's files may not be readable by the deploying user.
if sudo test -f "$PREV/composer.lock" && sudo cmp -s "$PREV/composer.lock" "$REL.tmp/composer.lock" && sudo test -d "$PREV/vendor"; then
    sudo -u "$OWNER" cp -a "$PREV/vendor" "$REL.tmp/vendor"
    echo "copied from the live release (composer.lock unchanged)"
    # Same packages: composer only rebuilds the autoloader; without it, new classes load through PSR-4 all the same.
    [ "$HAVE_COMPOSER" = yes ] || echo "no $COMPOSER on the box: the copied vendor is used as it is"
else
    [ "$HAVE_COMPOSER" = yes ] || stop "composer.lock changed and $COMPOSER is not on the box: nothing switched"
    echo "composer.lock changed: full install"
fi
[ "$HAVE_COMPOSER" = no ] || (cd "$REL.tmp" && sudo -u "$OWNER" -H "$COMPOSER" install --no-dev --prefer-dist --no-interaction --no-progress --optimize-autoloader 2>&1 | tail -n 3)
[ -f "$REL.tmp/vendor/autoload.php" ] || stop "vendor not installed"
art "$REL.tmp" package:discover --no-ansi | tail -n 1

say "5. the new folder boots"
echo "  $(art "$REL.tmp" --version)"
NR=$(routes "$REL.tmp"); PR=$(routes "$PREV")
echo "  routes: new $NR, live $PR"
[ "${NR:-0}" -gt 0 ] || stop "the new folder lists no route"
[ -e "$REL.tmp/bootstrap/cache/config.php" ] && stop "config.php appeared in the new folder: never cache the config"
ls "$REL.tmp"/bootstrap/cache/routes-v*.php >/dev/null 2>&1 && stop "routes are cached in the new folder"

say "6. migrations"
# A database that does not answer must stop the deploy, not read as « none pending ».
MS=$(art "$REL.tmp" migrate:status --no-ansi 2>&1) || { echo "$MS" | tail -n 5; stop "cannot read the migrations' state (the database?): nothing switched"; }
PENDING=$(echo "$MS" | grep 'Pending' | awk '{print $1}' || true)
MIGRATED=no
FD=""
if [ -z "$PENDING" ]; then
    echo "none pending"
else
    echo "pending:"; echo "$PENDING" | sed 's/^/  /'
    [ "$CUTOVER" = yes ] && stop "cutover with pending migrations: deploy them first the old way, or cut over on the live code"
    for m in $PENDING; do
        f=$(ls "$REL.tmp"/database/migrations/"$m".php 2>/dev/null || true)
        [ -n "$f" ] && grep -q 'ONE_WAY' "$f" && stop "$m is marked ONE_WAY: ask Mhamed before it runs"
    done
    if [ "$DUMP" = yes ]; then
        envv() { sudo grep -m1 "^$1=" "$BASE/shared/.env" | cut -d= -f2- | sed -e 's/\r$//' -e 's/^"\(.*\)"$/\1/' -e "s/^'\(.*\)'$/\1/"; }
        DBH=$(envv DB_HOST); DBP=$(envv DB_PORT); DBN=$(envv DB_DATABASE); DBU=$(envv DB_USERNAME); PW=$(envv DB_PASSWORD)
        [ -n "$DBH" ] && [ -n "$DBN" ] && [ -n "$DBU" ] || stop "DB settings not found in the shared .env"
        PW=${PW//\\/\\\\}; PW=${PW//\"/\\\"}
        CNF=$(sudo mktemp /root/.release-dump-XXXXXX.cnf)
        trap 'sudo rm -f "$CNF"' EXIT
        printf '[client]\nhost=%s\nport=%s\nuser=%s\npassword="%s"\n' "$DBH" "${DBP:-3306}" "$DBU" "$PW" | sudo tee "$CNF" >/dev/null
        unset PW
        TABLES=$(sudo mysql --defaults-extra-file="$CNF" -N -e "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='$DBN' AND table_type='BASE TABLE'") || stop "cannot reach the database"
        HAS_VDP=$(sudo mysql --defaults-extra-file="$CNF" -N -e "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='$DBN' AND table_name='vehicle_data_points'")
        sudo install -d -m 750 "$BACKUPS"
        FD="$BACKUPS/dbbackup-release-${SHA:0:9}-$STAMP-sans-vdp.sql.gz"
        # Root only: the dump holds every client's data, tee would create it 644 (SCRUM-548, QA F35).
        sudo install -m 600 /dev/null "$FD"
        S0=$(date +%s)
        sudo mysqldump --defaults-extra-file="$CNF" --single-transaction --quick --no-tablespaces --column-statistics=0 \
            --add-drop-table --ignore-table="$DBN.vehicle_data_points" "$DBN" | gzip -c | sudo tee "$FD" >/dev/null
        RC=("${PIPESTATUS[@]}")
        [ "${RC[0]}" = 0 ] && [ "${RC[1]}" = 0 ] && [ "${RC[2]}" = 0 ] || stop "dump failed (mysqldump ${RC[0]}, gzip ${RC[1]}, tee ${RC[2]})"
        sudo gzip -t "$FD" || stop "the dump is not a valid gzip"
        FE=$(sudo sh -c "zcat '$FD' | tail -c 200" | tail -1)
        case "$FE" in "-- Dump completed"*) ;; *) stop "the dump is incomplete" ;; esac
        NC=$(sudo sh -c "zcat '$FD' | grep -c '^CREATE TABLE '")
        [ "$NC" -eq $((TABLES - HAS_VDP)) ] || stop "the dump holds $NC tables, $((TABLES - HAS_VDP)) expected"
        echo "dump: $FD ($(sudo stat -c %s "$FD") bytes, $(( $(date +%s) - S0 )) s, $NC tables)"
        sudo rm -f "$CNF"
    fi
    art "$PREV" down --retry=60 || stop "could not enter maintenance mode"
    if ! art "$REL.tmp" migrate --force; then
        art "$PREV" up || true
        stop "migration failed: the live release serves again${FD:+; dump $FD}"
    fi
    MIGRATED=yes
fi

say "7. switch"
sudo chown -R -P "$OWNER:www-data" "$REL.tmp"
sudo chmod -R g+w "$REL.tmp/bootstrap/cache"
sudo mv -T "$REL.tmp" "$REL"
# Reload php-fpm (its realpath cache keeps the old folder up to 120 s), restart the workers and reverb on the new path.
# RELOAD=no leaves php-fpm and reverb alone: a reload in busy hours saturated the pool (2026-11-02), and reverb's
# restart makes every client reconnect at once. The cron and the queue workers run the new folder either way.
after_switch() {
    [ "$RELOAD" = no ] || sudo systemctl reload "$FPM"
    (cd "$APP" && sudo -u "$QRESTART_USER" "$PHP" artisan queue:restart >/dev/null) || echo "  queue:restart failed"
    [ "$RELOAD" = no ] || [ -z "$REVERB" ] || sudo supervisorctl restart "$REVERB" >/dev/null || echo "  reverb restart failed"
}
# A link replaced by rename: requests see the old folder or the new one, never nothing. Going back after a cutover
# links $APP to the legacy folder the same way (it runs from there as it did).
switch_to() { sudo ln -sfn "$1" "$APP.new" && sudo mv -T "$APP.new" "$APP" && after_switch; }
if [ "$CUTOVER" = yes ]; then
    LEGACY=$BASE/releases/legacy-$STAMP
    # A folder cannot be renamed over: moved aside, then the link, in one call (no request in between in practice).
    sudo sh -c "mv -T '$APP' '$LEGACY' && ln -s '$REL' '$APP'" || stop "cutover switch failed: check $APP and $LEGACY now"
    PREV=$LEGACY
    after_switch
    echo "old folder kept as $LEGACY"
else
    switch_to "$REL"
fi
[ "$MIGRATED" = yes ] && { art "$APP" up || echo "  could not leave maintenance mode: run artisan up"; }
echo "$APP -> $(readlink -e "$APP") at $(date -u '+%F %T') UTC"

say "8. probes"
[ "$RELOAD" = yes ] || echo "php-fpm not reloaded: probes in $SLEEP_PROBE s, once its workers have left the old folder"
sleep "$SLEEP_PROBE"
if ! probes_ok; then
    echo "a probe failed: back to $PREV"
    switch_to "$PREV"
    echo "$APP -> $(readlink -e "$APP")"
    probes_ok || echo "  the previous release answers differently too: look now"
    [ "$MIGRATED" = yes ] && echo "migrations ran and stay (the older code ignores new tables and columns)${FD:+; dump $FD}"
    echo "$(date -u '+%F %T') $SHA ${DEPLOYER:-?} ROLLED BACK to ${PREV##*/}" | sudo tee -a "$BASE/deploys.log" >/dev/null
    stop "the new release did not answer as expected; the previous one serves"
fi
[ -z "$REVERB" ] || sudo supervisorctl status "$REVERB" | cut -c1-120
sudo supervisorctl status 2>/dev/null | grep -i -E 'worker' | cut -c1-120 || true
sleep "$SLEEP_ERRORS"
E1=$(sudo grep -c '\.ERROR:' "$LL" 2>/dev/null || true); E1=${E1:-0}
echo "new errors in $(basename "$LL") since the start: $((E1 - E0))"
[ "$E1" -gt "$E0" ] && sudo grep '\.ERROR:' "$LL" | tail -n +$((E0 + 1)) | sed -E 's/^\[[^]]*\] [a-z]+\.ERROR: //; s/ \{".*$//' | cut -c1-140 | sort | uniq -c | sort -rn | head -5

say "9. keep $KEEP releases, record"
printf 'sha %s\ndeployed %s UTC\nby %s\nfrom %s\n' "$SHA" "$(date -u '+%F %T')" "${DEPLOYER:-?}" "${PREV##*/}" | sudo -u "$OWNER" tee "$REL/.release" >/dev/null
LIVE=$(readlink -e "$APP")
ls -1dt "$BASE"/releases/*/ 2>/dev/null | sed 's#/$##' | grep -E '/[0-9a-f]{40}$' | tail -n +$((KEEP + 1)) | while read -r OLD; do
    [ "$OLD" = "$LIVE" ] || [ "$OLD" = "$PREV" ] || { sudo rm -rf --one-file-system "$OLD"; echo "removed ${OLD##*/}"; }
done
NOTE=""; [ "$RELOAD" = yes ] || NOTE=" (no php-fpm reload)"
echo "$(date -u '+%F %T') $SHA ${DEPLOYER:-?} from ${PREV##*/} ok$NOTE" | sudo tee -a "$BASE/deploys.log" >/dev/null
sudo rm -f "$TGZ"
echo "live: $SHA"
if [ "$RELOAD" = yes ]; then
    echo "way back: sudo ln -sfn $PREV $APP.new && sudo mv -T $APP.new $APP && sudo systemctl reload $FPM && (cd $APP && sudo -u $QRESTART_USER $PHP artisan queue:restart)${REVERB:+ && sudo supervisorctl restart $REVERB}"
else
    echo "way back: sudo ln -sfn $PREV $APP.new && sudo mv -T $APP.new $APP && (cd $APP && sudo -u $QRESTART_USER $PHP artisan queue:restart)"
fi
} 2>&1
