#!/usr/bin/env bash
# Run the public CHNB experiment suite by invoking each experiment's run.sh.

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
. "$PROJECT_ROOT/scripts/chnb_env.sh"
TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
OUTPUT_ROOT="${CHNB_OUTPUT_ROOT:-$PROJECT_ROOT/output}"
LOG_DIR="${CHNB_RUN_ALL_LOG_DIR:-$OUTPUT_ROOT/run_all/$TIMESTAMP}"

RUN_END_TO_END=1
RUN_SUBTITLES=1
RUN_SOCRATIC=1
RUN_VIDEOAGENT=1
RUN_LETTA=0
DRY_RUN=0
FAIL_FAST=0

if [ -n "${SOCRATIC_BACKBONES:-}" ]; then
    # shellcheck disable=SC2206
    SOCRATIC_BACKBONES_ARR=($SOCRATIC_BACKBONES)
else
    SOCRATIC_BACKBONES_ARR=(gemini gpt claude qwen3vl glm4v)
fi

if [ -n "${VIDEOAGENT_BACKBONES:-}" ]; then
    # shellcheck disable=SC2206
    VIDEOAGENT_BACKBONES_ARR=($VIDEOAGENT_BACKBONES)
else
    VIDEOAGENT_BACKBONES_ARR=(gemini gpt claude qwen3vl glm4v)
fi

if [ -n "${LETTA_BACKBONES:-}" ]; then
    # shellcheck disable=SC2206
    LETTA_BACKBONES_ARR=($LETTA_BACKBONES)
else
    LETTA_BACKBONES_ARR=(gemini gpt claude)
fi

if [ -n "${LETTA_SOURCES:-}" ]; then
    # shellcheck disable=SC2206
    LETTA_SOURCES_ARR=($LETTA_SOURCES)
else
    LETTA_SOURCES_ARR=(captions wsh)
fi

if [ -n "${LETTA_CAPTION_VARIANTS:-}" ]; then
    # shellcheck disable=SC2206
    LETTA_CAPTION_VARIANTS_ARR=($LETTA_CAPTION_VARIANTS)
else
    LETTA_CAPTION_VARIANTS_ARR=(gemini_frames gemini_video_clip_shot_grouped)
fi

usage() {
    cat <<'EOF'
Usage:
  bash scripts/run_all_experiments.sh [options]

Default:
  Runs end-to-end runners, subtitles-only, Socratic, and VideoAgent.
  Letta is opt-in because it requires a separate running Letta server.

Options:
  --all                   Include Letta as well.
  --with-letta            Include Letta.
  --end-to-end-only       Run only end-to-end runners.
  --text-only             Run only subtitles-only.
  --socratic-only         Run only Socratic.
  --videoagent-only       Run only VideoAgent.
  --letta-only            Run only Letta.
  --skip-end-to-end       Skip end-to-end runners.
  --skip-subtitles        Skip subtitles-only.
  --skip-socratic         Skip Socratic.
  --skip-videoagent       Skip VideoAgent.
  --skip-letta            Skip Letta.
  --fail-fast             Stop at the first failed command.
  --dry-run               Print commands without running them.
  -h, --help              Show this help.

Environment overrides:
  CHNB_OUTPUT_ROOT
  CHNB_RUN_ALL_LOG_DIR
  SOCRATIC_BACKBONES="gemini gpt claude qwen3vl glm4v"
  SOCRATIC_CAPTIONERS="gemini_frames gemini_video_clip_shot_grouped"
  VIDEOAGENT_BACKBONES="gemini gpt claude qwen3vl glm4v"
  LETTA_BACKBONES="gemini gpt claude"
  LETTA_SOURCES="captions wsh"
  LETTA_CAPTION_VARIANTS="gemini_frames gemini_video_clip_shot_grouped"
EOF
}

only() {
    RUN_END_TO_END=0
    RUN_SUBTITLES=0
    RUN_SOCRATIC=0
    RUN_VIDEOAGENT=0
    RUN_LETTA=0
    case "$1" in
        end-to-end) RUN_END_TO_END=1 ;;
        text) RUN_SUBTITLES=1 ;;
        socratic) RUN_SOCRATIC=1 ;;
        videoagent) RUN_VIDEOAGENT=1 ;;
        letta) RUN_LETTA=1 ;;
    esac
}

while [ "$#" -gt 0 ]; do
    case "$1" in
        --all)
            RUN_LETTA=1
            shift
            ;;
        --with-letta)
            RUN_LETTA=1
            shift
            ;;
        --end-to-end-only)
            only end-to-end
            shift
            ;;
        --text-only)
            only text
            shift
            ;;
        --socratic-only)
            only socratic
            shift
            ;;
        --videoagent-only)
            only videoagent
            shift
            ;;
        --letta-only)
            only letta
            shift
            ;;
        --skip-end-to-end)
            RUN_END_TO_END=0
            shift
            ;;
        --skip-subtitles)
            RUN_SUBTITLES=0
            shift
            ;;
        --skip-socratic)
            RUN_SOCRATIC=0
            shift
            ;;
        --skip-videoagent)
            RUN_VIDEOAGENT=0
            shift
            ;;
        --skip-letta)
            RUN_LETTA=0
            shift
            ;;
        --fail-fast)
            FAIL_FAST=1
            shift
            ;;
        --dry-run)
            DRY_RUN=1
            shift
            ;;
        -h|--help)
            usage
            exit 0
            ;;
        *)
            echo "Unknown option: $1" >&2
            usage >&2
            exit 2
            ;;
    esac
done

SOCRATIC_CAPTIONER_ARGS=()
if [ -n "${SOCRATIC_CAPTIONERS:-}" ]; then
    # shellcheck disable=SC2206
    _captioners=($SOCRATIC_CAPTIONERS)
    SOCRATIC_CAPTIONER_ARGS=(--captioners "${_captioners[@]}")
fi

mkdir -p "$LOG_DIR"
SUMMARY="$LOG_DIR/summary.tsv"
printf 'status\tlabel\tlog\n' > "$SUMMARY"

run_cmd() {
    local label="$1"
    shift
    local safe_label="${label//[^A-Za-z0-9._-]/_}"
    local log="$LOG_DIR/$safe_label.log"

    echo
    echo "=== $label ==="
    printf '$'
    printf ' %q' "$@"
    echo
    echo "log: $log"

    if [ "$DRY_RUN" = "1" ]; then
        printf 'dry-run\t%s\t%s\n' "$label" "$log" >> "$SUMMARY"
        return 0
    fi

    set +e
    "$@" 2>&1 | tee "$log"
    local status="${PIPESTATUS[0]}"
    set -e

    if [ "$status" -eq 0 ]; then
        printf 'ok\t%s\t%s\n' "$label" "$log" >> "$SUMMARY"
    else
        printf 'error:%s\t%s\t%s\n' "$status" "$label" "$log" >> "$SUMMARY"
        if [ "$FAIL_FAST" = "1" ]; then
            exit "$status"
        fi
    fi
}

echo "Run-all log directory: $LOG_DIR"

if [ "$RUN_END_TO_END" = "1" ]; then
    run_cmd end-to-end-gemini3flash \
        bash "$PROJECT_ROOT/experiments/end-to-end/gemini3flash/run.sh"
    run_cmd end-to-end-gemini3flash-video \
        bash "$PROJECT_ROOT/experiments/end-to-end/gemini3flash_video/run.sh"
    run_cmd end-to-end-gpt5mini \
        bash "$PROJECT_ROOT/experiments/end-to-end/gpt5mini/run.sh"
    run_cmd end-to-end-claude45haiku \
        bash "$PROJECT_ROOT/experiments/end-to-end/claude45haiku/run.sh"
    run_cmd end-to-end-qwen3vl \
        bash "$PROJECT_ROOT/experiments/end-to-end/qwen3vl/run.sh"
    run_cmd end-to-end-glm4v \
        bash "$PROJECT_ROOT/experiments/end-to-end/glm4v/run.sh"
    run_cmd end-to-end-llava-next-video \
        bash "$PROJECT_ROOT/experiments/end-to-end/llava_next_video/run.sh"
    run_cmd end-to-end-hourllava \
        bash "$PROJECT_ROOT/experiments/end-to-end/hourllava/run.sh"
    run_cmd end-to-end-vamba \
        bash "$PROJECT_ROOT/experiments/end-to-end/vamba/run.sh"
    run_cmd end-to-end-videochat \
        bash "$PROJECT_ROOT/experiments/end-to-end/videochat/run.sh"
fi

if [ "$RUN_SUBTITLES" = "1" ]; then
    run_cmd subtitles-only \
        bash "$PROJECT_ROOT/experiments/subtitles_only/run.sh" \
            --qa-backbones gemini gpt claude qwen3vl glm4v
fi

if [ "$RUN_SOCRATIC" = "1" ]; then
    cmd=(bash "$PROJECT_ROOT/experiments/socratic/run.sh")
    if [ "${#SOCRATIC_CAPTIONER_ARGS[@]}" -gt 0 ]; then
        cmd+=("${SOCRATIC_CAPTIONER_ARGS[@]}")
    fi
    cmd+=(--qa-backbones "${SOCRATIC_BACKBONES_ARR[@]}")
    run_cmd socratic "${cmd[@]}"
fi

if [ "$RUN_VIDEOAGENT" = "1" ]; then
    for backbone in "${VIDEOAGENT_BACKBONES_ARR[@]}"; do
        run_cmd "videoagent-$backbone" \
            bash "$PROJECT_ROOT/experiments/agentic-retrieval/videoagent/run.sh" \
                --backbone "$backbone"
    done
fi

if [ "$RUN_LETTA" = "1" ]; then
    for backbone in "${LETTA_BACKBONES_ARR[@]}"; do
        for source in "${LETTA_SOURCES_ARR[@]}"; do
            for caption_variant in "${LETTA_CAPTION_VARIANTS_ARR[@]}"; do
                run_cmd "letta-$backbone-$source-$caption_variant" \
                    bash "$PROJECT_ROOT/experiments/agentic-retrieval/letta/run.sh" \
                        --backbone "$backbone" \
                        --source "$source" \
                        --caption-variant "$caption_variant"
            done
        done
    done
fi

echo
echo "Experiment orchestration complete."
echo "Summary: $SUMMARY"