#!/bin/bash
# rerun_failed_jobs.sh
# Rerun failed jobs by reading indir/outdir from PBS output files

failed_jobs_list="failed_jobs.txt"   # contains job output filenames (qc.*.o12345)
job_launcher="job_launcher_variables_rerun.sh"

while read -r outfile; do
    # Make sure file exists
    if [[ ! -f "$outfile" ]]; then
        echo "Warning: $outfile not found, skipping"
        continue
    fi

    # Parse freq/variable from filename
    base=$(echo "$outfile" | sed -E 's/^qc\.//; s/\.o[0-9]+$//')
    aset=$(echo "$base" | cut -d_ -f1)
    variable=$(echo "$base" | cut -d_ -f2)

    # Extract indir and outdir from inside the job output
    indir=$(grep -m1 -- '--indir' "$outfile" | sed -E 's/.*--indir[[:space:]]+([^[:space:]]+).*/\1/')

    # Extract outdir from --logfile argument (strip at /stats/)
    outdir=$(grep -m1 -- '--logfile' "$outfile" | sed -E 's|.*--logfile[[:space:]]+/([^ ]+)/stats/.*|\1|')
    outdir="/$outdir"

    echo "File: $outfile"
    echo "  aset     = $aset"
    echo "  variable = $variable"
    echo "  indir    = $indir"
    echo "  outdir   = $outdir"
    echo

    if [[ -z "$indir" || -z "$outdir" ]]; then
        echo "Failed to extract indir/outdir from $outfile"
        continue
    fi

    echo "Rerunning $aset / $variable"
    bash "$job_launcher" "$aset" "$variable" "$indir" "$outdir"

done < "$failed_jobs_list"