Submit and Monitor an Experiment

Submits an experiment from your local repository, streams its logs until it finishes, then sends a desktop notification (macOS) or terminal bell on completion or failure.

This script uses the local submit command, so you will need to have your project's code locally and either run the script from there or point to it with the --repo-dir option.

Usage

./02-experiment-submit-monitor.sh --message "My experiment" [--repo-dir PATH] [--project-name NAME] [--engine-name NAME]

Full Script

02-experiment-submit-monitor.sh

#!/usr/bin/env bash
# ==============================================================================
# 02-experiment-submit-monitor.sh
#
# Submits an experiment from your local repository, streams its logs until it
# finishes, then sends a desktop notification (macOS) or terminal bell on
# completion or failure.
#
# Usage:
#   ./02-experiment-submit-monitor.sh --message "My experiment" [options]
#
# Required:
#   --message TEXT        Commit message / label for the experiment
#
# Options:
#   --repo-dir PATH       Path to the local repo (default: current directory)
#   --project-name NAME   Override the project from CLI context
#   --engine-name NAME    Override the engine from CLI context
# Environment variables:
#   AICHOR_API_KEY        Required only if not already authenticated
# ==============================================================================

set -eo pipefail

# ---------------------------------------------------------------------------
# Defaults
# ---------------------------------------------------------------------------
MESSAGE=""
REPO_DIR="."
PROJECT_NAME=""
ENGINE_NAME=""
# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------
while [[ $# -gt 0 ]]; do
  case "$1" in
    --message)       MESSAGE="$2";        shift 2 ;;
    --repo-dir)      REPO_DIR="$2";       shift 2 ;;
    --project-name)  PROJECT_NAME="$2";   shift 2 ;;
    --engine-name)   ENGINE_NAME="$2";    shift 2 ;;
    -h|--help)
      sed -n '2,/^# =\+$/p' "$0" | grep '^#' | sed 's/^# \?//'
      exit 0
      ;;
    *)
      echo "Unknown argument: $1" >&2
      exit 1
      ;;
  esac
done

if [[ -z "$MESSAGE" ]]; then
  echo "Error: --message is required." >&2
  exit 1
fi

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
section() {
  echo
  echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
  echo "  $*"
  echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
}

# Send a desktop notification.
# Uses osascript on macOS, falls back to terminal bell elsewhere.
notify() {
  local title="$1"
  local body="$2"
  echo
  echo ">>> NOTIFICATION: [$title] $body"
  if command -v osascript &>/dev/null; then
    local safe_title="${title//\"/}"
    local safe_body="${body//\"/}"
    osascript -e "display notification \"$safe_body\" with title \"$safe_title\" sound name \"default\""
  else
    printf '\a'
  fi
}

# cli_flags: full set (project + engine) for commands that accept both
# project_flags: project only, for commands that don't accept --engine-name (e.g. experiments status)
cli_flags=()
project_flags=()
[[ -n "$PROJECT_NAME" ]] && cli_flags+=(--project-name "$PROJECT_NAME") && project_flags+=(--project-name "$PROJECT_NAME")
[[ -n "$ENGINE_NAME" ]]  && cli_flags+=(--engine-name  "$ENGINE_NAME")

# ---------------------------------------------------------------------------
# 1. Authentication check
# ---------------------------------------------------------------------------
section "1. Authentication check"
if ! aichor projects list --output json &>/dev/null; then
  echo "Not authenticated. Logging in..."
  aichor auth key --apikey "${AICHOR_API_KEY:?AICHOR_API_KEY not set}"
fi
echo "Authenticated."

# ---------------------------------------------------------------------------
# 2. Submit the experiment
# ---------------------------------------------------------------------------
section "2. Submitting experiment: \"$MESSAGE\""
echo "Repo dir:  $REPO_DIR"
[[ -n "$PROJECT_NAME" ]] && echo "Project:   $PROJECT_NAME"
[[ -n "$ENGINE_NAME"  ]] && echo "Engine:    $ENGINE_NAME"
echo

EXPERIMENT_ID=$(aichor experiments submit local \
  "${cli_flags[@]}" --message "$MESSAGE" --repo-dir "$REPO_DIR" \
  | jq -r '.experiment_id')

echo "Experiment ID: $EXPERIMENT_ID"
START_TIME=$(date +%s)

# ---------------------------------------------------------------------------
# 3. Wait for the experiment to become available
# ---------------------------------------------------------------------------
section "3. Waiting for experiment to become available"
until aichor experiments status "$EXPERIMENT_ID" "${project_flags[@]}" &>/dev/null; do
  echo "  Not available yet — retrying in 5s..."
  sleep 5
done
echo "Experiment is available."

# ---------------------------------------------------------------------------
# 4. Stream logs (blocks until the experiment finishes or is cancelled)
# ---------------------------------------------------------------------------
section "4. Streaming logs for experiment: $EXPERIMENT_ID"
echo "This will block until the experiment completes."
echo "Press Ctrl+C to detach — the experiment will keep running on AIchor."
echo

# logs stream writes logs to stderr (visible in terminal) and outputs
# {"experiment_status": <status>} as JSON to stdout when done.
# set +e so a failed experiment doesn't kill the script before we notify.
set +e
final_status=$(aichor experiments logs stream "$EXPERIMENT_ID" "${cli_flags[@]}" \
  | jq -r '.experiment_status')
set -e

END_TIME=$(date +%s)
ELAPSED=$(( END_TIME - START_TIME ))
ELAPSED_FMT=$(printf '%02dh %02dm %02ds' \
  $(( ELAPSED / 3600 )) $(( (ELAPSED % 3600) / 60 )) $(( ELAPSED % 60 )))

echo "Wall-clock time: $ELAPSED_FMT"
echo "Final status:    $final_status"

section "5. Experiment details"
aichor experiments list "$EXPERIMENT_ID" "${cli_flags[@]}" --output table

# ---------------------------------------------------------------------------
# 6. Notify
# ---------------------------------------------------------------------------
section "6. Notification"
case "$final_status" in
  Succeeded)
    notify "AIchor: Experiment succeeded" \
      "$MESSAGE ($EXPERIMENT_ID) finished in $ELAPSED_FMT."
    ;;
  Failed)
    notify "AIchor: Experiment FAILED" \
      "$MESSAGE ($EXPERIMENT_ID) failed after $ELAPSED_FMT. Check logs."
    ;;
  Cancelled)
    notify "AIchor: Experiment cancelled" \
      "$MESSAGE ($EXPERIMENT_ID) was cancelled after $ELAPSED_FMT."
    ;;
  *)
    notify "AIchor: Experiment finished" \
      "$MESSAGE ($EXPERIMENT_ID) ended with status: $final_status in $ELAPSED_FMT."
    ;;
esac

echo
echo "To review logs at any time:"
echo "  aichor experiments logs query $EXPERIMENT_ID --step run"