Storage Operations

Demonstrates common AIchor storage operations: listing buckets, creating directories, uploading and downloading files, copying between buckets, and deleting files safely with a dry-run step.

Usage

./04-storage-operations.sh --storage-id <id> [--dst-storage-id <id>] [--project-name NAME] [--engine-name NAME]

Full Script

04-storage-operations.sh

#!/usr/bin/env bash
# ==============================================================================
# 04-storage-operations.sh
#
# Demonstrates common AIchor storage operations:
#   - List project buckets
#   - Create a remote directory
#   - Upload a local directory to a bucket
#   - Browse bucket contents (tree, recursive, filtered)
#   - Download from a bucket to local disk
#   - Copy files between two buckets (requires two bucket IDs)
#   - Delete files (always dry-run first)
#
# IMPORTANT — path convention:
#   Directories MUST end with a trailing slash:   /path/to/dir/
#   Files must NOT end with a trailing slash:      /path/to/file.txt
#   Getting this wrong causes unexpected behaviour.
#
# Usage:
#   ./04-storage-operations.sh --storage-id <id> --dst-storage-id <id> [options]
#
# Required:
#   --storage-id ID       Source bucket ID
#   --dst-storage-id ID   Destination bucket for copy demo
#
# Options:
#   --project-name NAME   Override the project from CLI context
#   --engine-name NAME    Override the engine from CLI context
#
# Environment variables:
#   AICHOR_API_KEY        Required only if not already authenticated
# ==============================================================================

set -eo pipefail

# ---------------------------------------------------------------------------
# Defaults
# ---------------------------------------------------------------------------
STORAGE_ID=""
DST_STORAGE_ID=""
PROJECT_NAME=""
ENGINE_NAME=""
readonly REMOTE_DEMO_DIR="aichor-cli-demo/"
readonly LOCAL_UPLOAD_DIR="/tmp/aichor-upload-demo/"
readonly LOCAL_DOWNLOAD_DIR="/tmp/aichor-download-demo/"

# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------
while [[ $# -gt 0 ]]; do
  case "$1" in
    --storage-id)      STORAGE_ID="$2";      shift 2 ;;
    --dst-storage-id)  DST_STORAGE_ID="$2";  shift 2 ;;
    --project-name)    PROJECT_NAME="$2";    shift 2 ;;
    --engine-name)     ENGINE_NAME="$2";     shift 2 ;;
    -h|--help)
      sed -n '2,/^# =\+$/p' "$0" | grep '^#' | sed 's/^# \?//'
      exit 0
      ;;
    *)
      echo "Unknown argument: $1" >&2
      exit 1
      ;;
  esac
done

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
section() {
  echo
  echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
  echo "  $*"
  echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
}

pause() {
  echo
  read -r -p "Press Enter to continue to the next step..."
}

cli_flags=()
[[ -n "$PROJECT_NAME" ]] && cli_flags+=(--project-name "$PROJECT_NAME")
[[ -n "$ENGINE_NAME"  ]] && cli_flags+=(--engine-name  "$ENGINE_NAME")

# ---------------------------------------------------------------------------
# 1. Authentication check
# ---------------------------------------------------------------------------
section "1. Authentication check"
if ! aichor projects list --output json &>/dev/null; then
  echo "Not authenticated. Logging in..."
  aichor auth key --apikey "${AICHOR_API_KEY:?AICHOR_API_KEY not set}"
fi
echo "Authenticated."

# ---------------------------------------------------------------------------
# 2. List available buckets
# ---------------------------------------------------------------------------
section "2. Available buckets for this project"
aichor storage cloud list "${cli_flags[@]}" --output table

if [[ -z "$STORAGE_ID" ]]; then
  echo "ERROR: --storage-id is required." >&2
  echo "  Run 'aichor storage cloud list' to find your bucket ID." >&2
  exit 1
fi
if [[ -z "$DST_STORAGE_ID" ]]; then
  echo "ERROR: --dst-storage-id is required." >&2
  echo "  Run 'aichor storage cloud list' to find your bucket ID." >&2
  exit 1
fi

echo "Source bucket:      $STORAGE_ID"
echo "Destination bucket: $DST_STORAGE_ID"

# ---------------------------------------------------------------------------
# 3. Prepare local demo files
# ---------------------------------------------------------------------------
section "3. Preparing local demo files in $LOCAL_UPLOAD_DIR"
rm -rf "$LOCAL_UPLOAD_DIR"
mkdir -p "$LOCAL_UPLOAD_DIR/subdir/"

cat > "$LOCAL_UPLOAD_DIR/hello.txt" <<EOF
Hello from AIchor storage demo
Created: $(date)
EOF

cat > "$LOCAL_UPLOAD_DIR/data.csv" <<EOF
id,value,label
1,0.92,cat
2,0.87,dog
3,0.95,bird
EOF

echo "Nested file content" > "$LOCAL_UPLOAD_DIR/subdir/nested.txt"
echo "Another nested file" > "$LOCAL_UPLOAD_DIR/subdir/model.bin"

echo "Demo files created:"
find "$LOCAL_UPLOAD_DIR" -type f | sort | sed 's/^/  /'

# ---------------------------------------------------------------------------
# 4. Create a remote directory
# ---------------------------------------------------------------------------
section "4. Creating remote directory: $REMOTE_DEMO_DIR"
aichor storage cloud mkdir \
  --storage-id "$STORAGE_ID" \
  "${cli_flags[@]}" \
  --path "$REMOTE_DEMO_DIR"
echo "Directory created: $REMOTE_DEMO_DIR"

pause

# ---------------------------------------------------------------------------
# 5. Upload
# ---------------------------------------------------------------------------
section "5. Uploading $LOCAL_UPLOAD_DIR -> bucket:$REMOTE_DEMO_DIR"
echo "Using 4 parallel workers for faster transfer."
aichor storage cloud upload \
  --storage-id "$STORAGE_ID" \
  "${cli_flags[@]}" \
  --local-path  "$LOCAL_UPLOAD_DIR" \
  --remote-path "$REMOTE_DEMO_DIR" \
  --max-workers 4

pause

# ---------------------------------------------------------------------------
# 6. Browse bucket contents
# ---------------------------------------------------------------------------
section "6a. Demo directory — tree view (default)"
aichor storage cloud list-contents \
  --storage-id "$STORAGE_ID" \
  "${cli_flags[@]}" \
  --path "$REMOTE_DEMO_DIR" \
  --recursive

section "6b. Recursive listing with file sizes"
aichor storage cloud list-contents \
  --storage-id "$STORAGE_ID" \
  "${cli_flags[@]}" \
  --path "$REMOTE_DEMO_DIR" \
  --recursive \
  --output size

section "6c. Filter: only .txt files (simple one-per-line format)"
aichor storage cloud list-contents \
  --storage-id "$STORAGE_ID" \
  "${cli_flags[@]}" \
  --path "$REMOTE_DEMO_DIR" \
  --recursive \
  --include "*.txt" \
  --output simple

section "6d. Direct children only (max-depth 0)"
aichor storage cloud list-contents \
  --storage-id "$STORAGE_ID" \
  "${cli_flags[@]}" \
  --path "$REMOTE_DEMO_DIR" \
  --recursive \
  --max-depth 0 \
  --output simple

pause

# ---------------------------------------------------------------------------
# 7. Download
# ---------------------------------------------------------------------------
section "7. Downloading bucket:$REMOTE_DEMO_DIR -> $LOCAL_DOWNLOAD_DIR"
rm -rf "$LOCAL_DOWNLOAD_DIR"
mkdir -p "$LOCAL_DOWNLOAD_DIR"

aichor storage cloud download \
  --storage-id  "$STORAGE_ID" \
  "${cli_flags[@]}" \
  --remote-path "$REMOTE_DEMO_DIR" \
  --local-path  "$LOCAL_DOWNLOAD_DIR" \
  --max-workers 4 \
  --overwrite

echo "Downloaded files:"
find "$LOCAL_DOWNLOAD_DIR" -type f | sort | sed 's/^/  /'

pause

# ---------------------------------------------------------------------------
# 8. Download a single file
# ---------------------------------------------------------------------------
section "8. Downloading a single file: ${REMOTE_DEMO_DIR}hello.txt"
aichor storage cloud download \
  --storage-id  "$STORAGE_ID" \
  "${cli_flags[@]}" \
  --remote-path "${REMOTE_DEMO_DIR}hello.txt" \
  --local-path  "/tmp/hello-downloaded.txt" \
  --overwrite

echo "Contents of downloaded file:"
cat /tmp/hello-downloaded.txt

pause

# ---------------------------------------------------------------------------
# 9. Copy between buckets
# ---------------------------------------------------------------------------
section "9. Copy $REMOTE_DEMO_DIR from $STORAGE_ID to $DST_STORAGE_ID"
aichor storage cloud cp \
  --src-storage-id "$STORAGE_ID" \
  --src-path       "$REMOTE_DEMO_DIR" \
  --dst-storage-id "$DST_STORAGE_ID" \
  --dst-path       "$REMOTE_DEMO_DIR" \
  "${cli_flags[@]}"
echo "Copy complete."

echo
echo "Destination bucket after copy:"
aichor storage cloud list-contents \
  --storage-id "$DST_STORAGE_ID" \
  "${cli_flags[@]}" \
  --path "$REMOTE_DEMO_DIR" \
  --recursive

pause

# ---------------------------------------------------------------------------
# 10. Show how to delete (dry-run only — script does not delete)
# ---------------------------------------------------------------------------
section "10. How to delete the demo directory (dry-run preview)"
echo "The script does not delete anything. Run the dry-run below to preview,"
echo "then run the same command without --dry-run when you are ready."
echo
aichor storage cloud rm \
  --storage-id "$STORAGE_ID" \
  "${cli_flags[@]}" \
  --path "$REMOTE_DEMO_DIR" \
  --dry-run

echo
echo "To actually delete, run:"
echo "  aichor storage cloud rm --storage-id $STORAGE_ID --path $REMOTE_DEMO_DIR --force --max-workers 4"

pause

# ---------------------------------------------------------------------------
# 11. Final state
# ---------------------------------------------------------------------------
section "11. Final bucket state (root)"
aichor storage cloud list-contents \
  --storage-id "$STORAGE_ID" \
  "${cli_flags[@]}" \
  --recursive

echo
echo "Done."