[PATCH v14 3/6] devtools: add compare-reviews.sh for multi-provider analysis

[Stephen Hemminger]

Add script to run patch reviews across multiple AI providers for
comparison purposes.

The script automatically detects which providers have API keys
configured and runs analyze-patch.py for each one. This allows
users to compare review quality and feedback across different
AI models.

Features:
  - Auto-detects available providers based on environment variables
  - Optional provider selection via -p/--providers option
  - Saves individual reviews to separate files with -o/--output
  - Verbose mode passes through to underlying analyze-patch.py

Usage:
  ./devtools/compare-reviews.sh my-patch.patch
  ./devtools/compare-reviews.sh -p anthropic,xai my-patch.patch
  ./devtools/compare-reviews.sh -o ./reviews my-patch.patch

Output files are named <patch>-<provider>.txt when using the
output directory option.

Signed-off-by: Stephen Hemminger <stephen at networkplumber.org>
Acked-by: Aaron Conole <aconole at redhat.com>
---
 devtools/compare-reviews.sh | 263 ++++++++++++++++++++++++++++++++++++
 1 file changed, 263 insertions(+)
 create mode 100755 devtools/compare-reviews.sh

diff --git a/devtools/compare-reviews.sh b/devtools/compare-reviews.sh
new file mode 100755
index 0000000000..b4813cb6a7
--- /dev/null
+++ b/devtools/compare-reviews.sh
@@ -0,0 +1,263 @@
+#!/bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2026 Stephen Hemminger
+
+# Compare DPDK patch reviews across multiple AI providers
+# Runs analyze-patch.py with each available provider
+
+set -o pipefail
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+ANALYZE_SCRIPT="${SCRIPT_DIR}/analyze-patch.py"
+AGENTS_FILE="AGENTS.md"
+OUTPUT_DIR=""
+PROVIDERS=""
+FORMAT="text"
+VERBOSE=""
+EXTRA_ARGS=()
+
+usage() {
+    cat <<EOF
+Usage: $(basename "$0") [OPTIONS] <patch-file>
+
+Compare DPDK patch reviews across multiple AI providers.
+
+Options:
+    -a, --agents FILE      Path to AGENTS.md file (default: AGENTS.md)
+    -o, --output DIR       Save individual reviews to directory
+    -p, --providers LIST   Comma-separated list of providers to use
+                           (default: all providers with API keys set)
+    -f, --format FORMAT    Output format: text, markdown, html, json
+                           (default: text)
+    -t, --tokens N         Max tokens for response
+    -D, --date DATE        Review date context (YYYY-MM-DD)
+    -r, --release VERSION  Target DPDK release (e.g., 24.11, 23.11-lts)
+    --split-patches        Split mbox into individual patches
+    --patch-range N-M      Review only patches N through M
+    --large-file MODE      Handle large files: error, truncate, chunk,
+                           commits-only, summary
+    --max-tokens N         Max input tokens
+    -v, --verbose          Show verbose output from each provider
+    -h, --help             Show this help message
+
+Environment Variables:
+    Set API keys for providers you want to use:
+    ANTHROPIC_API_KEY, OPENAI_API_KEY, XAI_API_KEY, GOOGLE_API_KEY
+
+Examples:
+    $(basename "$0") my-patch.patch
+    $(basename "$0") -p anthropic,openai my-patch.patch
+    $(basename "$0") -o ./reviews -f markdown my-patch.patch
+    $(basename "$0") -r 24.11 --split-patches series.mbox
+EOF
+    exit "${1:-0}"
+}
+
+error() {
+    echo "Error: $1" >&2
+    exit 1
+}
+
+# Check which providers have API keys configured
+get_available_providers() {
+    local available=""
+
+    [[ -n "$ANTHROPIC_API_KEY" ]] && available="${available}anthropic,"
+    [[ -n "$OPENAI_API_KEY" ]] && available="${available}openai,"
+    [[ -n "$XAI_API_KEY" ]] && available="${available}xai,"
+    [[ -n "$GOOGLE_API_KEY" ]] && available="${available}google,"
+
+    # Remove trailing comma
+    echo "${available%,}"
+}
+
+# Get file extension for format
+get_extension() {
+    case "$1" in
+        text)     echo "txt" ;;
+        markdown) echo "md" ;;
+        html)     echo "html" ;;
+        json)     echo "json" ;;
+        *)        echo "txt" ;;
+    esac
+}
+
+# Parse command line options
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -a|--agents)
+            [[ -z "${2:-}" || "$2" == -* ]] && error "$1 requires an argument"
+            AGENTS_FILE="$2"
+            shift 2
+            ;;
+        -o|--output)
+            [[ -z "${2:-}" || "$2" == -* ]] && error "$1 requires an argument"
+            OUTPUT_DIR="$2"
+            shift 2
+            ;;
+        -p|--providers)
+            [[ -z "${2:-}" || "$2" == -* ]] && error "$1 requires an argument"
+            PROVIDERS="$2"
+            shift 2
+            ;;
+        -f|--format)
+            [[ -z "${2:-}" || "$2" == -* ]] && error "$1 requires an argument"
+            FORMAT="$2"
+            shift 2
+            ;;
+        -t|--tokens)
+            [[ -z "${2:-}" || "$2" == -* ]] && error "$1 requires an argument"
+            EXTRA_ARGS+=("-t" "$2")
+            shift 2
+            ;;
+        -D|--date)
+            [[ -z "${2:-}" || "$2" == -* ]] && error "$1 requires an argument"
+            EXTRA_ARGS+=("-D" "$2")
+            shift 2
+            ;;
+        -r|--release)
+            [[ -z "${2:-}" || "$2" == -* ]] && error "$1 requires an argument"
+            EXTRA_ARGS+=("-r" "$2")
+            shift 2
+            ;;
+        --split-patches)
+            EXTRA_ARGS+=("--split-patches")
+            shift
+            ;;
+        --patch-range)
+            [[ -z "${2:-}" || "$2" == -* ]] && error "$1 requires an argument"
+            EXTRA_ARGS+=("--patch-range" "$2")
+            shift 2
+            ;;
+        --large-file)
+            [[ -z "${2:-}" || "$2" == -* ]] && error "$1 requires an argument"
+            EXTRA_ARGS+=("--large-file" "$2")
+            shift 2
+            ;;
+        --large-file=*)
+            EXTRA_ARGS+=("$1")
+            shift
+            ;;
+        --max-tokens)
+            [[ -z "${2:-}" || "$2" == -* ]] && error "$1 requires an argument"
+            EXTRA_ARGS+=("--max-tokens" "$2")
+            shift 2
+            ;;
+        -v|--verbose)
+            VERBOSE="-v"
+            shift
+            ;;
+        -h|--help)
+            usage 0
+            ;;
+        -*)
+            error "Unknown option: $1"
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+# Check for required arguments
+if [[ $# -lt 1 ]]; then
+    echo "Error: No patch file specified" >&2
+    usage 1
+fi
+
+PATCH_FILE="$1"
+
+if [[ ! -f "$PATCH_FILE" ]]; then
+    error "Patch file not found: $PATCH_FILE"
+fi
+
+if [[ ! -f "$ANALYZE_SCRIPT" ]]; then
+    error "analyze-patch.py not found: $ANALYZE_SCRIPT"
+fi
+
+if [[ ! -f "$AGENTS_FILE" ]]; then
+    error "AGENTS.md not found: $AGENTS_FILE"
+fi
+
+# Validate format
+case "$FORMAT" in
+    text|markdown|html|json) ;;
+    *) error "Invalid format: $FORMAT (must be text, markdown, html, or json)" ;;
+esac
+
+# Get providers to use
+if [[ -z "$PROVIDERS" ]]; then
+    PROVIDERS=$(get_available_providers)
+fi
+
+if [[ -z "$PROVIDERS" ]]; then
+    error "No API keys configured. Set at least one of: "\
+"ANTHROPIC_API_KEY, OPENAI_API_KEY, XAI_API_KEY, GOOGLE_API_KEY"
+fi
+
+# Create output directory if specified
+if [[ -n "$OUTPUT_DIR" ]]; then
+    mkdir -p "$OUTPUT_DIR"
+fi
+
+PATCH_BASENAME=$(basename "$PATCH_FILE")
+PATCH_STEM="${PATCH_BASENAME%.*}"
+EXT=$(get_extension "$FORMAT")
+
+echo "Reviewing patch: $PATCH_BASENAME"
+echo "Providers: $PROVIDERS"
+echo "Format: $FORMAT"
+echo "========================================"
+echo ""
+
+# Run review for each provider, continue on failure
+IFS=',' read -ra PROVIDER_LIST <<< "$PROVIDERS"
+failures=0
+for provider in "${PROVIDER_LIST[@]}"; do
+    echo ">>> Running review with: $provider"
+    echo ""
+
+    if [[ -n "$OUTPUT_DIR" ]]; then
+        OUTPUT_FILE="${OUTPUT_DIR}/${PATCH_STEM}-${provider}.${EXT}"
+        if python3 "$ANALYZE_SCRIPT" \
+            -p "$provider" \
+            -a "$AGENTS_FILE" \
+            -f "$FORMAT" \
+            ${VERBOSE:+"$VERBOSE"} \
+            "${EXTRA_ARGS[@]}" \
+            "$PATCH_FILE" | tee "$OUTPUT_FILE"; then
+            echo ""
+            echo "Saved to: $OUTPUT_FILE"
+        else
+            echo "FAILED: $provider review failed" >&2
+            rm -f "$OUTPUT_FILE"
+            ((failures++)) || true
+        fi
+    else
+        if ! python3 "$ANALYZE_SCRIPT" \
+            -p "$provider" \
+            -a "$AGENTS_FILE" \
+            -f "$FORMAT" \
+            ${VERBOSE:+"$VERBOSE"} \
+            "${EXTRA_ARGS[@]}" \
+            "$PATCH_FILE"; then
+            echo "FAILED: $provider review failed" >&2
+            ((failures++)) || true
+        fi
+    fi
+
+    echo ""
+    echo "========================================"
+    echo ""
+done
+
+echo "Review comparison complete."
+
+if [[ -n "$OUTPUT_DIR" ]]; then
+    echo "All reviews saved to: $OUTPUT_DIR"
+fi
+
+if [[ $failures -gt 0 ]]; then
+    echo "$failures provider(s) failed." >&2
+    exit 1
+fi
-- 
2.53.0