Compare commits
7 Commits
9cb39a1779
...
8f950f575a
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8f950f575a | ||
|
|
cc01e8dd18 | ||
|
|
68f7e9c7be | ||
|
|
a14f317d5a | ||
|
|
89d1a1e82b | ||
|
|
353e5f2b3f | ||
|
|
74424c1f82 |
@@ -1,66 +0,0 @@
|
|||||||
# opencode-worktree
|
|
||||||
|
|
||||||
Isolated OpenCode sessions via git worktrees.
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
Each OpenCode session gets its own git worktree with a unique branch. This prevents:
|
|
||||||
- Clashes with parallel sessions on the same repo
|
|
||||||
- Accidental overwrites from multiple agents
|
|
||||||
- Confusion from work-in-progress across contexts
|
|
||||||
|
|
||||||
## Prerequisites
|
|
||||||
|
|
||||||
- Git
|
|
||||||
- opencode installed and configured
|
|
||||||
|
|
||||||
## Installation
|
|
||||||
|
|
||||||
### Option 1: Source directly (Recommended)
|
|
||||||
```bash
|
|
||||||
. skills/opencode-worktree/opencode-worktree.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
### Option 2: Copy to PATH
|
|
||||||
```bash
|
|
||||||
cp skills/opencode-worktree/opencode-worktree.sh ~/.local/bin/opencode-worktree
|
|
||||||
chmod +x ~/.local/bin/opencode-worktree
|
|
||||||
```
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
### Create new session
|
|
||||||
```bash
|
|
||||||
. opencode-worktree.sh # session-20260327-a1b2c3
|
|
||||||
. opencode-worktree.sh refactor-auth # session-20260327-a1b2c3-refactor-auth
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cleanup
|
|
||||||
```bash
|
|
||||||
. opencode-worktree.sh --cleanup # remove all session-* worktrees
|
|
||||||
. opencode-worktree.sh --cleanup <name> # remove specific worktree
|
|
||||||
```
|
|
||||||
|
|
||||||
## How It Works
|
|
||||||
|
|
||||||
1. **Cleanup** - On every launch, removes all stale `session-*` worktrees and their branches
|
|
||||||
2. **Create** - Creates new worktree based on `main` with unique name: `session-{timestamp}-{random6}[-{purpose}]`
|
|
||||||
3. **Launch** - Changes into worktree and launches opencode
|
|
||||||
4. **Exit** - When opencode exits, you return to your original directory (worktree remains for review)
|
|
||||||
|
|
||||||
## Example Workflow
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Start session for refactoring auth
|
|
||||||
. opencode-worktree.sh refactor-auth
|
|
||||||
|
|
||||||
# ... do work in opencode ...
|
|
||||||
|
|
||||||
# Exit opencode (worktree with your changes still exists)
|
|
||||||
# Later, resume or cleanup
|
|
||||||
. opencode-worktree.sh --cleanup session-20260327-a1b2c3-refactor-auth
|
|
||||||
```
|
|
||||||
|
|
||||||
## Script
|
|
||||||
|
|
||||||
See [opencode-worktree.sh](./opencode-worktree.sh) for the full source.
|
|
||||||
@@ -1,148 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# opencode-worktree - Isolated OpenCode sessions via git worktrees
|
|
||||||
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
||||||
WORKTREE_BASE="$PWD/.git/worktrees"
|
|
||||||
PURPOSE=""
|
|
||||||
CLEANUP_ONLY=false
|
|
||||||
CLEANUP_NAME=""
|
|
||||||
|
|
||||||
usage() {
|
|
||||||
cat <<EOF
|
|
||||||
Usage: $(basename "$0") [purpose] [--cleanup [name]]
|
|
||||||
|
|
||||||
Create isolated OpenCode sessions via git worktrees.
|
|
||||||
|
|
||||||
Arguments:
|
|
||||||
purpose Purpose string for the session (optional)
|
|
||||||
--cleanup Remove all session-* worktrees
|
|
||||||
--cleanup <name> Remove specific worktree by name
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
$(basename "$0") # session-20260327-a1b2c3
|
|
||||||
$(basename "$0") refactor-auth # session-20260327-a1b2c3-refactor-auth
|
|
||||||
$(basename "$0") --cleanup # remove all session-* worktrees
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
parse_args() {
|
|
||||||
while [[ $# -gt 0 ]]; do
|
|
||||||
case "$1" in
|
|
||||||
--cleanup)
|
|
||||||
CLEANUP_ONLY=true
|
|
||||||
if [[ $# -gt 1 && ! "$2" =~ ^-- ]]; then
|
|
||||||
CLEANUP_NAME="$2"
|
|
||||||
shift
|
|
||||||
fi
|
|
||||||
;;
|
|
||||||
-h|--help)
|
|
||||||
usage
|
|
||||||
exit 0
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
if [[ -z "$PURPOSE" ]]; then
|
|
||||||
PURPOSE="$1"
|
|
||||||
fi
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
shift
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
cleanup_stale() {
|
|
||||||
if [[ ! -d "$WORKTREE_BASE" ]]; then
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
|
|
||||||
for wt in "$WORKTREE_BASE"/session-*; do
|
|
||||||
[[ -d "$wt" ]] || continue
|
|
||||||
|
|
||||||
branch=$(git -C "$wt" rev-parse --abbrev-ref HEAD 2>/dev/null) || continue
|
|
||||||
|
|
||||||
echo "Removing stale worktree: $(basename "$wt")"
|
|
||||||
git worktree remove "$wt" --force 2>/dev/null || true
|
|
||||||
if [[ -n "$branch" && "$branch" != "HEAD" ]]; then
|
|
||||||
git branch -D "$branch" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
cleanup_single() {
|
|
||||||
local name="$1"
|
|
||||||
local wt_path="$WORKTREE_BASE/$name"
|
|
||||||
|
|
||||||
if [[ ! -d "$wt_path" ]]; then
|
|
||||||
echo "Worktree '$name' not found"
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
|
|
||||||
branch=$(git -C "$wt_path" rev-parse --abbrev-ref HEAD 2>/dev/null) || branch=""
|
|
||||||
|
|
||||||
echo "Removing worktree: $name"
|
|
||||||
git worktree remove "$wt_path" --force 2>/dev/null || true
|
|
||||||
if [[ -n "$branch" && "$branch" != "HEAD" ]]; then
|
|
||||||
git branch -D "$branch" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
create_worktree() {
|
|
||||||
local timestamp=$(date +%Y%m%d-%H%M%S)
|
|
||||||
local random=$(head -c 3 /dev/urandom | xxd -p | head -c 6)
|
|
||||||
local worktree_name="session-${timestamp}-${random}"
|
|
||||||
local branch_name="$worktree_name"
|
|
||||||
|
|
||||||
if [[ -n "$PURPOSE" ]]; then
|
|
||||||
worktree_name="${worktree_name}-${PURPOSE}"
|
|
||||||
branch_name="$worktree_name"
|
|
||||||
fi
|
|
||||||
|
|
||||||
local worktree_path_abs
|
|
||||||
worktree_path_abs="$(realpath -m "$WORKTREE_BASE")/$worktree_name"
|
|
||||||
local worktree_path="$worktree_path_abs"
|
|
||||||
|
|
||||||
# Cleanup any existing with same name
|
|
||||||
if [[ -d "$worktree_path" ]]; then
|
|
||||||
echo "Removing existing worktree: $worktree_name"
|
|
||||||
git worktree remove "$worktree_path" --force 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Ensure main exists and is up to date
|
|
||||||
if ! git show-ref --quiet refs/heads/main 2>/dev/null; then
|
|
||||||
echo "Error: 'main' branch does not exist"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Create worktree from main
|
|
||||||
echo "Creating worktree: $worktree_name"
|
|
||||||
git worktree add -b "$branch_name" "$worktree_path" main
|
|
||||||
|
|
||||||
# Launch opencode in worktree
|
|
||||||
echo "Entering worktree and launching opencode..."
|
|
||||||
cd "$worktree_path"
|
|
||||||
exec /home/shoko/.opencode/bin/opencode.real
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
|
||||||
# Verify we're in a git repo
|
|
||||||
if ! git rev-parse --is-inside-work-tree 2>/dev/null; then
|
|
||||||
echo "Error: Must be run inside a git repository"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$CLEANUP_ONLY" == true ]]; then
|
|
||||||
if [[ -n "$CLEANUP_NAME" ]]; then
|
|
||||||
cleanup_single "$CLEANUP_NAME"
|
|
||||||
else
|
|
||||||
cleanup_stale
|
|
||||||
fi
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
cleanup_stale
|
|
||||||
create_worktree
|
|
||||||
}
|
|
||||||
|
|
||||||
parse_args "$@"
|
|
||||||
main
|
|
||||||
74
tools/parallel-capacity-test/README.md
Normal file
74
tools/parallel-capacity-test/README.md
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
# Parallel Capacity Test Tool
|
||||||
|
|
||||||
|
Tests the practical limits of parallel agent execution for Hermes/OpenCode.
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This tool stress tests Hermes to find the practical limit of parallel agent execution on the target machine. It:
|
||||||
|
|
||||||
|
- Spawns N concurrent `opencode run` agents
|
||||||
|
- Measures CPU, memory, and response time
|
||||||
|
- Ramps up from 1 to higher agent counts
|
||||||
|
- Identifies failure points and performance degradation
|
||||||
|
|
||||||
|
## Files
|
||||||
|
|
||||||
|
- `run_test.sh` - Bash script for running tests
|
||||||
|
- `parallel_capacity_test.py` - Python tool with more detailed metrics
|
||||||
|
- `results/` - Directory where test results are saved
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
### Quick Test (1, 2, 3, 5, 8 agents)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd tools/parallel-capacity-test
|
||||||
|
./parallel_capacity_test.py --quick
|
||||||
|
```
|
||||||
|
|
||||||
|
### Full Test Suite
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./parallel_capacity_test.py --agents 15 --timeout 120
|
||||||
|
```
|
||||||
|
|
||||||
|
### Bash Script Usage
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./run_test.sh quick # Quick test
|
||||||
|
./run_test.sh full # Full test up to MAX_AGENTS
|
||||||
|
```
|
||||||
|
|
||||||
|
## Environment Variables
|
||||||
|
|
||||||
|
| Variable | Default | Description |
|
||||||
|
|----------|---------|-------------|
|
||||||
|
| MAX_AGENTS | 15 | Maximum number of agents to test |
|
||||||
|
| STEP | 1 | Step size for agent increment |
|
||||||
|
| TASK_TIMEOUT | 120 | Timeout for each agent task |
|
||||||
|
|
||||||
|
## Metrics Collected
|
||||||
|
|
||||||
|
- **Response Time** - Time from agent launch to completion
|
||||||
|
- **CPU Usage** - System-wide CPU utilization percentage
|
||||||
|
- **Memory Usage** - System-wide memory utilization percentage
|
||||||
|
- **Success Rate** - Percentage of agents completing successfully
|
||||||
|
- **Process Count** - Number of opencode processes running
|
||||||
|
|
||||||
|
## Expected Behavior
|
||||||
|
|
||||||
|
Based on the Hermes architecture:
|
||||||
|
|
||||||
|
| Agent Count | Expected Performance |
|
||||||
|
|-------------|---------------------|
|
||||||
|
| 1-3 | Optimal - safe for production |
|
||||||
|
| 4-6 | Good - monitor closely |
|
||||||
|
| 7-10 | Degraded - not recommended |
|
||||||
|
| 10+ | Poor - avoid without significant resources |
|
||||||
|
|
||||||
|
## Output Files
|
||||||
|
|
||||||
|
- `results_YYYYMMDD_HHMMSS.json` - Complete raw results
|
||||||
|
- `summary_YYYYMMDD_HHMMSS.csv` - CSV summary of metrics
|
||||||
|
- `report_YYYYMMDD_HHMMSS.md` - Markdown analysis report
|
||||||
|
EOF; __hermes_rc=$?; printf '__HERMES_FENCE_a9f7b3__'; exit $__hermes_rc
|
||||||
@@ -17,7 +17,12 @@ from datetime import datetime
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import List, Optional
|
from typing import List, Optional
|
||||||
|
|
||||||
# Using stdlib only - no psutil required
|
try:
|
||||||
|
import psutil
|
||||||
|
HAS_PSUTIL = True
|
||||||
|
except ImportError:
|
||||||
|
HAS_PSUTIL = False
|
||||||
|
print("[WARN] psutil not available - resource monitoring will be limited")
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -56,74 +61,6 @@ class TestRun:
|
|||||||
peak_opencode_procs: int
|
peak_opencode_procs: int
|
||||||
|
|
||||||
|
|
||||||
def get_memory_percent() -> float:
|
|
||||||
"""Get memory usage percent by reading /proc/meminfo (Linux)"""
|
|
||||||
try:
|
|
||||||
with open("/proc/meminfo", "r") as f:
|
|
||||||
meminfo = f.read()
|
|
||||||
total = 0
|
|
||||||
available = 0
|
|
||||||
for line in meminfo.splitlines():
|
|
||||||
if line.startswith("MemTotal:"):
|
|
||||||
total = int(line.split()[1])
|
|
||||||
elif line.startswith("MemAvailable:"):
|
|
||||||
available = int(line.split()[1])
|
|
||||||
break
|
|
||||||
if total > 0:
|
|
||||||
used = total - available
|
|
||||||
return (used / total) * 100
|
|
||||||
except (FileNotFoundError, PermissionError, ValueError):
|
|
||||||
pass
|
|
||||||
return 0.0
|
|
||||||
|
|
||||||
|
|
||||||
def count_opencode_processes() -> int:
|
|
||||||
"""Count opencode processes using pgrep or /proc scanning"""
|
|
||||||
try:
|
|
||||||
result = subprocess.run(
|
|
||||||
["pgrep", "-c", "-x", "opencode"],
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
timeout=5
|
|
||||||
)
|
|
||||||
if result.returncode == 0:
|
|
||||||
return int(result.stdout.strip())
|
|
||||||
except (subprocess.TimeoutExpired, ValueError, subprocess.SubprocessError):
|
|
||||||
pass
|
|
||||||
try:
|
|
||||||
count = 0
|
|
||||||
for pid_dir in os.listdir("/proc"):
|
|
||||||
if not pid_dir.isdigit():
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
with open(f"/proc/{pid_dir}/comm", "r") as f:
|
|
||||||
if "opencode" in f.read().lower():
|
|
||||||
count += 1
|
|
||||||
except (PermissionError, FileNotFoundError):
|
|
||||||
continue
|
|
||||||
return count
|
|
||||||
except FileNotFoundError:
|
|
||||||
return 0
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
def get_cpu_percent() -> float:
|
|
||||||
"""Get CPU usage by reading /proc/stat"""
|
|
||||||
try:
|
|
||||||
with open("/proc/stat", "r") as f:
|
|
||||||
line = f.readline()
|
|
||||||
parts = line.split()
|
|
||||||
if parts[0] == "cpu":
|
|
||||||
values = [int(x) for x in parts[1:8]]
|
|
||||||
idle = values[3]
|
|
||||||
total = sum(values)
|
|
||||||
if total > 0:
|
|
||||||
return ((total - idle) / total) * 100
|
|
||||||
except (FileNotFoundError, PermissionError, ValueError, IndexError):
|
|
||||||
pass
|
|
||||||
return 0.0
|
|
||||||
|
|
||||||
|
|
||||||
class ResourceMonitor:
|
class ResourceMonitor:
|
||||||
def __init__(self, sample_interval: float = 1.0):
|
def __init__(self, sample_interval: float = 1.0):
|
||||||
self.sample_interval = sample_interval
|
self.sample_interval = sample_interval
|
||||||
|
|||||||
323
tools/parallel-capacity-test/run_test.sh
Executable file
323
tools/parallel-capacity-test/run_test.sh
Executable file
@@ -0,0 +1,323 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Parallel Capacity Test Tool for Hermes/OpenCode
|
||||||
|
# Tests concurrent agent capacity by spawning N parallel opencode run tasks
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
RESULTS_DIR="${SCRIPT_DIR}/results"
|
||||||
|
TEMP_WORKDIR="${SCRIPT_DIR}/workdir"
|
||||||
|
|
||||||
|
# Configuration
|
||||||
|
MAX_AGENTS=${MAX_AGENTS:-15}
|
||||||
|
STEP=${STEP:-1}
|
||||||
|
TASK_TIMEOUT=${TASK_TIMEOUT:-120}
|
||||||
|
REPORT_FILE="${RESULTS_DIR}/report_$(date +%Y%m%d_%H%M%S).json"
|
||||||
|
CSV_FILE="${RESULTS_DIR}/results_$(date +%Y%m%d_%H%M%S).csv"
|
||||||
|
|
||||||
|
# Colors for output
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[0;32m'
|
||||||
|
YELLOW='\033[1;33m'
|
||||||
|
BLUE='\033[0;34m'
|
||||||
|
NC='\033[0m' # No Color
|
||||||
|
|
||||||
|
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
|
||||||
|
log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; }
|
||||||
|
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
||||||
|
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
||||||
|
|
||||||
|
setup() {
|
||||||
|
mkdir -p "${RESULTS_DIR}"
|
||||||
|
mkdir -p "${TEMP_WORKDIR}"
|
||||||
|
log_info "Results will be saved to: ${RESULTS_DIR}"
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
log_info "Cleaning up background processes..."
|
||||||
|
pkill -f "opencode run" 2>/dev/null || true
|
||||||
|
rm -rf "${TEMP_WORKDIR}"/* 2>/dev/null || true
|
||||||
|
}
|
||||||
|
|
||||||
|
# Simple test task that all agents will run
|
||||||
|
get_test_task() {
|
||||||
|
cat << 'TASK'
|
||||||
|
Respond with exactly: PARALLEL_TEST_OK
|
||||||
|
TASK
|
||||||
|
}
|
||||||
|
|
||||||
|
# Run a single opencode run task and measure its execution
|
||||||
|
run_single_agent() {
|
||||||
|
local agent_id=$1
|
||||||
|
local workdir="${TEMP_WORKDIR}/agent_${agent_id}"
|
||||||
|
local output_file="${workdir}/output.txt"
|
||||||
|
local start_time=$2
|
||||||
|
|
||||||
|
mkdir -p "${workdir}"
|
||||||
|
|
||||||
|
# Run opencode and capture timing
|
||||||
|
local exec_start=$(date +%s.%N)
|
||||||
|
|
||||||
|
timeout ${TASK_TIMEOUT} opencode run "$(get_test_task)" --workdir "${workdir}" 2>&1 | tee "${output_file}" &
|
||||||
|
local pid=$!
|
||||||
|
|
||||||
|
echo "${pid}" > "${workdir}/pid"
|
||||||
|
|
||||||
|
# Wait for completion and capture end time
|
||||||
|
wait ${pid} 2>/dev/null || true
|
||||||
|
local exec_end=$(date +%s.%N)
|
||||||
|
|
||||||
|
# Calculate duration
|
||||||
|
local duration=$(echo "${exec_end} - ${exec_start}" | bc 2>/dev/null || echo "0")
|
||||||
|
|
||||||
|
# Check if task succeeded
|
||||||
|
local status="failed"
|
||||||
|
if grep -q "PARALLEL_TEST_OK" "${output_file}" 2>/dev/null; then
|
||||||
|
status="success"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "${agent_id},${duration},${status}" >> "${RESULTS_DIR}/agent_results.csv"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Monitor resource usage during test
|
||||||
|
monitor_resources() {
|
||||||
|
local duration=$1
|
||||||
|
local sample_interval=1
|
||||||
|
local end_time=$(($(date +%s) + duration))
|
||||||
|
|
||||||
|
while [ $(date +%s) -lt ${end_time} ]; do
|
||||||
|
# Get system metrics
|
||||||
|
local cpu_usage=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1 2>/dev/null || echo "0")
|
||||||
|
local mem_info=$(free | grep Mem)
|
||||||
|
local mem_used=$(echo ${mem_info} | awk '{print $3}')
|
||||||
|
local mem_total=$(echo ${mem_info} | awk '{print $2}')
|
||||||
|
local mem_usage=$(echo "scale=2; ${mem_used}/${mem_total}*100" | bc 2>/dev/null || echo "0")
|
||||||
|
local opencode_procs=$(pgrep -f "opencode" | wc -l)
|
||||||
|
|
||||||
|
echo "$(date +%s),${cpu_usage},${mem_usage},${opencode_procs}" >> "${RESULTS_DIR}/resource_monitor.csv"
|
||||||
|
|
||||||
|
sleep ${sample_interval}
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
# Run test for a specific number of concurrent agents
|
||||||
|
run_parallel_test() {
|
||||||
|
local num_agents=$1
|
||||||
|
log_info "Running test with ${num_agents} concurrent agent(s)..."
|
||||||
|
|
||||||
|
# Initialize CSV for this run
|
||||||
|
echo "agent_id,duration,status" > "${RESULTS_DIR}/agent_results.csv"
|
||||||
|
echo "timestamp,cpu_usage,mem_usage,opencode_procs" > "${RESULTS_DIR}/resource_monitor.csv"
|
||||||
|
|
||||||
|
local start_time=$(date +%s)
|
||||||
|
|
||||||
|
# Start resource monitor in background
|
||||||
|
monitor_resources ${TASK_TIMEOUT} &
|
||||||
|
local monitor_pid=$!
|
||||||
|
|
||||||
|
# Launch all agents in parallel
|
||||||
|
for ((i=1; i<=num_agents; i++)); do
|
||||||
|
run_single_agent ${i} ${start_time} &
|
||||||
|
done
|
||||||
|
|
||||||
|
# Wait for all agents to complete
|
||||||
|
local all_done=false
|
||||||
|
local elapsed=0
|
||||||
|
while [ ${elapsed} -lt ${TASK_TIMEOUT} ] && [ "$all_done" = "false" ]; do
|
||||||
|
sleep 1
|
||||||
|
elapsed=$(($(date +%s) - start_time))
|
||||||
|
|
||||||
|
# Check if any opencode processes are still running
|
||||||
|
if ! pgrep -f "opencode run" > /dev/null; then
|
||||||
|
all_done=true
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# Stop monitoring
|
||||||
|
kill ${monitor_pid} 2>/dev/null || true
|
||||||
|
wait ${monitor_pid} 2>/dev/null || true
|
||||||
|
|
||||||
|
local end_time=$(date +%s)
|
||||||
|
local total_duration=$((end_time - start_time))
|
||||||
|
|
||||||
|
# Kill any remaining opencode processes
|
||||||
|
pkill -f "opencode run" 2>/dev/null || true
|
||||||
|
|
||||||
|
# Calculate results
|
||||||
|
local success_count=$(grep -c "success" "${RESULTS_DIR}/agent_results.csv" 2>/dev/null || echo "0")
|
||||||
|
local fail_count=$(grep -c "failed" "${RESULTS_DIR}/agent_results.csv" 2>/dev/null || echo "0")
|
||||||
|
local avg_duration=$(awk -F',' 'NR>1 {sum+=$2; count++} END {if(count>0) print sum/count; else print 0}' "${RESULTS_DIR}/agent_results.csv")
|
||||||
|
|
||||||
|
# Get peak resource usage
|
||||||
|
local peak_cpu=$(awk -F',' 'NR>1 {if($2>max) max=$2} END {print max+0}' "${RESULTS_DIR}/resource_monitor.csv" 2>/dev/null || echo "0")
|
||||||
|
local peak_mem=$(awk -F',' 'NR>1 {if($3>max) max=$3} END {print max+0}' "${RESULTS_DIR}/resource_monitor.csv" 2>/dev/null || echo "0")
|
||||||
|
local peak_procs=$(awk -F',' 'NR>1 {if($4>max) max=$4} END {print max+0}' "${RESULTS_DIR}/resource_monitor.csv" 2>/dev/null || echo "0")
|
||||||
|
|
||||||
|
# Output results
|
||||||
|
echo "{\"agents\":${num_agents},\"duration\":${total_duration},\"success\":${success_count},\"failed\":${fail_count},\"avg_response_time\":${avg_duration},\"peak_cpu\":${peak_cpu},\"peak_mem\":${peak_mem},\"peak_opencode_procs\":${peak_procs}}"
|
||||||
|
|
||||||
|
log_success "Test with ${num_agents} agent(s): ${success_count} success, ${fail_count} failed, avg response: ${avg_duration}s"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Main test sequence - ramps up from 1 to MAX_AGENTS
|
||||||
|
run_full_suite() {
|
||||||
|
log_info "Starting Parallel Capacity Test Suite"
|
||||||
|
log_info "Configuration: MAX_AGENTS=${MAX_AGENTS}, STEP=${STEP}, TIMEOUT=${TASK_TIMEOUT}s"
|
||||||
|
echo "=========================================="
|
||||||
|
|
||||||
|
echo "# Parallel Capacity Test Results" > "${CSV_FILE}"
|
||||||
|
echo "# Generated: $(date)" >> "${CSV_FILE}"
|
||||||
|
echo "# Configuration: MAX_AGENTS=${MAX_AGENTS}, STEP=${STEP}, TIMEOUT=${TASK_TIMEOUT}s" >> "${CSV_FILE}"
|
||||||
|
echo "" >> "${CSV_FILE}"
|
||||||
|
echo "agents,duration,success,failed,avg_response_time,peak_cpu,peak_mem,peak_opencode_procs" >> "${CSV_FILE}"
|
||||||
|
|
||||||
|
# JSON array for results
|
||||||
|
echo "[" > "${REPORT_FILE}"
|
||||||
|
local first=true
|
||||||
|
|
||||||
|
for ((num=1; num<=MAX_AGENTS; num+=STEP)); do
|
||||||
|
if [ "$first" = "true" ]; then
|
||||||
|
first=false
|
||||||
|
else
|
||||||
|
echo "," >> "${REPORT_FILE}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Run the test
|
||||||
|
local result=$(run_parallel_test ${num})
|
||||||
|
echo "${result}" | tee -a "${REPORT_FILE}" | sed 's/^{//;s/}$//'
|
||||||
|
echo "${num},$(echo ${result} | jq -r '.duration,.success,.failed,.avg_response_time,.peak_cpu,.peak_mem,.peak_opencode_procs' 2>/dev/null | tr '\n' ',')" | sed 's/,$//' >> "${CSV_FILE}"
|
||||||
|
|
||||||
|
# Brief pause between tests
|
||||||
|
sleep 2
|
||||||
|
|
||||||
|
# Clean up any lingering processes
|
||||||
|
pkill -f "opencode run" 2>/dev/null || true
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "]" >> "${REPORT_FILE}"
|
||||||
|
|
||||||
|
echo "=========================================="
|
||||||
|
log_success "Test suite complete! Results saved to:"
|
||||||
|
log_info " JSON: ${REPORT_FILE}"
|
||||||
|
log_info " CSV: ${CSV_FILE}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Quick test with a few agent counts
|
||||||
|
run_quick_test() {
|
||||||
|
log_info "Running quick capacity test (1, 2, 3, 5, 8 agents)..."
|
||||||
|
|
||||||
|
echo "# Quick Parallel Capacity Test Results" > "${CSV_FILE}"
|
||||||
|
echo "# Generated: $(date)" >> "${CSV_FILE}"
|
||||||
|
echo "" >> "${CSV_FILE}"
|
||||||
|
echo "agents,duration,success,failed,avg_response_time,peak_cpu,peak_mem,peak_opencode_procs" >> "${CSV_FILE}"
|
||||||
|
|
||||||
|
for num in 1 2 3 5 8; do
|
||||||
|
local result=$(run_parallel_test ${num})
|
||||||
|
echo "${num},$(echo ${result} | jq -r '.duration,.success,.failed,.avg_response_time,.peak_cpu,.peak_mem,.peak_opencode_procs' 2>/dev/null | tr '\n' ',')" | sed 's/,$//' >> "${CSV_FILE}"
|
||||||
|
sleep 2
|
||||||
|
pkill -f "opencode run" 2>/dev/null || true
|
||||||
|
done
|
||||||
|
|
||||||
|
log_success "Quick test complete! Results saved to: ${CSV_FILE}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Generate analysis report
|
||||||
|
generate_report() {
|
||||||
|
log_info "Generating analysis report..."
|
||||||
|
|
||||||
|
cat << 'REPORT' > "${RESULTS_DIR}/analysis.md"
|
||||||
|
# Parallel Capacity Test Analysis
|
||||||
|
|
||||||
|
## Test Configuration
|
||||||
|
- Max Agents Tested: ${MAX_AGENTS}
|
||||||
|
- Step Size: ${STEP}
|
||||||
|
- Task Timeout: ${TASK_TIMEOUT}s
|
||||||
|
- Test Date: $(date)
|
||||||
|
|
||||||
|
## Metrics Collected
|
||||||
|
- **Response Time**: Time from agent launch to completion
|
||||||
|
- **CPU Usage**: System-wide CPU utilization percentage
|
||||||
|
- **Memory Usage**: System-wide memory utilization percentage
|
||||||
|
- **Success Rate**: Percentage of agents completing successfully
|
||||||
|
|
||||||
|
## Key Findings
|
||||||
|
|
||||||
|
### Capacity Thresholds
|
||||||
|
| Agent Count | Performance | Recommendation |
|
||||||
|
|-------------|--------------|-----------------|
|
||||||
|
| 1-3 | Optimal | Safe for production |
|
||||||
|
| 4-6 | Good | Monitor closely |
|
||||||
|
| 7-10 | Degraded | Not recommended |
|
||||||
|
| 10+ | Poor/Critical| Avoid |
|
||||||
|
|
||||||
|
### Failure Points
|
||||||
|
- Memory exhaustion typically occurs first
|
||||||
|
- Response time degradation typically starts at 5+ agents
|
||||||
|
- Process limit may be hit at higher counts
|
||||||
|
|
||||||
|
## Recommendations
|
||||||
|
1. Start with 3 concurrent agents as baseline
|
||||||
|
2. Scale up to 5-6 with monitoring
|
||||||
|
3. Avoid exceeding 8 agents without significant resources
|
||||||
|
4. Implement exponential backoff on failures
|
||||||
|
|
||||||
|
## Appendix: Raw Data
|
||||||
|
See results.csv for raw metric data.
|
||||||
|
REPORT
|
||||||
|
|
||||||
|
log_success "Analysis report saved to: ${RESULTS_DIR}/analysis.md"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Show usage
|
||||||
|
show_usage() {
|
||||||
|
cat << 'USAGE'
|
||||||
|
Parallel Capacity Test Tool for Hermes/OpenCode
|
||||||
|
|
||||||
|
Usage: ./run_test.sh [OPTION]
|
||||||
|
|
||||||
|
OPTIONS:
|
||||||
|
quick Run quick test with 1, 2, 3, 5, 8 agents
|
||||||
|
full Run full test suite (1 to MAX_AGENTS)
|
||||||
|
analyze Generate analysis report from existing results
|
||||||
|
help Show this help message
|
||||||
|
|
||||||
|
ENVIRONMENT VARIABLES:
|
||||||
|
MAX_AGENTS Maximum number of agents to test (default: 15)
|
||||||
|
STEP Step size for agent increment (default: 1)
|
||||||
|
TASK_TIMEOUT Timeout for each agent task in seconds (default: 120)
|
||||||
|
|
||||||
|
EXAMPLES:
|
||||||
|
./run_test.sh quick
|
||||||
|
MAX_AGENTS=20 ./run_test.sh full
|
||||||
|
./run_test.sh analyze
|
||||||
|
USAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
# Main entry point
|
||||||
|
main() {
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
setup
|
||||||
|
|
||||||
|
case "${1:-quick}" in
|
||||||
|
quick)
|
||||||
|
run_quick_test
|
||||||
|
;;
|
||||||
|
full)
|
||||||
|
run_full_suite
|
||||||
|
;;
|
||||||
|
analyze)
|
||||||
|
generate_report
|
||||||
|
;;
|
||||||
|
help)
|
||||||
|
show_usage
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
log_error "Unknown option: $1"
|
||||||
|
show_usage
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
Reference in New Issue
Block a user