Compare commits
12 Commits
8f950f575a
...
9cb39a1779
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9cb39a1779 | ||
|
|
3a841716fc | ||
|
|
2b60ec1acd | ||
|
|
bb11d665a3 | ||
|
|
dc26098918 | ||
|
|
1b51229f88 | ||
|
|
7eb83454ba | ||
|
|
94f08c1b8d | ||
|
|
2010275dda | ||
| 6102022be0 | |||
|
|
b1dc002b09 | ||
|
|
63b89eed6d |
66
skills/opencode-worktree/SKILL.md
Normal file
66
skills/opencode-worktree/SKILL.md
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
# opencode-worktree
|
||||||
|
|
||||||
|
Isolated OpenCode sessions via git worktrees.
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
Each OpenCode session gets its own git worktree with a unique branch. This prevents:
|
||||||
|
- Clashes with parallel sessions on the same repo
|
||||||
|
- Accidental overwrites from multiple agents
|
||||||
|
- Confusion from work-in-progress across contexts
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
- Git
|
||||||
|
- opencode installed and configured
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
|
### Option 1: Source directly (Recommended)
|
||||||
|
```bash
|
||||||
|
. skills/opencode-worktree/opencode-worktree.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
### Option 2: Copy to PATH
|
||||||
|
```bash
|
||||||
|
cp skills/opencode-worktree/opencode-worktree.sh ~/.local/bin/opencode-worktree
|
||||||
|
chmod +x ~/.local/bin/opencode-worktree
|
||||||
|
```
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
### Create new session
|
||||||
|
```bash
|
||||||
|
. opencode-worktree.sh # session-20260327-a1b2c3
|
||||||
|
. opencode-worktree.sh refactor-auth # session-20260327-a1b2c3-refactor-auth
|
||||||
|
```
|
||||||
|
|
||||||
|
### Cleanup
|
||||||
|
```bash
|
||||||
|
. opencode-worktree.sh --cleanup # remove all session-* worktrees
|
||||||
|
. opencode-worktree.sh --cleanup <name> # remove specific worktree
|
||||||
|
```
|
||||||
|
|
||||||
|
## How It Works
|
||||||
|
|
||||||
|
1. **Cleanup** - On every launch, removes all stale `session-*` worktrees and their branches
|
||||||
|
2. **Create** - Creates new worktree based on `main` with unique name: `session-{timestamp}-{random6}[-{purpose}]`
|
||||||
|
3. **Launch** - Changes into worktree and launches opencode
|
||||||
|
4. **Exit** - When opencode exits, you return to your original directory (worktree remains for review)
|
||||||
|
|
||||||
|
## Example Workflow
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Start session for refactoring auth
|
||||||
|
. opencode-worktree.sh refactor-auth
|
||||||
|
|
||||||
|
# ... do work in opencode ...
|
||||||
|
|
||||||
|
# Exit opencode (worktree with your changes still exists)
|
||||||
|
# Later, resume or cleanup
|
||||||
|
. opencode-worktree.sh --cleanup session-20260327-a1b2c3-refactor-auth
|
||||||
|
```
|
||||||
|
|
||||||
|
## Script
|
||||||
|
|
||||||
|
See [opencode-worktree.sh](./opencode-worktree.sh) for the full source.
|
||||||
148
skills/opencode-worktree/opencode-worktree.sh
Normal file
148
skills/opencode-worktree/opencode-worktree.sh
Normal file
@@ -0,0 +1,148 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# opencode-worktree - Isolated OpenCode sessions via git worktrees
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
WORKTREE_BASE="$PWD/.git/worktrees"
|
||||||
|
PURPOSE=""
|
||||||
|
CLEANUP_ONLY=false
|
||||||
|
CLEANUP_NAME=""
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<EOF
|
||||||
|
Usage: $(basename "$0") [purpose] [--cleanup [name]]
|
||||||
|
|
||||||
|
Create isolated OpenCode sessions via git worktrees.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
purpose Purpose string for the session (optional)
|
||||||
|
--cleanup Remove all session-* worktrees
|
||||||
|
--cleanup <name> Remove specific worktree by name
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
$(basename "$0") # session-20260327-a1b2c3
|
||||||
|
$(basename "$0") refactor-auth # session-20260327-a1b2c3-refactor-auth
|
||||||
|
$(basename "$0") --cleanup # remove all session-* worktrees
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_args() {
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--cleanup)
|
||||||
|
CLEANUP_ONLY=true
|
||||||
|
if [[ $# -gt 1 && ! "$2" =~ ^-- ]]; then
|
||||||
|
CLEANUP_NAME="$2"
|
||||||
|
shift
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
-h|--help)
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
if [[ -z "$PURPOSE" ]]; then
|
||||||
|
PURPOSE="$1"
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup_stale() {
|
||||||
|
if [[ ! -d "$WORKTREE_BASE" ]]; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
for wt in "$WORKTREE_BASE"/session-*; do
|
||||||
|
[[ -d "$wt" ]] || continue
|
||||||
|
|
||||||
|
branch=$(git -C "$wt" rev-parse --abbrev-ref HEAD 2>/dev/null) || continue
|
||||||
|
|
||||||
|
echo "Removing stale worktree: $(basename "$wt")"
|
||||||
|
git worktree remove "$wt" --force 2>/dev/null || true
|
||||||
|
if [[ -n "$branch" && "$branch" != "HEAD" ]]; then
|
||||||
|
git branch -D "$branch" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup_single() {
|
||||||
|
local name="$1"
|
||||||
|
local wt_path="$WORKTREE_BASE/$name"
|
||||||
|
|
||||||
|
if [[ ! -d "$wt_path" ]]; then
|
||||||
|
echo "Worktree '$name' not found"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
branch=$(git -C "$wt_path" rev-parse --abbrev-ref HEAD 2>/dev/null) || branch=""
|
||||||
|
|
||||||
|
echo "Removing worktree: $name"
|
||||||
|
git worktree remove "$wt_path" --force 2>/dev/null || true
|
||||||
|
if [[ -n "$branch" && "$branch" != "HEAD" ]]; then
|
||||||
|
git branch -D "$branch" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
create_worktree() {
|
||||||
|
local timestamp=$(date +%Y%m%d-%H%M%S)
|
||||||
|
local random=$(head -c 3 /dev/urandom | xxd -p | head -c 6)
|
||||||
|
local worktree_name="session-${timestamp}-${random}"
|
||||||
|
local branch_name="$worktree_name"
|
||||||
|
|
||||||
|
if [[ -n "$PURPOSE" ]]; then
|
||||||
|
worktree_name="${worktree_name}-${PURPOSE}"
|
||||||
|
branch_name="$worktree_name"
|
||||||
|
fi
|
||||||
|
|
||||||
|
local worktree_path_abs
|
||||||
|
worktree_path_abs="$(realpath -m "$WORKTREE_BASE")/$worktree_name"
|
||||||
|
local worktree_path="$worktree_path_abs"
|
||||||
|
|
||||||
|
# Cleanup any existing with same name
|
||||||
|
if [[ -d "$worktree_path" ]]; then
|
||||||
|
echo "Removing existing worktree: $worktree_name"
|
||||||
|
git worktree remove "$worktree_path" --force 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Ensure main exists and is up to date
|
||||||
|
if ! git show-ref --quiet refs/heads/main 2>/dev/null; then
|
||||||
|
echo "Error: 'main' branch does not exist"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Create worktree from main
|
||||||
|
echo "Creating worktree: $worktree_name"
|
||||||
|
git worktree add -b "$branch_name" "$worktree_path" main
|
||||||
|
|
||||||
|
# Launch opencode in worktree
|
||||||
|
echo "Entering worktree and launching opencode..."
|
||||||
|
cd "$worktree_path"
|
||||||
|
exec /home/shoko/.opencode/bin/opencode.real
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
# Verify we're in a git repo
|
||||||
|
if ! git rev-parse --is-inside-work-tree 2>/dev/null; then
|
||||||
|
echo "Error: Must be run inside a git repository"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$CLEANUP_ONLY" == true ]]; then
|
||||||
|
if [[ -n "$CLEANUP_NAME" ]]; then
|
||||||
|
cleanup_single "$CLEANUP_NAME"
|
||||||
|
else
|
||||||
|
cleanup_stale
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
cleanup_stale
|
||||||
|
create_worktree
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_args "$@"
|
||||||
|
main
|
||||||
@@ -1,74 +0,0 @@
|
|||||||
# Parallel Capacity Test Tool
|
|
||||||
|
|
||||||
Tests the practical limits of parallel agent execution for Hermes/OpenCode.
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
This tool stress tests Hermes to find the practical limit of parallel agent execution on the target machine. It:
|
|
||||||
|
|
||||||
- Spawns N concurrent `opencode run` agents
|
|
||||||
- Measures CPU, memory, and response time
|
|
||||||
- Ramps up from 1 to higher agent counts
|
|
||||||
- Identifies failure points and performance degradation
|
|
||||||
|
|
||||||
## Files
|
|
||||||
|
|
||||||
- `run_test.sh` - Bash script for running tests
|
|
||||||
- `parallel_capacity_test.py` - Python tool with more detailed metrics
|
|
||||||
- `results/` - Directory where test results are saved
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
### Quick Test (1, 2, 3, 5, 8 agents)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd tools/parallel-capacity-test
|
|
||||||
./parallel_capacity_test.py --quick
|
|
||||||
```
|
|
||||||
|
|
||||||
### Full Test Suite
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./parallel_capacity_test.py --agents 15 --timeout 120
|
|
||||||
```
|
|
||||||
|
|
||||||
### Bash Script Usage
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./run_test.sh quick # Quick test
|
|
||||||
./run_test.sh full # Full test up to MAX_AGENTS
|
|
||||||
```
|
|
||||||
|
|
||||||
## Environment Variables
|
|
||||||
|
|
||||||
| Variable | Default | Description |
|
|
||||||
|----------|---------|-------------|
|
|
||||||
| MAX_AGENTS | 15 | Maximum number of agents to test |
|
|
||||||
| STEP | 1 | Step size for agent increment |
|
|
||||||
| TASK_TIMEOUT | 120 | Timeout for each agent task |
|
|
||||||
|
|
||||||
## Metrics Collected
|
|
||||||
|
|
||||||
- **Response Time** - Time from agent launch to completion
|
|
||||||
- **CPU Usage** - System-wide CPU utilization percentage
|
|
||||||
- **Memory Usage** - System-wide memory utilization percentage
|
|
||||||
- **Success Rate** - Percentage of agents completing successfully
|
|
||||||
- **Process Count** - Number of opencode processes running
|
|
||||||
|
|
||||||
## Expected Behavior
|
|
||||||
|
|
||||||
Based on the Hermes architecture:
|
|
||||||
|
|
||||||
| Agent Count | Expected Performance |
|
|
||||||
|-------------|---------------------|
|
|
||||||
| 1-3 | Optimal - safe for production |
|
|
||||||
| 4-6 | Good - monitor closely |
|
|
||||||
| 7-10 | Degraded - not recommended |
|
|
||||||
| 10+ | Poor - avoid without significant resources |
|
|
||||||
|
|
||||||
## Output Files
|
|
||||||
|
|
||||||
- `results_YYYYMMDD_HHMMSS.json` - Complete raw results
|
|
||||||
- `summary_YYYYMMDD_HHMMSS.csv` - CSV summary of metrics
|
|
||||||
- `report_YYYYMMDD_HHMMSS.md` - Markdown analysis report
|
|
||||||
EOF; __hermes_rc=$?; printf '__HERMES_FENCE_a9f7b3__'; exit $__hermes_rc
|
|
||||||
@@ -17,12 +17,7 @@ from datetime import datetime
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import List, Optional
|
from typing import List, Optional
|
||||||
|
|
||||||
try:
|
# Using stdlib only - no psutil required
|
||||||
import psutil
|
|
||||||
HAS_PSUTIL = True
|
|
||||||
except ImportError:
|
|
||||||
HAS_PSUTIL = False
|
|
||||||
print("[WARN] psutil not available - resource monitoring will be limited")
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -61,6 +56,74 @@ class TestRun:
|
|||||||
peak_opencode_procs: int
|
peak_opencode_procs: int
|
||||||
|
|
||||||
|
|
||||||
|
def get_memory_percent() -> float:
|
||||||
|
"""Get memory usage percent by reading /proc/meminfo (Linux)"""
|
||||||
|
try:
|
||||||
|
with open("/proc/meminfo", "r") as f:
|
||||||
|
meminfo = f.read()
|
||||||
|
total = 0
|
||||||
|
available = 0
|
||||||
|
for line in meminfo.splitlines():
|
||||||
|
if line.startswith("MemTotal:"):
|
||||||
|
total = int(line.split()[1])
|
||||||
|
elif line.startswith("MemAvailable:"):
|
||||||
|
available = int(line.split()[1])
|
||||||
|
break
|
||||||
|
if total > 0:
|
||||||
|
used = total - available
|
||||||
|
return (used / total) * 100
|
||||||
|
except (FileNotFoundError, PermissionError, ValueError):
|
||||||
|
pass
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def count_opencode_processes() -> int:
|
||||||
|
"""Count opencode processes using pgrep or /proc scanning"""
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
["pgrep", "-c", "-x", "opencode"],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=5
|
||||||
|
)
|
||||||
|
if result.returncode == 0:
|
||||||
|
return int(result.stdout.strip())
|
||||||
|
except (subprocess.TimeoutExpired, ValueError, subprocess.SubprocessError):
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
count = 0
|
||||||
|
for pid_dir in os.listdir("/proc"):
|
||||||
|
if not pid_dir.isdigit():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
with open(f"/proc/{pid_dir}/comm", "r") as f:
|
||||||
|
if "opencode" in f.read().lower():
|
||||||
|
count += 1
|
||||||
|
except (PermissionError, FileNotFoundError):
|
||||||
|
continue
|
||||||
|
return count
|
||||||
|
except FileNotFoundError:
|
||||||
|
return 0
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def get_cpu_percent() -> float:
|
||||||
|
"""Get CPU usage by reading /proc/stat"""
|
||||||
|
try:
|
||||||
|
with open("/proc/stat", "r") as f:
|
||||||
|
line = f.readline()
|
||||||
|
parts = line.split()
|
||||||
|
if parts[0] == "cpu":
|
||||||
|
values = [int(x) for x in parts[1:8]]
|
||||||
|
idle = values[3]
|
||||||
|
total = sum(values)
|
||||||
|
if total > 0:
|
||||||
|
return ((total - idle) / total) * 100
|
||||||
|
except (FileNotFoundError, PermissionError, ValueError, IndexError):
|
||||||
|
pass
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
class ResourceMonitor:
|
class ResourceMonitor:
|
||||||
def __init__(self, sample_interval: float = 1.0):
|
def __init__(self, sample_interval: float = 1.0):
|
||||||
self.sample_interval = sample_interval
|
self.sample_interval = sample_interval
|
||||||
|
|||||||
@@ -1,323 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Parallel Capacity Test Tool for Hermes/OpenCode
|
|
||||||
# Tests concurrent agent capacity by spawning N parallel opencode run tasks
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
||||||
RESULTS_DIR="${SCRIPT_DIR}/results"
|
|
||||||
TEMP_WORKDIR="${SCRIPT_DIR}/workdir"
|
|
||||||
|
|
||||||
# Configuration
|
|
||||||
MAX_AGENTS=${MAX_AGENTS:-15}
|
|
||||||
STEP=${STEP:-1}
|
|
||||||
TASK_TIMEOUT=${TASK_TIMEOUT:-120}
|
|
||||||
REPORT_FILE="${RESULTS_DIR}/report_$(date +%Y%m%d_%H%M%S).json"
|
|
||||||
CSV_FILE="${RESULTS_DIR}/results_$(date +%Y%m%d_%H%M%S).csv"
|
|
||||||
|
|
||||||
# Colors for output
|
|
||||||
RED='\033[0;31m'
|
|
||||||
GREEN='\033[0;32m'
|
|
||||||
YELLOW='\033[1;33m'
|
|
||||||
BLUE='\033[0;34m'
|
|
||||||
NC='\033[0m' # No Color
|
|
||||||
|
|
||||||
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
|
|
||||||
log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; }
|
|
||||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
|
||||||
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
|
||||||
|
|
||||||
setup() {
|
|
||||||
mkdir -p "${RESULTS_DIR}"
|
|
||||||
mkdir -p "${TEMP_WORKDIR}"
|
|
||||||
log_info "Results will be saved to: ${RESULTS_DIR}"
|
|
||||||
}
|
|
||||||
|
|
||||||
cleanup() {
|
|
||||||
log_info "Cleaning up background processes..."
|
|
||||||
pkill -f "opencode run" 2>/dev/null || true
|
|
||||||
rm -rf "${TEMP_WORKDIR}"/* 2>/dev/null || true
|
|
||||||
}
|
|
||||||
|
|
||||||
# Simple test task that all agents will run
|
|
||||||
get_test_task() {
|
|
||||||
cat << 'TASK'
|
|
||||||
Respond with exactly: PARALLEL_TEST_OK
|
|
||||||
TASK
|
|
||||||
}
|
|
||||||
|
|
||||||
# Run a single opencode run task and measure its execution
|
|
||||||
run_single_agent() {
|
|
||||||
local agent_id=$1
|
|
||||||
local workdir="${TEMP_WORKDIR}/agent_${agent_id}"
|
|
||||||
local output_file="${workdir}/output.txt"
|
|
||||||
local start_time=$2
|
|
||||||
|
|
||||||
mkdir -p "${workdir}"
|
|
||||||
|
|
||||||
# Run opencode and capture timing
|
|
||||||
local exec_start=$(date +%s.%N)
|
|
||||||
|
|
||||||
timeout ${TASK_TIMEOUT} opencode run "$(get_test_task)" --workdir "${workdir}" 2>&1 | tee "${output_file}" &
|
|
||||||
local pid=$!
|
|
||||||
|
|
||||||
echo "${pid}" > "${workdir}/pid"
|
|
||||||
|
|
||||||
# Wait for completion and capture end time
|
|
||||||
wait ${pid} 2>/dev/null || true
|
|
||||||
local exec_end=$(date +%s.%N)
|
|
||||||
|
|
||||||
# Calculate duration
|
|
||||||
local duration=$(echo "${exec_end} - ${exec_start}" | bc 2>/dev/null || echo "0")
|
|
||||||
|
|
||||||
# Check if task succeeded
|
|
||||||
local status="failed"
|
|
||||||
if grep -q "PARALLEL_TEST_OK" "${output_file}" 2>/dev/null; then
|
|
||||||
status="success"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "${agent_id},${duration},${status}" >> "${RESULTS_DIR}/agent_results.csv"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Monitor resource usage during test
|
|
||||||
monitor_resources() {
|
|
||||||
local duration=$1
|
|
||||||
local sample_interval=1
|
|
||||||
local end_time=$(($(date +%s) + duration))
|
|
||||||
|
|
||||||
while [ $(date +%s) -lt ${end_time} ]; do
|
|
||||||
# Get system metrics
|
|
||||||
local cpu_usage=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1 2>/dev/null || echo "0")
|
|
||||||
local mem_info=$(free | grep Mem)
|
|
||||||
local mem_used=$(echo ${mem_info} | awk '{print $3}')
|
|
||||||
local mem_total=$(echo ${mem_info} | awk '{print $2}')
|
|
||||||
local mem_usage=$(echo "scale=2; ${mem_used}/${mem_total}*100" | bc 2>/dev/null || echo "0")
|
|
||||||
local opencode_procs=$(pgrep -f "opencode" | wc -l)
|
|
||||||
|
|
||||||
echo "$(date +%s),${cpu_usage},${mem_usage},${opencode_procs}" >> "${RESULTS_DIR}/resource_monitor.csv"
|
|
||||||
|
|
||||||
sleep ${sample_interval}
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
# Run test for a specific number of concurrent agents
|
|
||||||
run_parallel_test() {
|
|
||||||
local num_agents=$1
|
|
||||||
log_info "Running test with ${num_agents} concurrent agent(s)..."
|
|
||||||
|
|
||||||
# Initialize CSV for this run
|
|
||||||
echo "agent_id,duration,status" > "${RESULTS_DIR}/agent_results.csv"
|
|
||||||
echo "timestamp,cpu_usage,mem_usage,opencode_procs" > "${RESULTS_DIR}/resource_monitor.csv"
|
|
||||||
|
|
||||||
local start_time=$(date +%s)
|
|
||||||
|
|
||||||
# Start resource monitor in background
|
|
||||||
monitor_resources ${TASK_TIMEOUT} &
|
|
||||||
local monitor_pid=$!
|
|
||||||
|
|
||||||
# Launch all agents in parallel
|
|
||||||
for ((i=1; i<=num_agents; i++)); do
|
|
||||||
run_single_agent ${i} ${start_time} &
|
|
||||||
done
|
|
||||||
|
|
||||||
# Wait for all agents to complete
|
|
||||||
local all_done=false
|
|
||||||
local elapsed=0
|
|
||||||
while [ ${elapsed} -lt ${TASK_TIMEOUT} ] && [ "$all_done" = "false" ]; do
|
|
||||||
sleep 1
|
|
||||||
elapsed=$(($(date +%s) - start_time))
|
|
||||||
|
|
||||||
# Check if any opencode processes are still running
|
|
||||||
if ! pgrep -f "opencode run" > /dev/null; then
|
|
||||||
all_done=true
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
# Stop monitoring
|
|
||||||
kill ${monitor_pid} 2>/dev/null || true
|
|
||||||
wait ${monitor_pid} 2>/dev/null || true
|
|
||||||
|
|
||||||
local end_time=$(date +%s)
|
|
||||||
local total_duration=$((end_time - start_time))
|
|
||||||
|
|
||||||
# Kill any remaining opencode processes
|
|
||||||
pkill -f "opencode run" 2>/dev/null || true
|
|
||||||
|
|
||||||
# Calculate results
|
|
||||||
local success_count=$(grep -c "success" "${RESULTS_DIR}/agent_results.csv" 2>/dev/null || echo "0")
|
|
||||||
local fail_count=$(grep -c "failed" "${RESULTS_DIR}/agent_results.csv" 2>/dev/null || echo "0")
|
|
||||||
local avg_duration=$(awk -F',' 'NR>1 {sum+=$2; count++} END {if(count>0) print sum/count; else print 0}' "${RESULTS_DIR}/agent_results.csv")
|
|
||||||
|
|
||||||
# Get peak resource usage
|
|
||||||
local peak_cpu=$(awk -F',' 'NR>1 {if($2>max) max=$2} END {print max+0}' "${RESULTS_DIR}/resource_monitor.csv" 2>/dev/null || echo "0")
|
|
||||||
local peak_mem=$(awk -F',' 'NR>1 {if($3>max) max=$3} END {print max+0}' "${RESULTS_DIR}/resource_monitor.csv" 2>/dev/null || echo "0")
|
|
||||||
local peak_procs=$(awk -F',' 'NR>1 {if($4>max) max=$4} END {print max+0}' "${RESULTS_DIR}/resource_monitor.csv" 2>/dev/null || echo "0")
|
|
||||||
|
|
||||||
# Output results
|
|
||||||
echo "{\"agents\":${num_agents},\"duration\":${total_duration},\"success\":${success_count},\"failed\":${fail_count},\"avg_response_time\":${avg_duration},\"peak_cpu\":${peak_cpu},\"peak_mem\":${peak_mem},\"peak_opencode_procs\":${peak_procs}}"
|
|
||||||
|
|
||||||
log_success "Test with ${num_agents} agent(s): ${success_count} success, ${fail_count} failed, avg response: ${avg_duration}s"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Main test sequence - ramps up from 1 to MAX_AGENTS
|
|
||||||
run_full_suite() {
|
|
||||||
log_info "Starting Parallel Capacity Test Suite"
|
|
||||||
log_info "Configuration: MAX_AGENTS=${MAX_AGENTS}, STEP=${STEP}, TIMEOUT=${TASK_TIMEOUT}s"
|
|
||||||
echo "=========================================="
|
|
||||||
|
|
||||||
echo "# Parallel Capacity Test Results" > "${CSV_FILE}"
|
|
||||||
echo "# Generated: $(date)" >> "${CSV_FILE}"
|
|
||||||
echo "# Configuration: MAX_AGENTS=${MAX_AGENTS}, STEP=${STEP}, TIMEOUT=${TASK_TIMEOUT}s" >> "${CSV_FILE}"
|
|
||||||
echo "" >> "${CSV_FILE}"
|
|
||||||
echo "agents,duration,success,failed,avg_response_time,peak_cpu,peak_mem,peak_opencode_procs" >> "${CSV_FILE}"
|
|
||||||
|
|
||||||
# JSON array for results
|
|
||||||
echo "[" > "${REPORT_FILE}"
|
|
||||||
local first=true
|
|
||||||
|
|
||||||
for ((num=1; num<=MAX_AGENTS; num+=STEP)); do
|
|
||||||
if [ "$first" = "true" ]; then
|
|
||||||
first=false
|
|
||||||
else
|
|
||||||
echo "," >> "${REPORT_FILE}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Run the test
|
|
||||||
local result=$(run_parallel_test ${num})
|
|
||||||
echo "${result}" | tee -a "${REPORT_FILE}" | sed 's/^{//;s/}$//'
|
|
||||||
echo "${num},$(echo ${result} | jq -r '.duration,.success,.failed,.avg_response_time,.peak_cpu,.peak_mem,.peak_opencode_procs' 2>/dev/null | tr '\n' ',')" | sed 's/,$//' >> "${CSV_FILE}"
|
|
||||||
|
|
||||||
# Brief pause between tests
|
|
||||||
sleep 2
|
|
||||||
|
|
||||||
# Clean up any lingering processes
|
|
||||||
pkill -f "opencode run" 2>/dev/null || true
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "]" >> "${REPORT_FILE}"
|
|
||||||
|
|
||||||
echo "=========================================="
|
|
||||||
log_success "Test suite complete! Results saved to:"
|
|
||||||
log_info " JSON: ${REPORT_FILE}"
|
|
||||||
log_info " CSV: ${CSV_FILE}"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Quick test with a few agent counts
|
|
||||||
run_quick_test() {
|
|
||||||
log_info "Running quick capacity test (1, 2, 3, 5, 8 agents)..."
|
|
||||||
|
|
||||||
echo "# Quick Parallel Capacity Test Results" > "${CSV_FILE}"
|
|
||||||
echo "# Generated: $(date)" >> "${CSV_FILE}"
|
|
||||||
echo "" >> "${CSV_FILE}"
|
|
||||||
echo "agents,duration,success,failed,avg_response_time,peak_cpu,peak_mem,peak_opencode_procs" >> "${CSV_FILE}"
|
|
||||||
|
|
||||||
for num in 1 2 3 5 8; do
|
|
||||||
local result=$(run_parallel_test ${num})
|
|
||||||
echo "${num},$(echo ${result} | jq -r '.duration,.success,.failed,.avg_response_time,.peak_cpu,.peak_mem,.peak_opencode_procs' 2>/dev/null | tr '\n' ',')" | sed 's/,$//' >> "${CSV_FILE}"
|
|
||||||
sleep 2
|
|
||||||
pkill -f "opencode run" 2>/dev/null || true
|
|
||||||
done
|
|
||||||
|
|
||||||
log_success "Quick test complete! Results saved to: ${CSV_FILE}"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Generate analysis report
|
|
||||||
generate_report() {
|
|
||||||
log_info "Generating analysis report..."
|
|
||||||
|
|
||||||
cat << 'REPORT' > "${RESULTS_DIR}/analysis.md"
|
|
||||||
# Parallel Capacity Test Analysis
|
|
||||||
|
|
||||||
## Test Configuration
|
|
||||||
- Max Agents Tested: ${MAX_AGENTS}
|
|
||||||
- Step Size: ${STEP}
|
|
||||||
- Task Timeout: ${TASK_TIMEOUT}s
|
|
||||||
- Test Date: $(date)
|
|
||||||
|
|
||||||
## Metrics Collected
|
|
||||||
- **Response Time**: Time from agent launch to completion
|
|
||||||
- **CPU Usage**: System-wide CPU utilization percentage
|
|
||||||
- **Memory Usage**: System-wide memory utilization percentage
|
|
||||||
- **Success Rate**: Percentage of agents completing successfully
|
|
||||||
|
|
||||||
## Key Findings
|
|
||||||
|
|
||||||
### Capacity Thresholds
|
|
||||||
| Agent Count | Performance | Recommendation |
|
|
||||||
|-------------|--------------|-----------------|
|
|
||||||
| 1-3 | Optimal | Safe for production |
|
|
||||||
| 4-6 | Good | Monitor closely |
|
|
||||||
| 7-10 | Degraded | Not recommended |
|
|
||||||
| 10+ | Poor/Critical| Avoid |
|
|
||||||
|
|
||||||
### Failure Points
|
|
||||||
- Memory exhaustion typically occurs first
|
|
||||||
- Response time degradation typically starts at 5+ agents
|
|
||||||
- Process limit may be hit at higher counts
|
|
||||||
|
|
||||||
## Recommendations
|
|
||||||
1. Start with 3 concurrent agents as baseline
|
|
||||||
2. Scale up to 5-6 with monitoring
|
|
||||||
3. Avoid exceeding 8 agents without significant resources
|
|
||||||
4. Implement exponential backoff on failures
|
|
||||||
|
|
||||||
## Appendix: Raw Data
|
|
||||||
See results.csv for raw metric data.
|
|
||||||
REPORT
|
|
||||||
|
|
||||||
log_success "Analysis report saved to: ${RESULTS_DIR}/analysis.md"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Show usage
|
|
||||||
show_usage() {
|
|
||||||
cat << 'USAGE'
|
|
||||||
Parallel Capacity Test Tool for Hermes/OpenCode
|
|
||||||
|
|
||||||
Usage: ./run_test.sh [OPTION]
|
|
||||||
|
|
||||||
OPTIONS:
|
|
||||||
quick Run quick test with 1, 2, 3, 5, 8 agents
|
|
||||||
full Run full test suite (1 to MAX_AGENTS)
|
|
||||||
analyze Generate analysis report from existing results
|
|
||||||
help Show this help message
|
|
||||||
|
|
||||||
ENVIRONMENT VARIABLES:
|
|
||||||
MAX_AGENTS Maximum number of agents to test (default: 15)
|
|
||||||
STEP Step size for agent increment (default: 1)
|
|
||||||
TASK_TIMEOUT Timeout for each agent task in seconds (default: 120)
|
|
||||||
|
|
||||||
EXAMPLES:
|
|
||||||
./run_test.sh quick
|
|
||||||
MAX_AGENTS=20 ./run_test.sh full
|
|
||||||
./run_test.sh analyze
|
|
||||||
USAGE
|
|
||||||
}
|
|
||||||
|
|
||||||
# Main entry point
|
|
||||||
main() {
|
|
||||||
trap cleanup EXIT
|
|
||||||
|
|
||||||
setup
|
|
||||||
|
|
||||||
case "${1:-quick}" in
|
|
||||||
quick)
|
|
||||||
run_quick_test
|
|
||||||
;;
|
|
||||||
full)
|
|
||||||
run_full_suite
|
|
||||||
;;
|
|
||||||
analyze)
|
|
||||||
generate_report
|
|
||||||
;;
|
|
||||||
help)
|
|
||||||
show_usage
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
log_error "Unknown option: $1"
|
|
||||||
show_usage
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
Reference in New Issue
Block a user