146 lines
4.3 KiB
Go
146 lines
4.3 KiB
Go
package eval
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os/exec"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// maxOutputBytes caps Claude CLI stdout. A normal A-G eval is well under 100KB
|
|
// of markdown; 5MB leaves headroom for streaming tool output while bounding
|
|
// any runaway subprocess.
|
|
const maxOutputBytes = 5 * 1024 * 1024
|
|
|
|
// Client wraps the Claude CLI subprocess. Safe to reuse across evals.
|
|
type Client struct {
|
|
CLIPath string
|
|
Timeout time.Duration
|
|
}
|
|
|
|
// NewClient returns a Client bound to the system `claude` binary with a
|
|
// generous default timeout suited to A-G evaluations (WebSearch calls for
|
|
// Block D can push total runtime past the 2-minute mark).
|
|
func NewClient() *Client {
|
|
return &Client{
|
|
CLIPath: "claude",
|
|
Timeout: 10 * time.Minute,
|
|
}
|
|
}
|
|
|
|
// ChunkHandler receives raw text chunks as they stream from the subprocess.
|
|
// Returning false cancels the eval. Use to implement Block-G-first ghost-job
|
|
// cutoff from the caller side without coupling this package to policy.
|
|
type ChunkHandler func(text string) bool
|
|
|
|
// Evaluate runs the Claude CLI against the given prompt and returns the
|
|
// parsed Evaluation. The prompt is piped via stdin (never argv) so that:
|
|
// - arbitrarily long prompts don't hit ARG_MAX,
|
|
// - prompt contents don't leak to /proc or `ps` output,
|
|
// - shell metacharacters in the prompt are inert (no bash -c).
|
|
//
|
|
// onChunk is optional; nil means buffer-only. Returning false from onChunk
|
|
// cancels the context and terminates the subprocess.
|
|
func (c *Client) Evaluate(ctx context.Context, prompt string, onChunk ChunkHandler) (Evaluation, error) {
|
|
if c.CLIPath == "" {
|
|
return Evaluation{}, errors.New("claude cli path not configured")
|
|
}
|
|
if strings.TrimSpace(prompt) == "" {
|
|
return Evaluation{}, errors.New("empty prompt")
|
|
}
|
|
|
|
runCtx, cancel := context.WithTimeout(ctx, c.Timeout)
|
|
defer cancel()
|
|
|
|
// --print: non-interactive mode. Prompt is read from stdin when no
|
|
// positional prompt arg is supplied. --output-format text keeps stdout
|
|
// as plain markdown, which is what parser.go consumes.
|
|
cmd := exec.CommandContext(runCtx, c.CLIPath, "--print", "--output-format", "text")
|
|
|
|
stdin, err := cmd.StdinPipe()
|
|
if err != nil {
|
|
return Evaluation{}, fmt.Errorf("stdin pipe: %w", err)
|
|
}
|
|
stdout, err := cmd.StdoutPipe()
|
|
if err != nil {
|
|
return Evaluation{}, fmt.Errorf("stdout pipe: %w", err)
|
|
}
|
|
var stderr bytes.Buffer
|
|
cmd.Stderr = &stderr
|
|
|
|
started := time.Now()
|
|
if err := cmd.Start(); err != nil {
|
|
return Evaluation{}, fmt.Errorf("start claude: %w", err)
|
|
}
|
|
|
|
// Feed the prompt on stdin in a goroutine so we can start reading stdout
|
|
// immediately. Close stdin when done so claude knows the input is complete.
|
|
go func() {
|
|
defer stdin.Close()
|
|
_, _ = io.WriteString(stdin, prompt)
|
|
}()
|
|
|
|
// Read stdout line-by-line, buffering the full output while also routing
|
|
// chunks to the caller. LimitReader enforces the output cap regardless of
|
|
// subprocess behavior.
|
|
var buf bytes.Buffer
|
|
reader := bufio.NewReader(io.LimitReader(stdout, maxOutputBytes))
|
|
for {
|
|
line, err := reader.ReadString('\n')
|
|
if len(line) > 0 {
|
|
buf.WriteString(line)
|
|
if onChunk != nil && !onChunk(line) {
|
|
cancel()
|
|
break
|
|
}
|
|
}
|
|
if err != nil {
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
// Drain any remaining bytes before surfacing the error — this
|
|
// ensures stderr is captured even when the stream breaks mid-way.
|
|
_ = cmd.Wait()
|
|
return Evaluation{}, fmt.Errorf("read stdout: %w (stderr: %s)", err, stderr.String())
|
|
}
|
|
}
|
|
|
|
waitErr := cmd.Wait()
|
|
elapsed := time.Since(started).Milliseconds()
|
|
|
|
if waitErr != nil && runCtx.Err() == nil {
|
|
return Evaluation{}, fmt.Errorf("claude exit: %w (stderr: %s)", waitErr, stderr.String())
|
|
}
|
|
if runCtx.Err() == context.DeadlineExceeded {
|
|
return Evaluation{}, fmt.Errorf("claude timeout after %s", c.Timeout)
|
|
}
|
|
|
|
eval, err := ParseEvaluation(buf.String())
|
|
if err != nil {
|
|
return Evaluation{Raw: buf.String(), ElapsedMS: elapsed, ReceivedAt: time.Now()},
|
|
fmt.Errorf("parse: %w", err)
|
|
}
|
|
eval.Raw = buf.String()
|
|
eval.ElapsedMS = elapsed
|
|
eval.ReceivedAt = time.Now()
|
|
return eval, nil
|
|
}
|
|
|
|
// Available verifies the Claude CLI is on PATH. Called at startup to fail
|
|
// loudly rather than at first eval.
|
|
func (c *Client) Available() error {
|
|
path := c.CLIPath
|
|
if path == "" {
|
|
path = "claude"
|
|
}
|
|
if _, err := exec.LookPath(path); err != nil {
|
|
return fmt.Errorf("claude cli not found on PATH: %w", err)
|
|
}
|
|
return nil
|
|
}
|