Skip to content

chore: remove verify script dead code #4102

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
melmennaoui wants to merge 1 commit into from
+0 −178
Closed

chore: remove verify script dead code #4102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 0 additions & 79 deletions pkg/evaluation/verify.go
View file
Open in desktop

This file was deleted.

78 changes: 0 additions & 78 deletions pkg/evaluation/verify_test.go
View file
Open in desktop

This file was deleted.

4 changes: 0 additions & 4 deletions pkg/session/branch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -367,10 +367,6 @@ func cloneEvalResultChecks(src EvalResultChecks) EvalResultChecks {
assertions.Results = slices.Clone(src.Assertions.Results)
cp.Assertions = &assertions
}
if src.Verify != nil {
verify := *src.Verify
cp.Verify = &verify
}
return cp
}

Expand Down
9 changes: 0 additions & 9 deletions pkg/session/session.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -642,7 +642,6 @@ type EvalResultChecks struct {
ToolCalls *ToolCallsCheck `json:"tool_calls,omitempty"`
Relevance *RelevanceCheck `json:"relevance,omitempty"`
Assertions *AssertionsCheck `json:"assertions,omitempty"`
Verify *VerifyCheck `json:"verify,omitempty"`
}

// SizeCheck contains the result of the response size check.
Expand Down Expand Up @@ -689,18 +688,10 @@ type AssertionResult struct {
Reason string `json:"reason,omitempty"`
}

// VerifyCheck contains the result of the post-agent verify script.
type VerifyCheck struct {
Passed bool `json:"passed"`
ExitCode int `json:"exit_code"`
Output string `json:"output,omitempty"`
}

// EvalCriteria contains the evaluation criteria for a session.
type EvalCriteria struct {
Relevance []string `json:"relevance"` // Statements that should be true about the response
Assertions []Assertion `json:"assertions,omitempty"` // Code-based assertions evaluated against the agent output
Verify string `json:"verify,omitempty"` // Shell script for post-agent outcome verification
WorkingDir string `json:"working_dir,omitempty"` // Subdirectory under evals/working_dirs/
Size string `json:"size,omitempty"` // Expected response size: S, M, L, XL
Setup string `json:"setup,omitempty"` // Optional sh script to run in the container before docker agent run --exec
Expand Down
8 changes: 0 additions & 8 deletions pkg/session/session_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -520,14 +520,6 @@ func TestEvalCriteriaUnmarshalJSON(t *testing.T) {
Assertions: []Assertion{{Name: "has greeting", Type: "contains", Value: "hello"}},
},
},
{
name: "valid with verify",
input: `{"relevance":[],"verify":"test -f output.txt"}`,
want: EvalCriteria{
Relevance: []string{},
Verify: "test -f output.txt",
},
},
{
name: "empty object",
input: `{}`,
Expand Down
Loading