From 94ec3b5ea42695f834793a0d9f32a9f549a68971 Mon Sep 17 00:00:00 2001 From: JY Tan Date: Tue, 24 Mar 2026 15:42:31 -0700 Subject: [PATCH 1/3] Commit --- cmd/short_docs/unit/overview.md | 6 ++-- cmd/unit_feedback.go | 17 +++++++++ cmd/unit_get_run.go | 3 ++ cmd/unit_retry.go | 63 +++++++++++++++++++++++++++++++++ docs/unit/README.md | 29 +++++++++++++-- internal/api/unit_runs.go | 11 ++++++ 6 files changed, 124 insertions(+), 5 deletions(-) create mode 100644 cmd/unit_retry.go diff --git a/cmd/short_docs/unit/overview.md b/cmd/short_docs/unit/overview.md index 1211e6e..621780c 100644 --- a/cmd/short_docs/unit/overview.md +++ b/cmd/short_docs/unit/overview.md @@ -10,9 +10,11 @@ The data-returning `tusk unit` subcommands output JSON, which makes them work we 2. Inspect the run and its generated scenarios with `tusk unit get-run `. 3. Review a specific scenario with `tusk unit get-scenario --run-id --scenario-id `. 4. Submit feedback from a file or stdin with `tusk unit feedback --run-id --file feedback.json`. - Use `positive_feedback` or `negative_feedback` to indicate the feedback type, and `applied_locally` if you kept the change locally. + Use `run_feedback.comment` for broad run-level guidance, `positive_feedback` or `negative_feedback` for scenario feedback, and `applied_locally` if you kept the change locally. See `tusk unit feedback --help` for more details. -5. Apply generated diffs with `tusk unit get-diffs | jq -r '.files[].diff' | git apply`. +5. Retry with broad run-level guidance when the run was broadly wrong: `tusk unit feedback --run-id --file feedback.json --retry` or `tusk unit retry --run-id --comment "Wrong mocks for this run"`. + This may a take a while. If the tests are mostly correct, prefer small local edits instead of a full retry. +6. Apply generated diffs with `tusk unit get-diffs | jq -r '.files[].diff' | git apply`. ## Authentication diff --git a/cmd/unit_feedback.go b/cmd/unit_feedback.go index 0fb6034..92a94ca 100644 --- a/cmd/unit_feedback.go +++ b/cmd/unit_feedback.go @@ -14,6 +14,7 @@ import ( var ( unitFeedbackRunID string unitFeedbackFile string + unitFeedbackRetry bool ) var unitFeedbackCmd = &cobra.Command{ @@ -22,9 +23,12 @@ var unitFeedbackCmd = &cobra.Command{ Long: `Submit feedback for one or more unit test scenarios. The feedback payload must be JSON, provided via --file or --file - for stdin. +It must include at least one run_feedback.comment or one scenario entry. +Use run_feedback.comment when the overall run should be retried with different guidance. Example usage: tusk unit feedback --run-id --file feedback.json +tusk unit feedback --run-id --file feedback.json --retry tusk unit feedback --run-id --file - <<'EOF' { "scenarios": [ @@ -40,6 +44,9 @@ EOF Example payload (schema reference): { + "run_feedback": { + "comment": "The run targeted the right files, but the mocks do not match the real service contracts and several scenarios are asserting on implementation details. Use simpler setup assumptions and focus on externally observable behavior." + }, "scenarios": [ { "scenario_id": "uuid", @@ -57,9 +64,11 @@ Example payload (schema reference): } Notes: +- Use run_feedback.comment mainly for broad retry guidance, such as wrong mocks, wrong symbols, or an overall incorrect test strategy. - Use either positive_feedback or negative_feedback for a scenario. - Allowed positive_feedback values: "covers_critical_path", "valid_edge_case", "caught_a_bug", "other" - Allowed negative_feedback values: "incorrect_business_assumption", "duplicates_existing_test", "no_value", "incorrect_assertion", "poor_coding_practice", "other" +- Add --retry when Tusk should start a new retry run after saving the feedback. This may a take a while. If the tests are mostly correct, prefer small local edits instead of a full retry. Thank you for your feedback and helping to improve Tusk! `, @@ -76,6 +85,13 @@ Thank you for your feedback and helping to improve Tusk! if err != nil { return err } + if unitFeedbackRetry { + obj, ok := payload.(map[string]any) + if !ok { + return fmt.Errorf("feedback payload must be a JSON object when using --retry") + } + obj["retry"] = true + } client, authOptions, err := setupUnitCloud() if err != nil { @@ -133,6 +149,7 @@ func init() { unitFeedbackCmd.Flags().StringVar(&unitFeedbackRunID, "run-id", "", "Unit test run ID") unitFeedbackCmd.Flags().StringVar(&unitFeedbackFile, "file", "", "Path to feedback JSON file, or `-` to read from stdin") + unitFeedbackCmd.Flags().BoolVar(&unitFeedbackRetry, "retry", false, "Trigger a retry after saving feedback") _ = unitFeedbackCmd.MarkFlagRequired("run-id") _ = unitFeedbackCmd.MarkFlagRequired("file") diff --git a/cmd/unit_get_run.go b/cmd/unit_get_run.go index c7d1a6c..58ceee0 100644 --- a/cmd/unit_get_run.go +++ b/cmd/unit_get_run.go @@ -48,6 +48,9 @@ func buildNextSteps(run map[string]any) []string { if hasScenarios { steps = append(steps, fmt.Sprintf("Review a test scenario: `tusk unit get-scenario --run-id %s --scenario-id `", runID)) steps = append(steps, fmt.Sprintf("Apply all diffs: `tusk unit get-diffs %s | jq -r '.files[].diff' | git apply`", runID)) + steps = append(steps, "If the tests are mostly correct, prefer small local edits instead of a full retry.") + steps = append(steps, fmt.Sprintf("If the run used the wrong mocks, symbols, or overall approach, submit feedback and retry: `tusk unit feedback --run-id %s --file feedback.json --retry`", runID)) + steps = append(steps, fmt.Sprintf("Or trigger an explicit retry with run-level guidance: `tusk unit retry --run-id %s --comment \"Wrong mocks for this run\"`", runID)) } else { steps = append(steps, "Run completed but no test scenarios were generated.") } diff --git a/cmd/unit_retry.go b/cmd/unit_retry.go new file mode 100644 index 0000000..862f462 --- /dev/null +++ b/cmd/unit_retry.go @@ -0,0 +1,63 @@ +package cmd + +import ( + "context" + "fmt" + "strings" + + "github.com/spf13/cobra" +) + +var ( + unitRetryRunID string + unitRetryComment string +) + +var unitRetryCmd = &cobra.Command{ + Use: "retry", + Short: "Trigger a retry for a unit test run", + Long: `Trigger a retry for a unit test run. + +Use --comment for broad retry guidance when the generated mocks, symbols, or +overall test direction were wrong for the run. + +Retries can take a while, so prefer small local edits when the generated tests +are mostly correct. + +Example usage: +tusk unit retry --run-id +tusk unit retry --run-id --comment "The run targeted the right files, but the mocks do not match the real service contracts and several scenarios assert on implementation details. Use simpler setup assumptions and focus on externally observable behavior." +`, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + if strings.TrimSpace(unitRetryRunID) == "" { + return fmt.Errorf("--run-id must be non-empty") + } + + payload := map[string]any{} + if trimmedComment := strings.TrimSpace(unitRetryComment); trimmedComment != "" { + payload["comment"] = trimmedComment + } + + client, authOptions, err := setupUnitCloud() + if err != nil { + return err + } + + result, err := client.RetryUnitTestRun(context.Background(), unitRetryRunID, payload, authOptions) + if err != nil { + return formatApiError(err) + } + + return printJSON(result) + }, +} + +func init() { + unitCmd.AddCommand(unitRetryCmd) + + unitRetryCmd.Flags().StringVar(&unitRetryRunID, "run-id", "", "Unit test run ID") + unitRetryCmd.Flags().StringVar(&unitRetryComment, "comment", "", "Optional run-level guidance to save before retrying") + + _ = unitRetryCmd.MarkFlagRequired("run-id") +} diff --git a/docs/unit/README.md b/docs/unit/README.md index 3ac9cd9..61761fb 100644 --- a/docs/unit/README.md +++ b/docs/unit/README.md @@ -51,10 +51,11 @@ tusk unit get-diffs ### `tusk unit feedback` -Submit feedback for one or more scenarios in a unit test run. +Submit run-level and/or scenario-level feedback for a unit test run. Add `--retry` to save the feedback and immediately trigger a retry. ```bash tusk unit feedback --run-id --file feedback.json +tusk unit feedback --run-id --file feedback.json --retry ``` You can also submit feedback inline with stdin: @@ -62,6 +63,9 @@ You can also submit feedback inline with stdin: ```bash tusk unit feedback --run-id --file - <<'EOF' { + "run_feedback": { + "comment": "The run targeted the right files, but the mocks do not match the real service contracts and several scenarios are asserting on implementation details. Use simpler setup assumptions and focus on externally observable behavior." + }, "scenarios": [ { "scenario_id": "uuid", @@ -80,6 +84,7 @@ tusk unit feedback --run-id --file - <<'EOF' EOF ``` +Use `run_feedback.comment` mainly for broad retry guidance, such as wrong mocks, wrong symbols, or an overall incorrect test strategy. Use either `positive_feedback` or `negative_feedback` for a scenario. Allowed `positive_feedback` values: @@ -98,6 +103,15 @@ Allowed `negative_feedback` values: - `poor_coding_practice` - `other` +### `tusk unit retry` + +Trigger a retry for a unit test run, optionally storing a broad run-level comment first. + +```bash +tusk unit retry --run-id +tusk unit retry --run-id --comment "The run targeted the right files, but the mocks do not match the real service contracts and several scenarios assert on implementation details. Use simpler setup assumptions and focus on externally observable behavior." +``` + ## Typical workflow 1. Check the latest run on your branch: @@ -118,13 +132,22 @@ Allowed `negative_feedback` values: tusk unit get-scenario --run-id --scenario-id ``` -4. Submit feedback on scenarios you kept or rejected: +4. Submit feedback on scenarios you kept or rejected, and optionally add broad run-level guidance: ```bash tusk unit feedback --run-id --file feedback.json ``` -5. Apply all generated tests to your working tree: +5. If the run was broadly wrong, trigger a retry with feedback instead of editing everything locally: + + ```bash + tusk unit feedback --run-id --file feedback.json --retry + tusk unit retry --run-id --comment "Wrong mocks for this run" + ``` + + Retries may take a while. If the tests are mostly correct, prefer small local edits instead of a full retry. + +6. Apply all generated tests to your working tree when they are close and only need small edits: ```bash tusk unit get-diffs | jq -r '.files[].diff' | git apply diff --git a/internal/api/unit_runs.go b/internal/api/unit_runs.go index fb047ad..66d6176 100644 --- a/internal/api/unit_runs.go +++ b/internal/api/unit_runs.go @@ -89,6 +89,17 @@ func (c *TuskClient) SubmitUnitTestFeedback(ctx context.Context, runID string, p return out, nil } +type UnitTestRetryResult map[string]any + +func (c *TuskClient) RetryUnitTestRun(ctx context.Context, runID string, payload any, auth AuthOptions) (UnitTestRetryResult, error) { + var out UnitTestRetryResult + path := fmt.Sprintf("/api/v1/unit_test_run/%s/retry", url.PathEscape(runID)) + if err := c.makeJSONRequestWithBody(ctx, http.MethodPost, path, nil, payload, &out, auth); err != nil { + return nil, err + } + return out, nil +} + func (c *TuskClient) GetUnitTestRun(ctx context.Context, runID string, auth AuthOptions) (UnitTestRunDetails, error) { var out UnitTestRunDetails if err := c.makeJSONRequest(ctx, http.MethodGet, "/api/v1/unit_test_run/"+url.PathEscape(runID), nil, &out, auth); err != nil { From e072c7b97ac6660bad28d8f976a80ea3eed0d597 Mon Sep 17 00:00:00 2001 From: JY Tan Date: Tue, 24 Mar 2026 15:47:01 -0700 Subject: [PATCH 2/3] Update cmd/short_docs/unit/overview.md Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> --- cmd/short_docs/unit/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/short_docs/unit/overview.md b/cmd/short_docs/unit/overview.md index 621780c..2a33f62 100644 --- a/cmd/short_docs/unit/overview.md +++ b/cmd/short_docs/unit/overview.md @@ -13,7 +13,7 @@ The data-returning `tusk unit` subcommands output JSON, which makes them work we Use `run_feedback.comment` for broad run-level guidance, `positive_feedback` or `negative_feedback` for scenario feedback, and `applied_locally` if you kept the change locally. See `tusk unit feedback --help` for more details. 5. Retry with broad run-level guidance when the run was broadly wrong: `tusk unit feedback --run-id --file feedback.json --retry` or `tusk unit retry --run-id --comment "Wrong mocks for this run"`. - This may a take a while. If the tests are mostly correct, prefer small local edits instead of a full retry. + This may take a while. If the tests are mostly correct, prefer small local edits instead of a full retry. 6. Apply generated diffs with `tusk unit get-diffs | jq -r '.files[].diff' | git apply`. ## Authentication From ef05e55ec3d97ee88df14c8d7460a0e531c10679 Mon Sep 17 00:00:00 2001 From: JY Tan Date: Tue, 24 Mar 2026 16:20:48 -0700 Subject: [PATCH 3/3] Update retry guidance --- cmd/short_docs/unit/overview.md | 2 +- cmd/unit_feedback.go | 5 +++-- cmd/unit_retry.go | 5 +++-- docs/unit/README.md | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd/short_docs/unit/overview.md b/cmd/short_docs/unit/overview.md index 2a33f62..f267154 100644 --- a/cmd/short_docs/unit/overview.md +++ b/cmd/short_docs/unit/overview.md @@ -6,7 +6,7 @@ The data-returning `tusk unit` subcommands output JSON, which makes them work we ## Typical workflow -1. Check the latest run for the current repo and branch with `tusk unit latest-run`. +1. Check the latest run for the current repo and branch with `tusk unit latest-run` to see the latest result plus recent history, including a readable `run_type_label`, the raw `run_type`, `commit_sha`, and any `retry_feedback`. 2. Inspect the run and its generated scenarios with `tusk unit get-run `. 3. Review a specific scenario with `tusk unit get-scenario --run-id --scenario-id `. 4. Submit feedback from a file or stdin with `tusk unit feedback --run-id --file feedback.json`. diff --git a/cmd/unit_feedback.go b/cmd/unit_feedback.go index 92a94ca..a48126e 100644 --- a/cmd/unit_feedback.go +++ b/cmd/unit_feedback.go @@ -24,7 +24,7 @@ var unitFeedbackCmd = &cobra.Command{ The feedback payload must be JSON, provided via --file or --file - for stdin. It must include at least one run_feedback.comment or one scenario entry. -Use run_feedback.comment when the overall run should be retried with different guidance. +Use run_feedback.comment when the user wants Tusk to retry the overall run with different guidance, or when the required fixes are too broad to make locally. Example usage: tusk unit feedback --run-id --file feedback.json @@ -64,11 +64,12 @@ Example payload (schema reference): } Notes: +- Prefer local edits by default when the generated tests are mostly correct. - Use run_feedback.comment mainly for broad retry guidance, such as wrong mocks, wrong symbols, or an overall incorrect test strategy. - Use either positive_feedback or negative_feedback for a scenario. - Allowed positive_feedback values: "covers_critical_path", "valid_edge_case", "caught_a_bug", "other" - Allowed negative_feedback values: "incorrect_business_assumption", "duplicates_existing_test", "no_value", "incorrect_assertion", "poor_coding_practice", "other" -- Add --retry when Tusk should start a new retry run after saving the feedback. This may a take a while. If the tests are mostly correct, prefer small local edits instead of a full retry. +- Add --retry when the user has asked Tusk to regenerate the run, or when the changes are too large to fix locally. This may take a while. Thank you for your feedback and helping to improve Tusk! `, diff --git a/cmd/unit_retry.go b/cmd/unit_retry.go index 862f462..5be348a 100644 --- a/cmd/unit_retry.go +++ b/cmd/unit_retry.go @@ -21,8 +21,9 @@ var unitRetryCmd = &cobra.Command{ Use --comment for broad retry guidance when the generated mocks, symbols, or overall test direction were wrong for the run. -Retries can take a while, so prefer small local edits when the generated tests -are mostly correct. +For agents: prefer small local edits when the generated tests are mostly correct. Use this +command when the user has asked to regenerate the run, or when the required changes are too +broad to fix locally. Example usage: tusk unit retry --run-id diff --git a/docs/unit/README.md b/docs/unit/README.md index 61761fb..485ff88 100644 --- a/docs/unit/README.md +++ b/docs/unit/README.md @@ -23,7 +23,7 @@ tusk unit latest-run tusk unit latest-run --repo owner/repo --branch feature-branch ``` -Returns a summary of the latest run plus a history of recent runs on the branch. +Returns a summary of the latest run plus a history of recent runs on the branch. History items include a readable `run_type_label`, the raw `run_type`, `commit_sha`, and `retry_feedback` when present. ### `tusk unit get-run `