Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion cmd/short_docs/unit/overview.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,10 @@ The data-returning `tusk unit` subcommands output JSON, which makes them work we
1. Check the latest run for the current repo and branch with `tusk unit latest-run`.
2. Inspect the run and its generated scenarios with `tusk unit get-run <run-id>`.
3. Review a specific scenario with `tusk unit get-scenario --run-id <run-id> --scenario-id <scenario-id>`.
4. Apply generated diffs with `tusk unit get-diffs <run-id> | jq -r '.files[].diff' | git apply`.
4. Submit feedback from a file or stdin with `tusk unit feedback --run-id <run-id> --file feedback.json`.
Use `positive_feedback` or `negative_feedback` to indicate the feedback type, and `applied_locally` if you kept the change locally.
See `tusk unit feedback --help` for more details.
5. Apply generated diffs with `tusk unit get-diffs <run-id> | jq -r '.files[].diff' | git apply`.

## Authentication

Expand Down
139 changes: 139 additions & 0 deletions cmd/unit_feedback.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
package cmd

import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"strings"

"github.com/spf13/cobra"
)

var (
unitFeedbackRunID string
unitFeedbackFile string
)

var unitFeedbackCmd = &cobra.Command{
Use: "feedback",
Short: "Submit feedback for one or more unit test scenarios",
Long: `Submit feedback for one or more unit test scenarios.

The feedback payload must be JSON, provided via --file <path> or --file - for stdin.

Example usage:
tusk unit feedback --run-id <run-id> --file feedback.json
tusk unit feedback --run-id <run-id> --file - <<'EOF'
{
"scenarios": [
{
"scenario_id": "uuid",
"positive_feedback": ["covers_critical_path"],
"comment": "Good scenario and likely worth keeping.",
"applied_locally": true
}
]
}
EOF

Example payload (schema reference):
{
"scenarios": [
{
"scenario_id": "uuid",
"positive_feedback": ["covers_critical_path"],
"comment": "Good scenario and likely worth keeping.",
"applied_locally": true
},
{
"scenario_id": "uuid",
"negative_feedback": ["incorrect_assertion"],
"comment": "The generated assertion does not match the behavior we want to preserve, so we did not keep this test.",
"applied_locally": false
}
]
}

Notes:
- Use either positive_feedback or negative_feedback for a scenario.
- Allowed positive_feedback values: "covers_critical_path", "valid_edge_case", "caught_a_bug", "other"
- Allowed negative_feedback values: "incorrect_business_assumption", "duplicates_existing_test", "no_value", "incorrect_assertion", "poor_coding_practice", "other"

Thank you for your feedback and helping to improve Tusk!
`,
SilenceUsage: true,
RunE: func(cmd *cobra.Command, args []string) error {
if strings.TrimSpace(unitFeedbackRunID) == "" {
return fmt.Errorf("--run-id must be non-empty")
}
if strings.TrimSpace(unitFeedbackFile) == "" {
return fmt.Errorf("--file must be provided")
}

payload, err := readUnitFeedbackPayload(unitFeedbackFile)
if err != nil {
return err
}

client, authOptions, err := setupUnitCloud()
if err != nil {
return err
}

result, err := client.SubmitUnitTestFeedback(context.Background(), unitFeedbackRunID, payload, authOptions)
if err != nil {
return formatApiError(err)
}

return printJSON(result)
},
}

func readUnitFeedbackPayload(path string) (any, error) {
var raw []byte
var err error

if path == "-" {
raw, err = io.ReadAll(os.Stdin)
if err != nil {
return nil, fmt.Errorf("read stdin: %w", err)
}
} else {
raw, err = os.ReadFile(path) //nolint:gosec
if err != nil {
return nil, fmt.Errorf("read feedback file: %w", err)
}
}

if len(strings.TrimSpace(string(raw))) == 0 {
return nil, fmt.Errorf("feedback payload is empty")
}

var payload any
if err := json.Unmarshal(raw, &payload); err != nil {
return nil, fmt.Errorf("parse feedback json: %w", err)
}

if scenarios, ok := payload.([]any); ok {
return map[string]any{"scenarios": scenarios}, nil
}

obj, ok := payload.(map[string]any)
if !ok {
return nil, fmt.Errorf("feedback payload must be a JSON object or an array of scenario entries")
}

return obj, nil
}

func init() {
unitCmd.AddCommand(unitFeedbackCmd)

unitFeedbackCmd.Flags().StringVar(&unitFeedbackRunID, "run-id", "", "Unit test run ID")
unitFeedbackCmd.Flags().StringVar(&unitFeedbackFile, "file", "", "Path to feedback JSON file, or `-` to read from stdin")

_ = unitFeedbackCmd.MarkFlagRequired("run-id")
_ = unitFeedbackCmd.MarkFlagRequired("file")
}
57 changes: 56 additions & 1 deletion docs/unit/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,55 @@ Get file diffs for a unit test run. Diffs are in unified diff format, ready to a
tusk unit get-diffs <run-id>
```

### `tusk unit feedback`

Submit feedback for one or more scenarios in a unit test run.

```bash
tusk unit feedback --run-id <run-id> --file feedback.json
```

You can also submit feedback inline with stdin:

```bash
tusk unit feedback --run-id <run-id> --file - <<'EOF'
{
"scenarios": [
{
"scenario_id": "uuid",
"positive_feedback": ["covers_critical_path"],
"comment": "Good scenario and likely worth keeping.",
"applied_locally": true
},
{
"scenario_id": "uuid",
"negative_feedback": ["incorrect_assertion"],
"comment": "The generated assertion does not match the behavior we want to preserve, so we did not keep this test.",
"applied_locally": false
}
]
}
EOF
```

Use either `positive_feedback` or `negative_feedback` for a scenario.

Allowed `positive_feedback` values:

- `covers_critical_path`
- `valid_edge_case`
- `caught_a_bug`
- `other`

Allowed `negative_feedback` values:

- `incorrect_business_assumption`
- `duplicates_existing_test`
- `no_value`
- `incorrect_assertion`
- `poor_coding_practice`
- `other`

## Typical workflow

1. Check the latest run on your branch:
Expand All @@ -69,7 +118,13 @@ tusk unit get-diffs <run-id>
tusk unit get-scenario --run-id <run-id> --scenario-id <scenario-id>
```

4. Apply all generated tests to your working tree:
4. Submit feedback on scenarios you kept or rejected:

```bash
tusk unit feedback --run-id <run-id> --file feedback.json
```

5. Apply all generated tests to your working tree:

```bash
tusk unit get-diffs <run-id> | jq -r '.files[].diff' | git apply
Expand Down
32 changes: 31 additions & 1 deletion internal/api/unit_runs.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package api

import (
"bytes"
"context"
"encoding/json"
"fmt"
Expand All @@ -15,17 +16,35 @@ type (
)

func (c *TuskClient) makeJSONRequest(ctx context.Context, method string, path string, query url.Values, out any, auth AuthOptions) error {
return c.makeJSONRequestWithBody(ctx, method, path, query, nil, out, auth)
}

func (c *TuskClient) makeJSONRequestWithBody(ctx context.Context, method string, path string, query url.Values, payload any, out any, auth AuthOptions) error {
fullURL := c.baseURL + path
if len(query) > 0 {
fullURL += "?" + query.Encode()
}

httpReq, err := buildAuthenticatedRequest(ctx, method, fullURL, nil, auth)
var bodyReader *bytes.Reader
if payload == nil {
bodyReader = bytes.NewReader(nil)
} else {
body, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("encode json: %w", err)
}
bodyReader = bytes.NewReader(body)
}

httpReq, err := buildAuthenticatedRequest(ctx, method, fullURL, bodyReader, auth)
if err != nil {
return err
}
httpReq.Header.Set("Accept", "application/json")
httpReq.Header.Set("X-Tusk-Source", "cli")
if payload != nil {
httpReq.Header.Set("Content-Type", "application/json")
}

body, httpResp, err := c.executeRequest(httpReq)
if err != nil {
Expand Down Expand Up @@ -59,6 +78,17 @@ func (c *TuskClient) GetLatestUnitTestRun(ctx context.Context, repo string, bran
return out, nil
}

type UnitTestFeedbackResult map[string]any

func (c *TuskClient) SubmitUnitTestFeedback(ctx context.Context, runID string, payload any, auth AuthOptions) (UnitTestFeedbackResult, error) {
var out UnitTestFeedbackResult
path := fmt.Sprintf("/api/v1/unit_test_run/%s/feedback", url.PathEscape(runID))
if err := c.makeJSONRequestWithBody(ctx, http.MethodPost, path, nil, payload, &out, auth); err != nil {
return nil, err
}
return out, nil
}

func (c *TuskClient) GetUnitTestRun(ctx context.Context, runID string, auth AuthOptions) (UnitTestRunDetails, error) {
var out UnitTestRunDetails
if err := c.makeJSONRequest(ctx, http.MethodGet, "/api/v1/unit_test_run/"+url.PathEscape(runID), nil, &out, auth); err != nil {
Expand Down
Loading