Skip to content

Commit 2ad06b7

Browse files
committed
Implement Sift CLI and hosted runtime foundation
1 parent 3da00d7 commit 2ad06b7

52 files changed

Lines changed: 10064 additions & 11 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/ci.yml

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name: CI
2+
3+
on:
4+
push:
5+
pull_request:
6+
7+
jobs:
8+
validate:
9+
runs-on: ubuntu-latest
10+
11+
steps:
12+
- name: Checkout
13+
uses: actions/checkout@v4
14+
15+
- name: Setup Go
16+
uses: actions/setup-go@v5
17+
with:
18+
go-version-file: go.mod
19+
cache: true
20+
21+
- name: Run Unit Tests
22+
run: go test ./...
23+
24+
- name: Run Clustering Eval Gate
25+
run: go run ./cmd/sift eval clustering --dataset docs/contracts/clustering-eval.v0.json --threshold 0.82 --target-precision 0.90 --min-pairs 100 --format text

.gitignore

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,12 @@ build/
77
data/
88
tmp/
99

10+
# Runtime state (SQLite + backups + run artifacts)
11+
state/
12+
13+
# Derived projections
14+
output/
15+
16+
# Local build artifact
17+
/sift
18+
/siftd

README.md

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,14 +65,35 @@ Sift is informed by local work and adjacent patterns:
6565

6666
## Current status
6767

68-
Docs-first project.
68+
Docs-first project with implementation in progress.
6969

70-
The implementation does not exist yet. This repo currently defines:
70+
The current build remains local-first for v0.
7171

72-
- what the system is;
73-
- why it should exist;
74-
- what v0 must and must not do;
75-
- which contracts agents and future services will rely on.
72+
The first hosted paid slice is defined separately in [docs/plans/2026-03-07-sift-pro-mvp.md](docs/plans/2026-03-07-sift-pro-mvp.md).
73+
74+
The narrow implementation path for that hosted slice is defined in [docs/plans/2026-03-07-sift-pro-execution-plan.md](docs/plans/2026-03-07-sift-pro-execution-plan.md).
75+
76+
Implemented now:
77+
78+
- Go module and single `sift` CLI scaffold;
79+
- SQLite bootstrap with migrations, run logging, article persistence, and event storage;
80+
- source registry loader and validation;
81+
- working `sift sources`;
82+
- working `sift sync` modes (`full`, `--fetch-only`, `--cluster-only`) with live feed fetch, canonical URL normalization, article dedupe, and deterministic event clustering/scoring;
83+
- working retrieval commands: `sift latest`, `sift search`, `sift event get`, `sift digest`;
84+
- digest projections published atomically to `output/digests/<scope>/<window>.{json,md}`.
85+
- `sift sync` in `full` and `--cluster-only` modes automatically refreshes `crypto` digests for `24h` and `7d`.
86+
- `sift eval clustering` precision gate on labeled title pairs (`>=100` pairs, precision `>=0.90`).
87+
- hosted `siftd` server scaffold with:
88+
- Postgres canonical store bootstrap + migrations;
89+
- in-process scheduler (`pipeline.RunSync`) over Postgres;
90+
- health and readiness endpoints (`/healthz`, `/readyz`);
91+
- Zitadel-protected read API (`/v1/events`, `/v1/events/{event_id}`, `/v1/digests/{scope}/{window}`);
92+
- authenticated WebSocket stream endpoint (`/v1/ws`) with post-sync update notifications (`event.upserted`, `digest.updated`).
93+
94+
Not implemented yet:
95+
96+
- human web UI.
7697

7798
## Docs
7899

@@ -82,3 +103,6 @@ The implementation does not exist yet. This repo currently defines:
82103
- [project.manifest.json](project.manifest.json)
83104
- [docs/README.md](docs/README.md)
84105
- [docs/plans/2026-03-06-sift-v0-execution-spec.md](docs/plans/2026-03-06-sift-v0-execution-spec.md)
106+
- [docs/plans/2026-03-07-sift-pro-mvp.md](docs/plans/2026-03-07-sift-pro-mvp.md)
107+
- [docs/plans/2026-03-07-sift-pro-execution-plan.md](docs/plans/2026-03-07-sift-pro-execution-plan.md)
108+
- [docs/plans/2026-03-07-sift-pro-slice-1-blueprint.md](docs/plans/2026-03-07-sift-pro-slice-1-blueprint.md)

cmd/sift/digest.go

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"os"
7+
"strings"
8+
"time"
9+
10+
digestproj "sift/internal/digest"
11+
)
12+
13+
type digestOptions struct {
14+
StateDir string
15+
OutputDir string
16+
Scope string
17+
Window string
18+
Format string
19+
}
20+
21+
func runDigest(ctx context.Context, args []string) error {
22+
opts, err := parseDigestOptions(args)
23+
if err != nil {
24+
return &commandError{
25+
Code: exitInvalidArguments,
26+
Err: err,
27+
}
28+
}
29+
30+
store, err := openStore(ctx, opts.StateDir)
31+
if err != nil {
32+
return err
33+
}
34+
defer store.Close()
35+
36+
records, err := store.ListEvents(ctx)
37+
if err != nil {
38+
return &commandError{
39+
Code: exitOperationalFailure,
40+
Err: err,
41+
}
42+
}
43+
44+
projection, err := digestproj.BuildProjection(records, opts.OutputDir, opts.Scope, opts.Window, time.Now().UTC())
45+
if err != nil {
46+
return &commandError{
47+
Code: exitOperationalFailure,
48+
Err: err,
49+
}
50+
}
51+
52+
if err := digestproj.PublishProjection(projection); err != nil {
53+
return &commandError{
54+
Code: exitOperationalFailure,
55+
Err: err,
56+
}
57+
}
58+
59+
switch opts.Format {
60+
case "json":
61+
if err := writeJSON(os.Stdout, projection.Envelope); err != nil {
62+
return &commandError{
63+
Code: exitOperationalFailure,
64+
Err: err,
65+
}
66+
}
67+
case "md":
68+
fmt.Print(projection.Markdown)
69+
case "text":
70+
fmt.Printf(
71+
"scope=%s window=%s generated_at=%s event_count=%d markdown_path=%s\n",
72+
projection.Envelope.Scope,
73+
projection.Envelope.Window,
74+
projection.Envelope.GeneratedAt,
75+
len(projection.Envelope.EventIDs),
76+
projection.Envelope.MarkdownPath,
77+
)
78+
default:
79+
return &commandError{
80+
Code: exitInvalidArguments,
81+
Err: fmt.Errorf("unsupported format %q, expected json, text, or md", opts.Format),
82+
}
83+
}
84+
85+
return nil
86+
}
87+
88+
func parseDigestOptions(args []string) (digestOptions, error) {
89+
opts := digestOptions{
90+
StateDir: "state",
91+
OutputDir: "output",
92+
Window: "24h",
93+
Format: "json",
94+
}
95+
96+
for i := 0; i < len(args); i++ {
97+
arg := args[i]
98+
99+
switch {
100+
case arg == "--state-dir":
101+
i++
102+
if i >= len(args) || strings.TrimSpace(args[i]) == "" {
103+
return digestOptions{}, fmt.Errorf("missing value for --state-dir")
104+
}
105+
opts.StateDir = args[i]
106+
case strings.HasPrefix(arg, "--state-dir="):
107+
value := strings.TrimPrefix(arg, "--state-dir=")
108+
if strings.TrimSpace(value) == "" {
109+
return digestOptions{}, fmt.Errorf("missing value for --state-dir")
110+
}
111+
opts.StateDir = value
112+
case arg == "--output-dir":
113+
i++
114+
if i >= len(args) || strings.TrimSpace(args[i]) == "" {
115+
return digestOptions{}, fmt.Errorf("missing value for --output-dir")
116+
}
117+
opts.OutputDir = args[i]
118+
case strings.HasPrefix(arg, "--output-dir="):
119+
value := strings.TrimPrefix(arg, "--output-dir=")
120+
if strings.TrimSpace(value) == "" {
121+
return digestOptions{}, fmt.Errorf("missing value for --output-dir")
122+
}
123+
opts.OutputDir = value
124+
case arg == "--window":
125+
i++
126+
if i >= len(args) || strings.TrimSpace(args[i]) == "" {
127+
return digestOptions{}, fmt.Errorf("missing value for --window")
128+
}
129+
opts.Window = args[i]
130+
case strings.HasPrefix(arg, "--window="):
131+
value := strings.TrimPrefix(arg, "--window=")
132+
if strings.TrimSpace(value) == "" {
133+
return digestOptions{}, fmt.Errorf("missing value for --window")
134+
}
135+
opts.Window = value
136+
case arg == "--format":
137+
i++
138+
if i >= len(args) || strings.TrimSpace(args[i]) == "" {
139+
return digestOptions{}, fmt.Errorf("missing value for --format")
140+
}
141+
opts.Format = args[i]
142+
case strings.HasPrefix(arg, "--format="):
143+
value := strings.TrimPrefix(arg, "--format=")
144+
if strings.TrimSpace(value) == "" {
145+
return digestOptions{}, fmt.Errorf("missing value for --format")
146+
}
147+
opts.Format = value
148+
case strings.HasPrefix(arg, "-"):
149+
return digestOptions{}, fmt.Errorf("unknown flag: %s", arg)
150+
default:
151+
if opts.Scope != "" {
152+
return digestOptions{}, fmt.Errorf("usage: sift digest <scope> [--window 24h] [--format json|text|md]")
153+
}
154+
opts.Scope = strings.ToLower(strings.TrimSpace(arg))
155+
}
156+
}
157+
158+
if opts.Scope == "" {
159+
return digestOptions{}, fmt.Errorf("usage: sift digest <scope> [--window 24h] [--format json|text|md]")
160+
}
161+
162+
if opts.Format != "json" && opts.Format != "text" && opts.Format != "md" {
163+
return digestOptions{}, fmt.Errorf("unsupported format %q, expected json, text, or md", opts.Format)
164+
}
165+
166+
return opts, nil
167+
}

cmd/sift/digest_test.go

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
package main
2+
3+
import "testing"
4+
5+
func TestParseDigestOptionsScopeBeforeFlags(t *testing.T) {
6+
t.Parallel()
7+
8+
opts, err := parseDigestOptions([]string{"crypto", "--window", "7d", "--format", "md"})
9+
if err != nil {
10+
t.Fatalf("parseDigestOptions returned error: %v", err)
11+
}
12+
13+
if opts.Scope != "crypto" {
14+
t.Fatalf("unexpected scope: %s", opts.Scope)
15+
}
16+
if opts.Window != "7d" {
17+
t.Fatalf("unexpected window: %s", opts.Window)
18+
}
19+
if opts.Format != "md" {
20+
t.Fatalf("unexpected format: %s", opts.Format)
21+
}
22+
}
23+
24+
func TestParseDigestOptionsRejectsUnknownFlag(t *testing.T) {
25+
t.Parallel()
26+
27+
_, err := parseDigestOptions([]string{"crypto", "--unknown"})
28+
if err == nil {
29+
t.Fatal("expected parseDigestOptions to return error for unknown flag")
30+
}
31+
}

0 commit comments

Comments
 (0)