Skip to content

Commit 0bc367f

Browse files
committed
Merge branch 'whoisthey/text-file-attachments' into whoisthey/binary-file-attachments
2 parents 9acf95a + a7cdc9c commit 0bc367f

30 files changed

Lines changed: 1470 additions & 53 deletions

File tree

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
name: Check Prisma Migrations
2+
description: >-
3+
Verify Prisma migrations apply cleanly in order and reproduce schema.prisma
4+
(drift check), and that no new migration predates the latest on the base
5+
branch (ordering check). Designed to be embedded in an existing job so its
6+
failure turns that job's status red.
7+
8+
inputs:
9+
base-ref:
10+
description: >-
11+
Base git ref to diff migrations against (e.g. a PR's base branch). When
12+
set, the action skips work on PRs that don't touch migrations and runs the
13+
ordering check. When empty (release builds), the drift check always runs
14+
and the ordering check is skipped.
15+
required: false
16+
default: ""
17+
18+
runs:
19+
using: composite
20+
steps:
21+
- name: Detect Prisma changes
22+
id: detect
23+
shell: bash
24+
run: |
25+
if [ -z "${{ inputs.base-ref }}" ]; then
26+
echo "changed=true" >> "$GITHUB_OUTPUT"
27+
echo "No base-ref provided — running drift check unconditionally."
28+
exit 0
29+
fi
30+
git fetch --no-tags --depth=1 origin "+refs/heads/${{ inputs.base-ref }}:refs/remotes/origin/${{ inputs.base-ref }}"
31+
if git diff --name-only "origin/${{ inputs.base-ref }}" HEAD | grep -q '^packages/db/prisma/'; then
32+
echo "changed=true" >> "$GITHUB_OUTPUT"
33+
echo "Prisma changes detected — running migration checks."
34+
else
35+
echo "changed=false" >> "$GITHUB_OUTPUT"
36+
echo "No Prisma changes — skipping migration checks."
37+
fi
38+
39+
- name: Start Postgres
40+
if: steps.detect.outputs.changed == 'true'
41+
shell: bash
42+
run: |
43+
docker run -d --name prisma-check-pg \
44+
-e POSTGRES_USER=postgres \
45+
-e POSTGRES_PASSWORD=postgres \
46+
-e POSTGRES_DB=sourcebot \
47+
-p 5432:5432 postgres:16
48+
for i in $(seq 1 30); do
49+
if docker exec prisma-check-pg pg_isready -U postgres -q; then
50+
echo "Postgres ready."
51+
exit 0
52+
fi
53+
sleep 2
54+
done
55+
echo "Postgres failed to become ready." && exit 1
56+
57+
- name: Use Node.js
58+
if: steps.detect.outputs.changed == 'true'
59+
uses: actions/setup-node@v4
60+
with:
61+
node-version: "20.x"
62+
63+
- name: Install
64+
if: steps.detect.outputs.changed == 'true'
65+
shell: bash
66+
run: yarn install --frozen-lockfile
67+
68+
# Check 1: migrations apply cleanly in order AND reproduce schema.prisma.
69+
# `migrate deploy` fails if a migration is broken or applies out of sequence;
70+
# `migrate diff` exits 2 when the applied history drifts from the schema.
71+
- name: Apply migrations
72+
if: steps.detect.outputs.changed == 'true'
73+
shell: bash
74+
working-directory: packages/db
75+
env:
76+
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/sourcebot
77+
run: yarn prisma migrate deploy
78+
79+
- name: Check for schema drift
80+
if: steps.detect.outputs.changed == 'true'
81+
shell: bash
82+
working-directory: packages/db
83+
env:
84+
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/sourcebot
85+
run: |
86+
yarn prisma migrate diff \
87+
--from-url "$DATABASE_URL" \
88+
--to-schema-datamodel prisma/schema.prisma \
89+
--exit-code \
90+
&& echo "✅ No drift: migrations reproduce schema.prisma" \
91+
|| (echo "❌ schema.prisma has changes not captured in a migration. Run: yarn dev:prisma:migrate:dev --name <name>" && exit 1)
92+
93+
# Check 2 (PRs only): no new migration predates the latest on the base branch.
94+
- name: Check migration ordering
95+
if: steps.detect.outputs.changed == 'true' && inputs.base-ref != ''
96+
shell: bash
97+
run: |
98+
MIG_DIR=packages/db/prisma/migrations
99+
BASE="origin/${{ inputs.base-ref }}"
100+
LATEST_ON_BASE=$(git ls-tree -r --name-only "$BASE" -- "$MIG_DIR" \
101+
| sed -n "s#$MIG_DIR/\([0-9]\{14\}\)_.*#\1#p" | sort | tail -1)
102+
echo "Latest migration on ${{ inputs.base-ref }}: ${LATEST_ON_BASE:-<none>}"
103+
NEW=$(comm -23 \
104+
<(ls "$MIG_DIR" | sed -n 's/^\([0-9]\{14\}\)_.*/\1/p' | sort -u) \
105+
<(git ls-tree -r --name-only "$BASE" -- "$MIG_DIR" | sed -n "s#$MIG_DIR/\([0-9]\{14\}\)_.*#\1#p" | sort -u))
106+
FAIL=0
107+
for ts in $NEW; do
108+
if [ -n "$LATEST_ON_BASE" ] && [ "$ts" -lt "$LATEST_ON_BASE" ]; then
109+
echo "❌ New migration $ts predates latest migration on ${{ inputs.base-ref }} ($LATEST_ON_BASE). Rename it with a current timestamp."
110+
FAIL=1
111+
fi
112+
done
113+
[ "$FAIL" -eq 0 ] && echo "✅ Migration ordering OK"
114+
exit $FAIL

.github/workflows/_build.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,14 @@ jobs:
7676
fetch-depth: 0
7777
token: ${{ inputs.use_app_token && steps.generate_token.outputs.token || github.token }}
7878

79+
# Release backstop: fail the build if migrations drift from schema.prisma.
80+
# Runs once (amd64 only) since the check is platform-independent. base-ref
81+
# is omitted, so the drift check always runs and the (PR-only) ordering
82+
# check is skipped.
83+
- name: Check Prisma migrations
84+
if: matrix.platform == 'linux/amd64'
85+
uses: ./.github/actions/check-prisma-migrations
86+
7987
# Extract metadata (tags, labels) for Docker
8088
# https://github.com/docker/metadata-action
8189
- name: Extract Docker metadata
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: Generate OpenAPI Spec
2+
3+
on:
4+
pull_request:
5+
branches: ["main"]
6+
paths:
7+
- "packages/web/**"
8+
- "packages/shared/src/version.ts"
9+
10+
jobs:
11+
generate-openapi:
12+
runs-on: ubuntu-latest
13+
# Skip forks: the default GITHUB_TOKEN can't push back to a fork's branch.
14+
if: github.event.pull_request.head.repo.full_name == github.repository
15+
permissions:
16+
contents: write
17+
steps:
18+
- name: Checkout repository
19+
uses: actions/checkout@v4
20+
with:
21+
submodules: "true"
22+
ref: ${{ github.head_ref }}
23+
token: ${{ secrets.GITHUB_TOKEN }}
24+
25+
- name: Use Node.js
26+
uses: actions/setup-node@v4
27+
with:
28+
node-version: '20.x'
29+
cache: 'yarn'
30+
cache-dependency-path: '**/yarn.lock'
31+
32+
- name: Install
33+
run: yarn install --frozen-lockfile
34+
35+
- name: Generate OpenAPI spec
36+
run: yarn workspace @sourcebot/web openapi:generate
37+
38+
- name: Commit regenerated spec if changed
39+
run: |
40+
SPEC=docs/api-reference/sourcebot-public.openapi.json
41+
if [ -z "$(git status --porcelain "$SPEC")" ]; then
42+
echo "OpenAPI spec is up to date."
43+
exit 0
44+
fi
45+
git config user.name "github-actions[bot]"
46+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
47+
git add "$SPEC"
48+
git commit -m "chore: regenerate OpenAPI spec"
49+
git push

.github/workflows/pr-gate.yml

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
name: PR Gate
22

3-
# This gate simply validates that we can build the docker container.
3+
# This gate validates that Prisma migrations are in order and that we can build
4+
# the docker container.
45

56
on:
67
pull_request:
@@ -16,6 +17,15 @@ jobs:
1617
uses: actions/checkout@v4
1718
with:
1819
submodules: "true"
20+
# full history so migration checks can diff against the base branch
21+
fetch-depth: 0
22+
23+
# Fails fast (before the docker build) when migrations drift from
24+
# schema.prisma or a new migration is added out of timestamp order.
25+
- name: Check Prisma migrations
26+
uses: ./.github/actions/check-prisma-migrations
27+
with:
28+
base-ref: ${{ github.base_ref }}
1929

2030
- name: Build Docker image
2131
id: build

CHANGELOG.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1616
- [EE] Added mermaid diagram rendering to Ask Sourcebot answers, with pan/zoom, copy/export, in-thread deep links, and an interleaved right-panel view. [#1369](https://github.com/sourcebot-dev/sourcebot/pull/1369)
1717
- [EE] Added a context-window usage gauge to the Ask Sourcebot chat details, showing how much of the selected model's context window each turn occupies. Window sizes are resolved from the models.dev catalog. [#1370](https://github.com/sourcebot-dev/sourcebot/pull/1370)
1818
- Added language model input-modality and document capability resolution, automatically resolved from the models.dev catalog (falls back to text-only for uncatalogued/self-hosted models). [#1372](https://github.com/sourcebot-dev/sourcebot/pull/1372)
19+
- [EE] Added DPoP sender-constrained OAuth tokens for MCP clients. [#1395](https://github.com/sourcebot-dev/sourcebot/pull/1395)
1920
- [EE] Added text file attachments to Ask Sourcebot, letting users attach text/code/config files to a chat message via the paperclip button, drag-and-drop, or paste, with large pastes auto-converted to attachments. [#1374](https://github.com/sourcebot-dev/sourcebot/pull/1374)
2021

2122
### Fixed
2223
- Send anonymous server-side PostHog events as personless so unauthenticated requests don't inflate person counts. [#1367](https://github.com/sourcebot-dev/sourcebot/pull/1367)
2324
- [EE] Fixed Ask Sourcebot mermaid diagrams overflowing their container by contain-fitting them to both width and height, and made revealing a diagram from the answer jump it into view instantly to avoid over/undershooting. [#1373](https://github.com/sourcebot-dev/sourcebot/pull/1373)
25+
- Verified GitHub review webhook deliveries before processing them. [#1378](https://github.com/sourcebot-dev/sourcebot/pull/1378)
26+
- Passed Zoekt index parameters via argv to preserve revision names with punctuation. [#1376](https://github.com/sourcebot-dev/sourcebot/pull/1376)
2427

2528
## [5.0.4] - 2026-06-18
2629

docs/docs.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
"docs/overview",
4242
{
4343
"group": "Deploy Sourcebot",
44+
"root": "docs/deployment/deploy-sourcebot",
4445
"pages": [
4546
"docs/deployment/docker-compose",
4647
"docs/deployment/k8s"
@@ -250,6 +251,10 @@
250251
"strict": false
251252
},
252253
"redirects": [
254+
{
255+
"source": "/docs/deployment/overview",
256+
"destination": "/docs/deployment/deploy-sourcebot"
257+
},
253258
{
254259
"source": "/docs/features/search/overview",
255260
"destination": "/docs/features/search/code-search"
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
---
2+
title: "Deploy Sourcebot"
3+
---
4+
5+
Sourcebot runs entirely in your infrastructure. Your code, AI inputs/outputs, personal data, and all other sensitive data never leave your environment.
6+
7+
<CardGroup cols={2}>
8+
<Card title="Docker Compose" icon="docker" href="/docs/deployment/docker-compose">
9+
The fastest way to get started. Deploy Sourcebot with a single command.
10+
</Card>
11+
<Card title="Kubernetes (Helm)" icon="dharmachakra" href="/docs/deployment/k8s">
12+
Deploy Sourcebot into your Kubernetes cluster using the official Helm chart.
13+
</Card>
14+
</CardGroup>
15+
16+
Not sure how much to provision? See the [sizing guide](/docs/deployment/sizing-guide) for resource recommendations.

packages/backend/src/zoekt.test.ts

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import type { Repo } from "@sourcebot/db";
2+
import { execFile } from "child_process";
3+
import { beforeEach, describe, expect, test, vi } from "vitest";
4+
import type { Settings } from "./types.js";
5+
import { indexGitRepository } from "./zoekt.js";
6+
7+
vi.mock("child_process", () => ({
8+
execFile: vi.fn((_file, _args, _options, callback) => {
9+
callback(null, "", "");
10+
}),
11+
}));
12+
13+
vi.mock("@sourcebot/shared", () => ({
14+
createLogger: vi.fn(() => ({
15+
debug: vi.fn(),
16+
warn: vi.fn(),
17+
})),
18+
env: {},
19+
getRepoPath: vi.fn(() => ({
20+
path: "/tmp/repo.git",
21+
})),
22+
}));
23+
24+
vi.mock("./constants.js", () => ({
25+
INDEX_CACHE_DIR: "/tmp/index",
26+
}));
27+
28+
vi.mock("./utils.js", () => ({
29+
getShardPrefix: vi.fn(() => "1_2"),
30+
}));
31+
32+
describe("indexGitRepository", () => {
33+
beforeEach(() => {
34+
vi.clearAllMocks();
35+
});
36+
37+
test("preserves revision names with punctuation", async () => {
38+
const repo = {
39+
id: 2,
40+
orgId: 1,
41+
} as Repo;
42+
const settings = {
43+
maxTrigramCount: 100,
44+
maxFileSize: 200,
45+
} as Settings;
46+
const payloadBranch = 'refs/heads/release";metadata${IFS}v1;"x';
47+
48+
await indexGitRepository(repo, settings, [
49+
"refs/heads/main",
50+
payloadBranch,
51+
]);
52+
53+
expect(execFile).toHaveBeenCalledWith(
54+
"zoekt-git-index",
55+
[
56+
"-allow_missing_branches",
57+
"-index", "/tmp/index",
58+
"-max_trigram_count", "100",
59+
"-file_limit", "200",
60+
"-branches", `refs/heads/main,${payloadBranch}`,
61+
"-tenant_id", "1",
62+
"-repo_id", "2",
63+
"-shard_prefix_override", "1_2",
64+
"/tmp/repo.git",
65+
],
66+
{},
67+
expect.any(Function),
68+
);
69+
});
70+
});

packages/backend/src/zoekt.ts

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { Repo } from "@sourcebot/db";
22
import { createLogger, env, getRepoPath } from "@sourcebot/shared";
3-
import { exec } from "child_process";
3+
import { execFile } from "child_process";
44
import { readdir, rm } from "fs/promises";
55
import { INDEX_CACHE_DIR } from "./constants.js";
66
import { Settings } from "./types.js";
@@ -14,22 +14,21 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, revisio
1414

1515
const largeFileGlobPatterns = env.ALWAYS_INDEX_FILE_PATTERNS?.split(',').map(pattern => pattern.trim()) ?? [];
1616

17-
const command = [
18-
'zoekt-git-index',
17+
const args = [
1918
'-allow_missing_branches',
20-
`-index ${INDEX_CACHE_DIR}`,
21-
`-max_trigram_count ${settings.maxTrigramCount}`,
22-
`-file_limit ${settings.maxFileSize}`,
23-
`-branches "${revisions.join(',')}"`,
24-
`-tenant_id ${repo.orgId}`,
25-
`-repo_id ${repo.id}`,
26-
`-shard_prefix_override ${shardPrefix}`,
27-
...largeFileGlobPatterns.map((pattern) => `-large_file "${pattern}"`),
19+
'-index', INDEX_CACHE_DIR,
20+
'-max_trigram_count', settings.maxTrigramCount.toString(),
21+
'-file_limit', settings.maxFileSize.toString(),
22+
'-branches', revisions.join(','),
23+
'-tenant_id', repo.orgId.toString(),
24+
'-repo_id', repo.id.toString(),
25+
'-shard_prefix_override', shardPrefix,
26+
...largeFileGlobPatterns.flatMap((pattern) => ['-large_file', pattern]),
2827
repoPath
29-
].join(' ');
28+
];
3029

3130
return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => {
32-
exec(command, { signal }, (error, stdout, stderr) => {
31+
execFile('zoekt-git-index', args, { signal }, (error, stdout, stderr) => {
3332
if (error) {
3433
reject(error);
3534
return;
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
ALTER TABLE "OAuthAuthorizationCode" ADD COLUMN "dpopJkt" TEXT;
2+
3+
ALTER TABLE "OAuthRefreshToken" ADD COLUMN "dpopJkt" TEXT;
4+
5+
ALTER TABLE "OAuthToken" ADD COLUMN "dpopJkt" TEXT;

0 commit comments

Comments
 (0)