Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions scripts/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ pnpm test:db:seed:token # against the token server
pnpm test:db:seed:basic # against the basic-auth server
```

The seed data includes:

- `articles` and `code_snippets` with small 8-dimensional fixture embeddings.
- `huggingface_384d` with HuggingFace-style 384-dimensional fixture embeddings
for testing inferred collection dimensions and custom embedding server URLs.

Stop with `Ctrl+C`.

## Auth credentials
Expand Down
40 changes: 33 additions & 7 deletions scripts/seed-test-db.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,11 @@ if (authMode === 'token') {

const client = new ChromaClient({ host: 'localhost', port, ssl: false, headers })

const EMBED_DIM = 8

function seededVector(seed) {
function seededVector(seed, dimension) {
// Deterministic pseudo-random vector so re-seeds produce stable embeddings.
const out = new Array(EMBED_DIM)
const out = new Array(dimension)
let s = seed
for (let i = 0; i < EMBED_DIM; i++) {
for (let i = 0; i < dimension; i++) {
s = (s * 9301 + 49297) % 233280
out[i] = s / 233280 - 0.5
}
Expand All @@ -43,6 +41,7 @@ function seededVector(seed) {
const collections = [
{
name: 'articles',
dimension: 8,
metadata: { description: 'Sample blog articles for explorer testing' },
docs: [
{ id: 'a1', document: 'Getting started with vector databases.', metadata: { topic: 'intro', words: 5 } },
Expand All @@ -54,6 +53,7 @@ const collections = [
},
{
name: 'code_snippets',
dimension: 8,
metadata: { description: 'Code snippets across languages' },
docs: [
{ id: 'c1', document: 'def hello():\n print("hi")', metadata: { language: 'python', lines: 2 } },
Expand All @@ -63,6 +63,32 @@ const collections = [
{ id: 'c5', document: 'puts "hello"', metadata: { language: 'ruby', lines: 1 } },
],
},
{
name: 'huggingface_384d',
dimension: 384,
metadata: {
description: 'HuggingFace-style 384-dimensional embeddings for URL and dimension testing',
embedding_provider: 'huggingface-server',
embedding_model: 'sentence-transformers/all-MiniLM-L6-v2',
},
docs: [
{
id: 'hf1',
document: 'A local HuggingFace embedding server can expose sentence-transformer vectors.',
metadata: { provider: 'huggingface-server', model: 'all-MiniLM-L6-v2', dimension: 384 },
},
{
id: 'hf2',
document: 'MiniLM embeddings are compact enough for fast document exploration.',
metadata: { provider: 'huggingface-server', model: 'all-MiniLM-L6-v2', dimension: 384 },
},
{
id: 'hf3',
document: 'This fixture helps verify inferred dimensions in the collection UI.',
metadata: { provider: 'huggingface-server', model: 'all-MiniLM-L6-v2', dimension: 384 },
},
],
},
]

async function main() {
Expand All @@ -76,9 +102,9 @@ async function main() {
const ids = spec.docs.map((d) => d.id)
const documents = spec.docs.map((d) => d.document)
const metadatas = spec.docs.map((d) => d.metadata)
const embeddings = spec.docs.map((_, idx) => seededVector(seedBase + idx + 1))
const embeddings = spec.docs.map((_, idx) => seededVector(seedBase + idx + 1, spec.dimension))
await col.upsert({ ids, documents, metadatas, embeddings })
console.log(`[seed] upserted ${ids.length} docs into "${spec.name}"`)
console.log(`[seed] upserted ${ids.length} docs into "${spec.name}" (${spec.dimension}d)`)
}

console.log('[seed] done')
Expand Down
98 changes: 66 additions & 32 deletions src/components/collections/CollectionConfigView.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,16 @@ import { ChevronDown, ChevronRight, Plus, X } from 'lucide-react'
import { useDraftCollection, DraftHNSWConfig } from '../../context/DraftCollectionContext'
import { useChromaDB } from '../../providers/ChromaDBProvider'
import { useCollection } from '../../context/CollectionContext'
import { EMBEDDING_FUNCTIONS, EMBEDDING_FUNCTION_GROUPS, getEmbeddingFunctionById } from '../../constants/embedding-functions'
import {
buildEmbeddingFunctionOverride,
EMBEDDING_FUNCTIONS,
EMBEDDING_FUNCTION_GROUPS,
embeddingFunctionUsesUrl,
getEmbeddingFunctionById,
getEmbeddingFunctionUrlLabel,
getEmbeddingFunctionUrlPlaceholder,
validateEmbeddingFunctionUrl,
} from '../../constants/embedding-functions'
import { MetadataValueType, validateMetadataValue } from '../../types/metadata'
import { CopyProgressDialog } from './CopyProgressDialog'

Expand All @@ -29,19 +38,19 @@ export function CollectionConfigView() {
})

const selectedEf = draftCollection ? getEmbeddingFunctionById(draftCollection.embeddingFunctionId) : EMBEDDING_FUNCTIONS[0]
const embeddingFunctionUrlError = draftCollection && selectedEf
? validateEmbeddingFunctionUrl(selectedEf, draftCollection.embeddingFunctionUrl)
: null
const dimensionsLabel = selectedEf?.dimensions ? `${selectedEf.dimensions}d` : 'Inferred from first embedding'

// Handle embedding function change - auto-populate dimension
const handleEfChange = useCallback(
(efId: string) => {
updateDraft({ embeddingFunctionId: efId, dimensionOverride: '' })
},
[updateDraft]
)

// Handle dimension override change
const handleDimensionChange = useCallback(
(value: string) => {
updateDraft({ dimensionOverride: value })
const nextEf = getEmbeddingFunctionById(efId)
updateDraft({
embeddingFunctionId: efId,
embeddingFunctionUrl: nextEf?.url || '',
})
},
[updateDraft]
)
Expand Down Expand Up @@ -70,13 +79,16 @@ export function CollectionConfigView() {
if (!draftCollection?.sourceCollection) return false

const source = draftCollection.sourceCollection
const sourceType = source.embeddingFunction?.name === 'openai' ? 'openai' : 'default'
const sourceType = source.embeddingFunction?.name ?? 'default'
const sourceModel = source.embeddingFunction?.config?.model_name as string | undefined
const sourceUrl = source.embeddingFunction?.config?.url as string | undefined

const draftEf = getEmbeddingFunctionById(draftCollection.embeddingFunctionId)
const draftUrl = draftCollection.embeddingFunctionUrl.trim() || draftEf?.url

if (draftEf?.type !== sourceType) return true
if (draftEf?.type === 'openai' && draftEf?.modelName !== sourceModel) return true
if (sourceModel && draftEf?.modelName !== sourceModel) return true
if (draftUrl !== sourceUrl) return true

return false
}, [draftCollection])
Expand All @@ -89,6 +101,11 @@ export function CollectionConfigView() {
if (!draftCollection.name.trim()) {
errors.name = 'Collection name is required'
}
const draftEf = getEmbeddingFunctionById(draftCollection.embeddingFunctionId)
const urlError = validateEmbeddingFunctionUrl(draftEf, draftCollection.embeddingFunctionUrl)
if (urlError) {
errors.embeddingFunctionUrl = urlError
}

if (Object.keys(errors).length > 0) {
return
Expand Down Expand Up @@ -132,10 +149,9 @@ export function CollectionConfigView() {
const result = await window.electronAPI.chromadb.copyCollection(currentProfile.id, {
sourceCollectionName: draftCollection.sourceCollection.name,
targetName: draftCollection.name.trim(),
embeddingFunction: draftEf ? {
type: draftEf.type,
modelName: draftEf.modelName,
} : undefined,
embeddingFunction: draftEf
? buildEmbeddingFunctionOverride(draftEf, { url: draftCollection.embeddingFunctionUrl })
: undefined,
hnsw: Object.keys(hnswConfig).length > 0 ? hnswConfig : undefined,
regenerateEmbeddings: needsEmbeddingRegeneration(),
})
Expand Down Expand Up @@ -283,22 +299,40 @@ export function CollectionConfigView() {
{selectedEf && <p className="text-[10px] text-muted-foreground">{selectedEf.modelName}</p>}
</div>

{/* Dimension Input */}
{embeddingFunctionUsesUrl(selectedEf) && (
<div className="space-y-1">
<label htmlFor="ef-url" className="text-[11px] font-medium text-muted-foreground">
{getEmbeddingFunctionUrlLabel(selectedEf)}
</label>
<input
id="ef-url"
type="url"
value={draftCollection.embeddingFunctionUrl}
onChange={(e) => updateDraft({ embeddingFunctionUrl: e.target.value })}
placeholder={getEmbeddingFunctionUrlPlaceholder(selectedEf)}
className={`${inputClassName} ${embeddingFunctionUrlError ? 'border-destructive' : ''}`}
style={inputStyle}
/>
{embeddingFunctionUrlError ? (
<p className="text-[10px] text-destructive">{embeddingFunctionUrlError}</p>
) : (
<p className="text-[10px] text-muted-foreground">
Used by the client when generating embeddings.
</p>
)}
</div>
)}

{/* Dimension Info */}
<div className="space-y-1">
<label htmlFor="dimension" className="text-[11px] font-medium text-muted-foreground">
Dimension
</label>
<input
id="dimension"
type="number"
value={draftCollection.dimensionOverride || ''}
onChange={(e) => handleDimensionChange(e.target.value)}
placeholder={selectedEf?.dimensions?.toString() || 'auto'}
className={inputClassName}
style={inputStyle}
/>
<span className="text-[11px] font-medium text-muted-foreground">
Dimensions
</span>
<p className="h-6 flex items-center text-[11px] px-1.5 rounded-md bg-muted/40 text-foreground/70">
{dimensionsLabel}
</p>
<p className="text-[10px] text-muted-foreground">
Default: {selectedEf?.dimensions ?? 'auto'}
Chroma sets the collection dimension from generated embeddings.
</p>
</div>

Expand Down Expand Up @@ -586,7 +620,7 @@ export function CollectionConfigView() {
<button
type="button"
onClick={handleCopyCollection}
disabled={isCopying || !draftCollection.name.trim()}
disabled={isCopying || !draftCollection.name.trim() || Boolean(embeddingFunctionUrlError)}
className="h-6 px-2 text-[11px] rounded-md bg-[#007AFF] hover:bg-[#0071E3] active:bg-[#006DD9] text-white disabled:opacity-50 disabled:cursor-not-allowed"
>
{isCopying ? 'Copying…' : 'Copy Collection'}
Expand All @@ -595,7 +629,7 @@ export function CollectionConfigView() {
<button
type="button"
onClick={saveDraft}
disabled={isCreating || !draftCollection.name.trim()}
disabled={isCreating || !draftCollection.name.trim() || Boolean(embeddingFunctionUrlError)}
className="h-6 px-2 text-[11px] rounded-md bg-[#007AFF] hover:bg-[#0071E3] active:bg-[#006DD9] text-white disabled:opacity-50 disabled:cursor-not-allowed"
>
{isCreating ? 'Creating…' : 'Create'}
Expand Down
61 changes: 50 additions & 11 deletions src/components/documents/EmbeddingFunctionSelector.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,16 @@ import { ChevronDown } from 'lucide-react'
import { Popover, PopoverContent, PopoverTrigger } from '../ui/popover'
import { Button } from '../ui/button'
import { Label } from '../ui/label'
import { EMBEDDING_FUNCTIONS, EMBEDDING_FUNCTION_GROUPS } from '../../constants/embedding-functions'
import {
buildEmbeddingFunctionOverride,
EMBEDDING_FUNCTIONS,
EMBEDDING_FUNCTION_GROUPS,
embeddingFunctionUsesUrl,
getEmbeddingFunctionById,
getEmbeddingFunctionUrlLabel,
getEmbeddingFunctionUrlPlaceholder,
validateEmbeddingFunctionUrl,
} from '../../constants/embedding-functions'

interface EmbeddingFunctionSelectorProps {
collectionName: string
Expand Down Expand Up @@ -83,30 +92,36 @@ export function EmbeddingFunctionSelector({
}: EmbeddingFunctionSelectorProps) {
const [open, setOpen] = useState(false)
const [selectedId, setSelectedId] = useState(() => getSelectedEmbeddingId(currentOverride))
const [customUrl, setCustomUrl] = useState(currentOverride?.url || '')
const [saving, setSaving] = useState(false)

const handleOpenChange = (nextOpen: boolean) => {
if (nextOpen) {
setSelectedId(getSelectedEmbeddingId(currentOverride))
const nextSelectedId = getSelectedEmbeddingId(currentOverride)
const nextSelectedEF = getEmbeddingFunctionById(nextSelectedId)
setSelectedId(nextSelectedId)
setCustomUrl(currentOverride?.url || nextSelectedEF?.url || '')
}
setOpen(nextOpen)
}

const selectedEF = EMBEDDING_FUNCTIONS.find(ef => ef.id === selectedId)
const urlError = validateEmbeddingFunctionUrl(selectedEF, customUrl)

const serverFunction = EMBEDDING_FUNCTIONS.find(ef => ef.modelName === serverConfig?.config?.model_name)

const handleSelectedIdChange = (nextSelectedId: string) => {
const nextSelectedEF = getEmbeddingFunctionById(nextSelectedId)
setSelectedId(nextSelectedId)
setCustomUrl(nextSelectedEF?.url || '')
}

const handleSave = async () => {
if (!selectedEF) return
if (!selectedEF || urlError) return

setSaving(true)
try {
await onSave({
type: selectedEF.type,
modelName: selectedEF.modelName,
url: selectedEF.url,
accountId: selectedEF.accountId,
})
await onSave(buildEmbeddingFunctionOverride(selectedEF, { url: customUrl }))
setOpen(false)
} finally {
setSaving(false)
Expand Down Expand Up @@ -166,7 +181,7 @@ export function EmbeddingFunctionSelector({
<select
id="ef-select"
value={selectedId}
onChange={(e) => setSelectedId(e.target.value)}
onChange={(e) => handleSelectedIdChange(e.target.value)}
className="w-full h-[22px] appearance-none rounded-[5px] border-none bg-white/10 dark:bg-white/5 pl-2 pr-6 text-[13px] text-foreground shadow-[0_0.5px_1px_rgba(0,0,0,0.1),inset_0_0.5px_0.5px_rgba(255,255,255,0.1)] ring-1 ring-black/10 dark:ring-white/10 focus:outline-none focus:ring-2 focus:ring-primary/50 cursor-default"
>
<option value="">Select…</option>
Expand All @@ -184,6 +199,27 @@ export function EmbeddingFunctionSelector({
</div>
</div>

{embeddingFunctionUsesUrl(selectedEF) && (
<div className="px-1 space-y-1.5">
<Label htmlFor="ef-url" className="text-[11px] font-normal">
{getEmbeddingFunctionUrlLabel(selectedEF)}
</Label>
<input
id="ef-url"
type="url"
value={customUrl}
onChange={(e) => setCustomUrl(e.target.value)}
placeholder={getEmbeddingFunctionUrlPlaceholder(selectedEF)}
className={`w-full h-[22px] rounded-[5px] border-none bg-white/10 dark:bg-white/5 px-2 text-[12px] text-foreground placeholder:text-muted-foreground/60 shadow-[0_0.5px_1px_rgba(0,0,0,0.1),inset_0_0.5px_0.5px_rgba(255,255,255,0.1)] ring-1 focus:outline-none focus:ring-2 ${
urlError
? 'ring-destructive/50 focus:ring-destructive/50'
: 'ring-black/10 dark:ring-white/10 focus:ring-primary/50'
}`}
/>
{urlError && <p className="text-[10px] text-destructive">{urlError}</p>}
</div>
)}

{/* Selected info */}
{(() => {
const displayEF = selectedEF || serverFunction || EMBEDDING_FUNCTIONS.find(ef => ef.id === 'default')!
Expand Down Expand Up @@ -218,6 +254,9 @@ export function EmbeddingFunctionSelector({
<p className="text-muted-foreground"><span className="text-foreground/60">Type:</span> {displayEF.type}</p>
<p className="text-muted-foreground"><span className="text-foreground/60">Model:</span> {displayEF.modelName}</p>
<p className="text-muted-foreground"><span className="text-foreground/60">Dimensions:</span> {displayEF.dimensions ?? 'variable'}</p>
{embeddingFunctionUsesUrl(selectedEF) && customUrl.trim() && !urlError && (
<p className="text-muted-foreground break-all"><span className="text-foreground/60">URL:</span> {customUrl.trim()}</p>
)}
{embeddingDimension && (
<p className="text-muted-foreground pt-1 border-t border-black/5 dark:border-white/5 mt-1">
<span className="text-foreground/60">Collection:</span> {embeddingDimension}d
Expand Down Expand Up @@ -245,7 +284,7 @@ export function EmbeddingFunctionSelector({
size="sm"
className="h-7 text-[12px] px-3"
onClick={handleSave}
disabled={!selectedEF || saving}
disabled={!selectedEF || saving || Boolean(urlError)}
>
{saving ? 'Saving…' : 'Apply'}
</Button>
Expand Down
Loading
Loading