diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 23aff0c..e038b2a 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -13,7 +13,7 @@ "@lucide/svelte": "^0.511.0", "@rollup/plugin-dsv": "^3.0.5", "@tanstack/svelte-query": "^6.0.18", - "allotaxonometer-ui": "^0.1.19", + "allotaxonometer-ui": "^0.2.1", "better-auth": "^1.3.27", "bits-ui": "^2.16.1", "d3": "^7.9.0", @@ -1958,9 +1958,9 @@ } }, "node_modules/allotaxonometer-ui": { - "version": "0.1.19", - "resolved": "https://registry.npmjs.org/allotaxonometer-ui/-/allotaxonometer-ui-0.1.19.tgz", - "integrity": "sha512-Vz1pqBDr69KKCIkV4+34raAp39QbbmCY0JWzQFbRzvEmgW/tdfG9Jh7TdPDCaR68CowNLGvYOMR6z9yrrDdtyg==", + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/allotaxonometer-ui/-/allotaxonometer-ui-0.2.1.tgz", + "integrity": "sha512-1qYyU0bouH74xiQF60xsIfIfYoJlWvZe/f7mkBbTBuf32s42XKwqQ6+sMpDtgdrOQyMzacBobaaDePW9gOUIjg==", "license": "MIT", "peerDependencies": { "@ungap/structured-clone": "1.3.0", diff --git a/frontend/package.json b/frontend/package.json index 4f23dc1..6893a5a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -32,7 +32,7 @@ "@lucide/svelte": "^0.511.0", "@rollup/plugin-dsv": "^3.0.5", "@tanstack/svelte-query": "^6.0.18", - "allotaxonometer-ui": "^0.1.19", + "allotaxonometer-ui": "^0.2.1", "better-auth": "^1.3.27", "bits-ui": "^2.16.1", "d3": "^7.9.0", diff --git a/frontend/src/lib/stories/allotaxonometry/allotax.remote.js b/frontend/src/lib/stories/allotaxonometry/allotax.remote.js index be447d0..dfee235 100644 --- a/frontend/src/lib/stories/allotaxonometry/allotax.remote.js +++ b/frontend/src/lib/stories/allotaxonometry/allotax.remote.js @@ -2,22 +2,18 @@ import { query } from "$app/server"; import * as v from "valibot" import { error } from '@sveltejs/kit'; -import { API_BASE } from '$env/static/private' +import { env } from '$env/dynamic/private' -const API_BASE_URL = API_BASE || 'http://localhost:8000' +const API_BASE_URL = env.STORYWRANGLER_API_BASE || 'http://localhost:8000' export const getAdapter = query(async () => { - const url = `${API_BASE_URL}/registry/babynames/babynames/adapter` - console.log('Fetching available locations:', url) - + const url = `${API_BASE_URL}/registry/babynames/ngrams/adapter` const response = await fetch(url) if (!response.ok) { const errorText = await response.text() - console.error('Error response:', errorText) - throw Error(`๐Ÿ’ฃ๏ธ Failed to fetch available locations: ${response.status} - ${errorText}`) + throw Error(`๐Ÿ’ฃ๏ธ Failed to fetch adapter: ${response.status} - ${errorText}`) } - return await response.json() } ); @@ -40,7 +36,7 @@ export const getTopBabyNames = query( limit: limit, }) - const url = `${API_BASE_URL}/babynames/top-ngrams?${params.toString()}` + const url = `${API_BASE_URL}/babynames/ngrams?${params.toString()}` const response = await fetch(url) if (!response.ok) { diff --git a/frontend/src/lib/stories/allotaxonometry/components/Index.svelte b/frontend/src/lib/stories/allotaxonometry/components/Index.svelte index e499ca0..841a345 100644 --- a/frontend/src/lib/stories/allotaxonometry/components/Index.svelte +++ b/frontend/src/lib/stories/allotaxonometry/components/Index.svelte @@ -1,9 +1,7 @@
@@ -240,29 +244,6 @@ />
-
- - - {#if showTopNWarning && me} - {@const count1 = me[0]?.ranks?.length || 0} - {@const count2 = me[1]?.ranks?.length || 0} -
- - - -
- {#if count1 < fetchedTopN && count2 < fetchedTopN} - System 1: {count1.toLocaleString()} names, System 2: {count2.toLocaleString()} names (requested {fetchedTopN.toLocaleString()}) - {:else if count1 < fetchedTopN} - System 1: Only {count1.toLocaleString()} names available (requested {fetchedTopN.toLocaleString()}) - {:else} - System 2: Only {count2.toLocaleString()} names available (requested {fetchedTopN.toLocaleString()}) - {/if} -
-
- {/if} -
-
@@ -270,6 +251,16 @@
{/if} +
+ + {#if allotax.dat} +
+ max: 1{(sys1?.[0]?.totalunique || sys1?.length || 0).toLocaleString()} + 2{(sys2?.[0]?.totalunique || sys2?.length || 0).toLocaleString()} +
+ {/if} +
+
@@ -302,23 +293,32 @@ {dateMax} onJump={updateData} /> + {/if} - {/if} - {#if me && rtd} - + {#if allotax.dat} + {/if}
- + @@ -326,15 +326,15 @@
- {#if browser && dat} + {#if browser && allotax.dat}
{#key `${committedPeriod1Start}-${committedPeriod1End}-${committedPeriod2Start}-${committedPeriod2End}-${committedLocation}-${committedSex}-${committedLimit}-${alpha}-${hasUploadedFiles}`} - import YearSlider from './sidebar/YearSlider.svelte'; - import AlphaSlider from './sidebar/AlphaSlider.svelte'; - import LocationSelector from './sidebar/LocationSelector.svelte'; - import SexToggle from './sidebar/SexToggle.svelte'; - import TopNSelector from './sidebar/TopNSelector.svelte'; - import MultiFileUpload from './sidebar/MultiFileUpload.svelte'; - import PeriodJumpControls from './sidebar/PeriodJumpControls.svelte'; - import DataInfo from './sidebar/DataInfo.svelte'; - import DownloadSection from './sidebar/DownloadSection.svelte'; - import { fileState, dashboardState, alphas } from '../sidebar-state.svelte.ts'; - - // Props passed from parent (Index.svelte owns query and instance) - let { query, instance } = $props(); - - // Derived values from props - const displayTitles = $derived(query.data?.title || ['System 1', 'System 2']); - const me = $derived(instance?.me); - const rtd = $derived(instance?.rtd); - const isDataReady = $derived(!!query.data && !!instance); - - // Check if we got fewer results than requested - const showTopNWarning = $derived( - isDataReady && - query.data && - ((query.data.elem1?.length || 0) < dashboardState.fetchedTopN || - (query.data.elem2?.length || 0) < dashboardState.fetchedTopN) - ); - - // Auto-dismiss warning after 5 seconds - $effect(() => { - if (showTopNWarning) { - dashboardState.warningDismissed = false; - const timer = setTimeout(() => { - dashboardState.warningDismissed = true; - }, 5000); - return () => clearTimeout(timer); - } - }); - - -
-
-

Allotaxonograph

-
- -
- fileState.handleFileUpload(file, system)} - uploadStatus={fileState.uploadStatus} - uploadWarnings={fileState.uploadWarnings} - /> - -
- - {#if !fileState.hasUploadedFiles} -
- -
- -
- - - {#if showTopNWarning && !dashboardState.warningDismissed && query.data} - {@const count1 = query.data.elem1?.length || 0} - {@const count2 = query.data.elem2?.length || 0} -
- - - -
- {#if count1 < dashboardState.fetchedTopN && count2 < dashboardState.fetchedTopN} - System 1: {count1.toLocaleString()} names, System 2: {count2.toLocaleString()} names (requested {dashboardState.fetchedTopN.toLocaleString()}) - {:else if count1 < dashboardState.fetchedTopN} - System 1: Only {count1.toLocaleString()} names available (requested {dashboardState.fetchedTopN.toLocaleString()}) - {:else} - System 2: Only {count2.toLocaleString()} names available (requested {dashboardState.fetchedTopN.toLocaleString()}) - {/if} -
-
- {/if} -
- -
- -
- -
- -
- -
- -
- -
- -
- -
- -
- - - - - {/if} - - {#if isDataReady} - - {/if} - -
- -
- -
-
-
- - diff --git a/frontend/src/lib/stories/allotaxonometry/components/sidebar/AlphaSlider.svelte b/frontend/src/lib/stories/allotaxonometry/components/sidebar/AlphaSlider.svelte deleted file mode 100644 index 148cdcc..0000000 --- a/frontend/src/lib/stories/allotaxonometry/components/sidebar/AlphaSlider.svelte +++ /dev/null @@ -1,246 +0,0 @@ - - -
-
-
- ฮฑ = {dashboardState.currentAlpha === Infinity ? 'โˆž' : dashboardState.currentAlpha} -
-
โ–ผ
-
- - dashboardState.alphaIndex = +e.target.value} - class="alpha-slider" - /> - -
- {#each alphas as alpha, i} -
-
- {#if shouldShowLabel(alpha)} -
- -
- {/if} -
- {/each} -
-
- - \ No newline at end of file diff --git a/frontend/src/lib/stories/allotaxonometry/components/sidebar/DataInfo.svelte b/frontend/src/lib/stories/allotaxonometry/components/sidebar/DataInfo.svelte index 9a15d36..df6000c 100644 --- a/frontend/src/lib/stories/allotaxonometry/components/sidebar/DataInfo.svelte +++ b/frontend/src/lib/stories/allotaxonometry/components/sidebar/DataInfo.svelte @@ -1,16 +1,16 @@
Items - {me[0].ranks.length.toLocaleString()} + {itemCount.toLocaleString()}
Divergence - {rtd.normalization.toFixed(4)} + {normalization.toFixed(4)}
diff --git a/frontend/src/lib/stories/allotaxonometry/components/sidebar/LocationSelector.svelte b/frontend/src/lib/stories/allotaxonometry/components/sidebar/LocationSelector.svelte index 608d985..be60de6 100644 --- a/frontend/src/lib/stories/allotaxonometry/components/sidebar/LocationSelector.svelte +++ b/frontend/src/lib/stories/allotaxonometry/components/sidebar/LocationSelector.svelte @@ -23,8 +23,8 @@ class="location-dropdown" > {#each adapter as row} - {/each} diff --git a/frontend/src/lib/stories/allotaxonometry/components/sidebar/MultiFileUpload.svelte b/frontend/src/lib/stories/allotaxonometry/components/sidebar/MultiFileUpload.svelte deleted file mode 100644 index 3c36c4f..0000000 --- a/frontend/src/lib/stories/allotaxonometry/components/sidebar/MultiFileUpload.svelte +++ /dev/null @@ -1,512 +0,0 @@ - - -
-
-
- -
fileInput.click()} - role="button" - tabindex="0" - aria-label="Upload files by dragging and dropping or clicking to browse" - > - e.target.files?.length && handleFiles(Array.from(e.target.files))} - class="file-input-hidden" - /> - -
-
๐Ÿ“
-

- Drag files or - -

-
-
- -

- Hint: when browsing files, hold Ctrl to select multiple files to compare. -

- - - {#if availableFiles.length > 0} -
- -
-
- 1 - System 1 -
-
- {#each availableFiles as file, index (file.id)} -
selectSys1File(index)} - onkeydown={(e) => handleOptionKeyDown(e, index, 'sys1')} - role="option" - tabindex="0" - aria-selected={selectedSys1Index === index} - > - {file.name}.{file.fileType} -
- {/each} -
-
- - -
-
- 2 - System 2 -
-
- {#each availableFiles as file, index (file.id)} -
selectSys2File(index)} - onkeydown={(e) => handleOptionKeyDown(e, index, 'sys2')} - role="option" - tabindex="0" - aria-selected={selectedSys2Index === index} - > - {file.name}.{file.fileType} -
- {/each} -
-
-
- - -
- - Clear Files - -
- {/if} - - - {#if uploadStatus} -
- {uploadStatus} -
- {/if} - - {#if uploadWarnings.length > 0} -
-
โš ๏ธ Warnings:
- {#each uploadWarnings as warning} -
{warning}
- {/each} -
- {/if} -
-
-
- - \ No newline at end of file diff --git a/frontend/src/lib/stories/allotaxonometry/components/sidebar/MultiFileUploadLocal.svelte b/frontend/src/lib/stories/allotaxonometry/components/sidebar/MultiFileUploadLocal.svelte index 4ece9ac..109694b 100644 --- a/frontend/src/lib/stories/allotaxonometry/components/sidebar/MultiFileUploadLocal.svelte +++ b/frontend/src/lib/stories/allotaxonometry/components/sidebar/MultiFileUploadLocal.svelte @@ -8,6 +8,7 @@ let fileInput; let dragOver = $state(false); + let showFormatHint = $state(false); let availableFiles = $state([]); let selectedSys1Index = $state(0); let selectedSys2Index = $state(1); @@ -57,8 +58,8 @@ try { const result = await parseDataFile(fileEntry.file, { enableTruncation: true, - maxRows: 50000, - warnThreshold: 50000, + maxRows: 1_000_000, + warnThreshold: 50_000, maxFileSize: 50 * 1024 * 1024 // 50MB }); @@ -243,9 +244,21 @@
-

- Hint: when browsing files, hold Ctrl to select multiple files to compare. -

+
+ CSV or JSON ยท hold Ctrl for multiple ยท + +
+ + {#if showFormatHint} +
+ CSV + 
{`types,counts\nEmma,4589\nLiam,3201\n...`}
+ JSON + 
{`[\n  {"types":"Emma","counts":4589},\n  {"types":"Liam","counts":3201}\n...\n]`}
+
+ {/if} {#if availableFiles.length > 0} @@ -345,16 +358,6 @@ margin-bottom: 0.5rem; } - .upload-header { - display: flex; - justify-content: space-between; - align-items: center; - padding: 0.5rem 0; - font-weight: 500; - color: var(--color-text-primary); - font-size: 0.9rem; - } - .upload-content { margin-top: 0.25rem; } @@ -366,15 +369,8 @@ margin-top: 0.5rem; } - .file-count { - background: var(--color-good-blue); - color: white; - padding: 0.125rem 0.375rem; - border-radius: 10px; - font-size: var(--11px); - font-weight: 600; - min-width: 1.25rem; - text-align: center; + .file-count-placeholder { + display: none; } /* Drop Zone */ @@ -436,11 +432,48 @@ .upload-hint { font-size: var(--11px, 0.69rem); color: #999; - margin: 0.5rem 0 0 0; + margin: 0.375rem 0 0 0; text-align: center; font-style: italic; } + .format-toggle { + background: none; + border: none; + padding: 0; + color: var(--color-good-blue, #3b82f6); + font-size: inherit; + font-style: inherit; + cursor: pointer; + text-decoration: underline; + } + + .format-examples { + display: flex; + flex-direction: column; + gap: 0.25rem; + } + + .fmt-label { + font-size: var(--10px, 0.625rem); + font-weight: 600; + color: var(--color-text-secondary); + text-transform: uppercase; + letter-spacing: 0.05em; + } + + .fmt-block { + margin: 0; + padding: 0.375rem 0.5rem; + background-color: var(--color-input-bg); + border: 1px solid var(--color-border); + border-radius: 4px; + font-family: var(--mono, monospace); + font-size: var(--10px, 0.625rem); + color: var(--color-text-primary); + line-height: 1.5; + } + /* Multi-Select Container */ .multiselect-container { display: flex; diff --git a/frontend/src/lib/stories/allotaxonometry/components/sidebar/PeriodJumpControls.svelte b/frontend/src/lib/stories/allotaxonometry/components/sidebar/PeriodJumpControls.svelte deleted file mode 100644 index 901eee2..0000000 --- a/frontend/src/lib/stories/allotaxonometry/components/sidebar/PeriodJumpControls.svelte +++ /dev/null @@ -1,112 +0,0 @@ - - -
-
- - โ€ข - -
- -
- - - years -
-
- - diff --git a/frontend/src/lib/stories/allotaxonometry/components/sidebar/TopNSelector.svelte b/frontend/src/lib/stories/allotaxonometry/components/sidebar/TopNSelector.svelte index 9898a1e..899a65d 100644 --- a/frontend/src/lib/stories/allotaxonometry/components/sidebar/TopNSelector.svelte +++ b/frontend/src/lib/stories/allotaxonometry/components/sidebar/TopNSelector.svelte @@ -2,16 +2,24 @@ let { limit = $bindable(10000), label = "Top N tokens", - min = 100, - max = 50000, - step = 100 + min = 0, + max = 10_000_000, + step = 1000 } = $props(); - function handleChange(event) { - const newValue = parseInt(event.target.value); + let inputValue = $state(String(limit)); + + // Sync display when limit changes externally + $effect(() => { + inputValue = String(limit); + }); + + function handleBlur() { + const newValue = parseInt(inputValue); if (!isNaN(newValue)) { limit = Math.max(min, Math.min(max, newValue)); } + inputValue = String(limit); } @@ -23,8 +31,8 @@
0) { - this.uploadWarnings = result.warnings; - } - - this.uploadStatus = `${system.toUpperCase()}: ${file.name} loaded successfully!`; - setTimeout(() => this.uploadStatus = '', 3000); - - // Only trigger data refresh when BOTH files are uploaded - setTimeout(() => { - if (this.uploadedSys1 && this.uploadedSys2) { - this.uploadStatus = 'Both files loaded! Generating visualization...'; - dashboardState.loadData(); - } else { - this.uploadStatus = `Waiting for ${this.uploadedSys1 ? 'System 2' : 'System 1'} file...`; - } - }, 0); - - return { success: true, fileName: file.name }; - } catch (error: unknown) { - const errorMessage = error instanceof Error ? error.message : 'Unknown error'; - this.uploadStatus = `Error loading ${file.name}: ${errorMessage}`; - setTimeout(() => this.uploadStatus = '', 5000); - return { success: false, error: errorMessage }; - } - } -} - -export const fileState = new FileState(); - -// ============================================================================= -// DASHBOARD CONTROLS STATE & LOGIC -// ============================================================================= - -// URL params - created at module level to avoid effect_orphan error -const urlParams = typeof window !== 'undefined' - ? new SvelteURLSearchParams(window.location.search) - : new SvelteURLSearchParams(); - -class DashboardState { - // Time periods - initialize from URL if present - period1 = $state( - urlParams.has('period1') - ? urlParams.get('period1').split(',').map(Number) - : [1940, 1959] - ); - period2 = $state( - urlParams.has('period2') - ? urlParams.get('period2').split(',').map(Number) - : [1990, 2009] - ); - jumpYears = $state(5); - - // Alpha parameter - alphaIndex = $state(7); - - // Data filtering - initialize from URL if present - selectedLocation = $state(urlParams.get('location') || 'wikidata:Q30'); - selectedSex = $state(urlParams.get('sex') || 'M'); - selectedTopN = $state(Number(urlParams.get('topN')) || 10000); - - // Location adapter data - adapter = $state([]); - locationsLoading = $state(true); - locationsError = $state(false); - - // UI state - sidebarCollapsed = $state(false); - warningDismissed = $state(false); - - // What's currently fetched/displayed - fetchedPeriod1 = $state( - urlParams.has('period1') - ? urlParams.get('period1').split(',').map(Number) - : [1940, 1959] - ); - fetchedPeriod2 = $state( - urlParams.has('period2') - ? urlParams.get('period2').split(',').map(Number) - : [1990, 2009] - ); - fetchedLocation = $state(urlParams.get('location') || 'wikidata:Q30'); - fetchedSex = $state(urlParams.get('sex') || 'M'); - fetchedTopN = $state(Number(urlParams.get('topN')) || 10000); - - // Derived current alpha value - currentAlpha = $derived(alphas[this.alphaIndex]); - - // Derived date range based on selected location - dateRange = $derived.by(() => { - if (!this.adapter?.length) return { min: 1880, max: 2020 }; - const location = this.adapter.find(l => l[1] === this.selectedLocation); - if (location && location[4] && location[5]) { - return { min: location[4], max: location[5] }; - } - return { min: 1880, max: 2020 }; - }); - - dateMin = $derived(this.dateRange.min); - dateMax = $derived(this.dateRange.max); - - // Sync current state to URL params - syncToUrl() { - urlParams.set('period1', `${this.period1[0]},${this.period1[1]}`); - urlParams.set('period2', `${this.period2[0]},${this.period2[1]}`); - urlParams.set('location', this.selectedLocation); - urlParams.set('sex', this.selectedSex); - urlParams.set('topN', String(this.selectedTopN)); - } - - // Adjust periods to fit within the valid date range for current location - adjustPeriodsToRange() { - const range = this.dateRange; - - // Adjust period1 if out of range - if (this.period1[0] < range.min || this.period1[1] > range.max) { - const periodLength = this.period1[1] - this.period1[0]; - const newStart = Math.max(range.min, Math.min(range.max - periodLength, this.period1[0])); - const newEnd = Math.min(range.max, newStart + periodLength); - this.period1 = [newStart, newEnd]; - } - - // Adjust period2 if out of range - if (this.period2[0] < range.min || this.period2[1] > range.max) { - const periodLength = this.period2[1] - this.period2[0]; - const newStart = Math.max(range.min, Math.min(range.max - periodLength, this.period2[0])); - const newEnd = Math.min(range.max, newStart + periodLength); - this.period2 = [newStart, newEnd]; - } - } - - async initializeAdapter() { - try { - this.adapter = await getAdapter(); - this.locationsLoading = false; - } catch (error) { - console.error('Failed to fetch locations:', error); - this.locationsError = true; - this.locationsLoading = false; - } - } - - loadData() { - // Adjust periods if location changed and they're out of range - this.adjustPeriodsToRange(); - - this.fetchedPeriod1 = [...this.period1]; - this.fetchedPeriod2 = [...this.period2]; - this.fetchedLocation = this.selectedLocation; - this.fetchedSex = this.selectedSex; - this.fetchedTopN = this.selectedTopN; - this.syncToUrl(); - } - - shiftBothPeriodsLeft() { - this.period1 = calculateShiftedRange( - this.period1, - -this.jumpYears, - this.dateMin, - this.dateMax - ); - - this.period2 = calculateShiftedRange( - this.period2, - -this.jumpYears, - this.dateMin, - this.dateMax - ); - - setTimeout(() => this.loadData(), 0); - } - - shiftBothPeriodsRight() { - this.period1 = calculateShiftedRange( - this.period1, - this.jumpYears, - this.dateMin, - this.dateMax - ); - - this.period2 = calculateShiftedRange( - this.period2, - this.jumpYears, - this.dateMin, - this.dateMax - ); - - setTimeout(() => this.loadData(), 0); - } - - canShiftLeft() { - return this.period1[0] > this.dateMin || this.period2[0] > this.dateMin; - } - - canShiftRight() { - return this.period1[1] < this.dateMax || this.period2[1] < this.dateMax; - } -} - -export const dashboardState = new DashboardState(); diff --git a/frontend/src/lib/stories/allotaxonometry/utils.ts b/frontend/src/lib/stories/allotaxonometry/utils.ts index 1d1d92e..fbbafe2 100644 --- a/frontend/src/lib/stories/allotaxonometry/utils.ts +++ b/frontend/src/lib/stories/allotaxonometry/utils.ts @@ -55,8 +55,8 @@ interface RecalculationResult { const DEFAULT_SETTINGS: Required = { enableTruncation: true, - maxRows: 50000, - warnThreshold: 5000, + maxRows: 10_000_000, + warnThreshold: 10_000_000, maxFileSize: 50 * 1024 * 1024 // 50MB }; @@ -107,22 +107,29 @@ function formatFileSize(bytes: number): string { return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; } -function validateDataStructure(data: unknown): data is DataItem[] { +function validateDataStructure(data: unknown): data is Array<{types: string; counts: number; [key: string]: unknown}> { if (!Array.isArray(data) || data.length === 0) return false; - - return data.every((item): item is DataItem => - item && + + return data.every((item) => + item && typeof item === 'object' && - typeof item.types === 'string' && - typeof item.counts === 'number' && - typeof item.probs === 'number' && - typeof item.totalunique === 'number' && - item.counts > 0 && - item.probs >= 0 && - item.probs <= 1 + typeof (item as Record).types === 'string' && + typeof (item as Record).counts === 'number' && + (item as Record).counts > 0 ); } +function normalizeData(data: Array<{types: string; counts: number; [key: string]: unknown}>): DataItem[] { + const totalCounts = data.reduce((sum, item) => sum + item.counts, 0); + const totalunique = data.length; + return data.map(item => ({ + types: item.types, + counts: item.counts, + probs: typeof item.probs === 'number' ? item.probs : item.counts / totalCounts, + totalunique: typeof item.totalunique === 'number' ? item.totalunique : totalunique + })); +} + // ============================================================================= // CORE PROCESSING FUNCTIONS // ============================================================================= @@ -254,18 +261,21 @@ function parseJSON(jsonData: unknown, settings: Required): { data if (!Array.isArray(jsonData)) { throw new Error('JSON data must be an array'); } - + if (jsonData.length === 0) { throw new Error('JSON array is empty'); } - - // Validate structure + + // Validate minimum structure (types + counts required) if (!validateDataStructure(jsonData)) { - throw new Error('Invalid JSON structure. Expected array of objects with types, counts, probs, and totalunique fields.'); + throw new Error('Invalid JSON structure. Expected array of objects with at least "types" (string) and "counts" (number > 0) fields.'); } - + + // Normalize: compute probs and totalunique if missing + const normalized = normalizeData(jsonData); + // Apply truncation if needed - return applyTruncation(jsonData, 0, settings); + return applyTruncation(normalized, 0, settings); } function applyTruncation(