diff --git a/packages/core/package.json b/packages/core/package.json index b4ac9b30..8f8b5527 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@alleninstitute/vis-core", - "version": "0.0.6", + "version": "0.0.7", "contributors": [ { "name": "Lane Sawyer", diff --git a/packages/core/src/shared-priority-cache/shared-cache.ts b/packages/core/src/shared-priority-cache/shared-cache.ts index 0df0f32f..46d4d33d 100644 --- a/packages/core/src/shared-priority-cache/shared-cache.ts +++ b/packages/core/src/shared-priority-cache/shared-cache.ts @@ -19,7 +19,7 @@ type CacheInterface> = { }; export type ClientSpec> = { - isValue: (v: Record) => v is ItemContent; + isValue: (v: Record, item: Item) => v is ItemContent; cacheKeys: (item: Item) => { [k in keyof ItemContent]: string }; onDataArrived?: (cacheKey: string, result: FetchResult) => void; fetch: (item: Item) => { [k in keyof ItemContent]: (abort: AbortSignal) => Promise }; @@ -109,7 +109,7 @@ export class SharedPriorityCache { get: (k: Item) => { const keys = spec.cacheKeys(k); const v = mapFields, Cacheable | undefined>(keys, (k) => this.cache.get(k)); - return spec.isValue(v) ? v : undefined; + return spec.isValue(v, k) ? v : undefined; }, has: (k: Item) => { const atLeastOneMissing = Object.values(spec.cacheKeys(k)).some((ck) => !this.cache.has(ck)); diff --git a/packages/scatterbrain/package.json b/packages/scatterbrain/package.json index 28b5e4a5..dfa0abb1 100644 --- a/packages/scatterbrain/package.json +++ b/packages/scatterbrain/package.json @@ -1,6 +1,6 @@ { "name": "@alleninstitute/vis-scatterbrain", - "version": "0.0.2", + "version": "0.0.3", "contributors": [ { "name": "Lane Sawyer", diff --git a/packages/scatterbrain/src/cache-client.ts b/packages/scatterbrain/src/cache-client.ts new file mode 100644 index 00000000..dddadbce --- /dev/null +++ b/packages/scatterbrain/src/cache-client.ts @@ -0,0 +1,64 @@ +import type { Cacheable, SharedPriorityCache } from '@alleninstitute/vis-core'; +import reduce from 'lodash/reduce'; +import type { WebGLSafeBasicType } from './typed-array'; +import type { ColumnRequest, Item } from './types'; + +type Content = Record; + +export function buildScatterbrainCacheClient( + cache: SharedPriorityCache, + toCacheValue: (buffer: ArrayBuffer, type: WebGLSafeBasicType) => V, + onDataArrived: () => void, +) { + const client = cache.registerClient>({ + cacheKeys: (item) => { + const { dataset, node, columns } = item; + return reduce, Record>( + columns, + (acc, col, key) => ({ + ...acc, + [key]: `${dataset.metadata.metadataFileEndpoint}/${node.file}/${col.name}`, + }), + {}, + ); + }, + fetch: (item) => { + const { dataset, node, columns } = item; + const attrs = dataset.metadata.pointAttributes; + const getColumnUrl = (columnName: string) => + `${dataset.metadata.metadataFileEndpoint}${columnName}/${dataset.metadata.visualizationReferenceId}/${node.file}`; + const getGeneUrl = (columnName: string) => + `${dataset.metadata.geneFileEndpoint}${columnName}/${dataset.metadata.visualizationReferenceId}/${node.file}`; + const getColumnInfo = (col: ColumnRequest) => + col.type === 'QUANTITATIVE' + ? ({ url: getGeneUrl(col.name), elements: 1, type: 'float' } as const) + : { url: getColumnUrl(col.name), elements: attrs[col.name].elements, type: attrs[col.name].type }; + + const proms = reduce, Record Promise>>( + columns, + (getters, col, key) => { + const { url, type } = getColumnInfo(col); + return { + ...getters, + [key]: (signal) => + fetch(url, { signal }).then((b) => + b.arrayBuffer().then((buff) => toCacheValue(buff, type)), + ), + }; + }, + {}, + ); + return proms; + }, + isValue: (v, item: Item): v is Content => { + for (const column of Object.keys(item.columns)) { + if (!(column in v)) { + return false; + } + } + return true; + }, + onDataArrived, + }); + return client; +} diff --git a/packages/scatterbrain/src/index.ts b/packages/scatterbrain/src/index.ts index bcc39ea3..ebdd683f 100644 --- a/packages/scatterbrain/src/index.ts +++ b/packages/scatterbrain/src/index.ts @@ -1,8 +1,8 @@ +export { buildScatterbrainCacheClient } from './cache-client'; +export { getVisibleItems, loadDataset as loadScatterbrainDataset } from './dataset'; export { buildRenderFrameFn as buildScatterbrainRenderFn, - buildScatterbrainCacheClient, setCategoricalLookupTableValues, updateCategoricalValue, } from './renderer'; export * from './types'; -export { getVisibleItems, loadDataset as loadScatterbrainDataset } from './dataset'; diff --git a/packages/scatterbrain/src/renderer.ts b/packages/scatterbrain/src/renderer.ts index 10a9e983..539a6f79 100644 --- a/packages/scatterbrain/src/renderer.ts +++ b/packages/scatterbrain/src/renderer.ts @@ -1,85 +1,20 @@ import type { SharedPriorityCache } from '@alleninstitute/vis-core'; -import type REGL from 'regl'; -import type { ColumnRequest, ScatterbrainDataset, SlideviewScatterbrainDataset, TreeNode } from './types'; import { Box2D, type box2D, type vec4 } from '@alleninstitute/vis-geometry'; -import { MakeTaggedBufferView } from './typed-array'; import keys from 'lodash/keys'; import reduce from 'lodash/reduce'; +import type REGL from 'regl'; +import { buildScatterbrainCacheClient } from './cache-client'; import { getVisibleItems, type NodeWithBounds } from './dataset'; import { buildScatterbrainRenderCommand, type Config, configureShader, type ShaderSettings, VBO } from './shader'; +import { MakeTaggedBufferView } from './typed-array'; +import type { ColumnRequest, ScatterbrainDataset, SlideviewScatterbrainDataset, TreeNode } from './types'; + export type Item = Readonly<{ dataset: SlideviewScatterbrainDataset | ScatterbrainDataset; node: TreeNode; bounds: box2D; columns: Record; }>; -type Content = Record; - -export function buildScatterbrainCacheClient( - allNeededColumns: readonly string[], - regl: REGL.Regl, - cache: SharedPriorityCache, - onDataArrived: () => void, -) { - const client = cache.registerClient({ - cacheKeys: (item) => { - const { dataset, node, columns } = item; - return reduce, Record>( - columns, - (acc, col, key) => ({ - ...acc, - [key]: `${dataset.metadata.metadataFileEndpoint}/${node.file}/${col.name}`, - }), - {}, - ); - }, - fetch: (item) => { - const { dataset, node, columns } = item; - const attrs = dataset.metadata.pointAttributes; - const getColumnUrl = (columnName: string) => - `${dataset.metadata.metadataFileEndpoint}${columnName}/${dataset.metadata.visualizationReferenceId}/${node.file}`; - const getGeneUrl = (columnName: string) => - `${dataset.metadata.geneFileEndpoint}${columnName}/${dataset.metadata.visualizationReferenceId}/${node.file}`; - const getColumnInfo = (col: ColumnRequest) => - col.type === 'QUANTITATIVE' - ? ({ url: getGeneUrl(col.name), elements: 1, type: 'float' } as const) - : { url: getColumnUrl(col.name), elements: attrs[col.name].elements, type: attrs[col.name].type }; - - const proms = reduce, Record Promise>>( - columns, - (getters, col, key) => { - const { url, type } = getColumnInfo(col); - return { - ...getters, - [key]: (signal) => - fetch(url, { signal }).then((b) => - b.arrayBuffer().then((buff) => { - const typed = MakeTaggedBufferView(type, buff); - return new VBO({ - buffer: regl.buffer({ type: type, data: typed.data }), - bytes: buff.byteLength, - type: 'buffer', - }); - }), - ), - }; - }, - {}, - ); - return proms; - }, - isValue: (v): v is Content => { - for (const column of allNeededColumns) { - if (!(column in v)) { - return false; - } - } - return true; - }, - onDataArrived, - }); - return client; -} function columnsForItem( config: Config, @@ -175,7 +110,7 @@ export function updateCategoricalValue( type ScatterbrainRenderProps = Omit>[0], 'item'> & { visibilityThresholdPx: number; dataset: ScatterbrainDataset | SlideviewScatterbrainDataset; - client: ReturnType; + client: ReturnType>; }; /** * @@ -214,8 +149,18 @@ export function buildRenderFrameFn(regl: REGL.Regl, settings: ShaderSettings) { } }; const connectToCache = (cache: SharedPriorityCache, onDataArrived: () => void) => { - const allColumns = [...config.categoricalColumns, ...config.quantitativeColumns, config.positionColumn]; - const client = buildScatterbrainCacheClient(allColumns, regl, cache, onDataArrived); + const client = buildScatterbrainCacheClient( + cache, + (buff, type) => { + const typed = MakeTaggedBufferView(type, buff); + return new VBO({ + buffer: regl.buffer({ type: type, data: typed.data }), + bytes: buff.byteLength, + type: 'buffer', + }); + }, + onDataArrived, + ); return client; }; return { render, connectToCache }; diff --git a/packages/scatterbrain/src/types.ts b/packages/scatterbrain/src/types.ts index 50a442ec..21782531 100644 --- a/packages/scatterbrain/src/types.ts +++ b/packages/scatterbrain/src/types.ts @@ -1,5 +1,8 @@ /// Types describing the metadata that gets loaded from scatterbrain.json files /// // there are 2 variants, slideview and regular - they are distinguished at runtime + +import type { box2D } from '@alleninstitute/vis-geometry'; + // by checking the parsed metadata for the 'slides' field export type WebGLSafeBasicType = 'uint8' | 'uint16' | 'int8' | 'int16' | 'uint32' | 'int32' | 'float'; @@ -82,3 +85,12 @@ export type SlideviewMetadata = CommonMetadata & { export type SlideviewScatterbrainDataset = { type: 'slideview'; metadata: SlideviewMetadata }; export type ScatterbrainDataset = { type: 'normal'; metadata: ScatterbrainMetadata }; + +// renderer-specific types: + +export type Item = Readonly<{ + dataset: SlideviewScatterbrainDataset | ScatterbrainDataset; + node: TreeNode; + bounds: box2D; + columns: Record; +}>;