Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
6e678af
Upgrade to whisper 3.4.2 and fix LD_LIBRARY_PATH issues
awong-dev Jun 30, 2025
acca603
More workarounds to library issues with whisper-3.4.2
awong-dev Jul 1, 2025
7ba23f5
get huggingface secret from docker secrets
awong-dev Jul 18, 2025
cd93eaa
update the transcription to handle missing publishing_date
awong-dev Aug 10, 2025
ee876d5
Make handling description errors more robust
awong-dev Sep 11, 2025
e7ee3f5
handle missing description
awong-dev Sep 11, 2025
e8025f9
Do sps-board first. Avoid all caps + basic spelling errors"
awong-dev Sep 28, 2025
160f478
Fix path mistakes and publishDate construciton
awong-dev Sep 29, 2025
83ac697
Fix initial prompt to confuse whisper less
awong-dev Sep 29, 2025
2da0b3b
move metadata population into the cloud function
awong-dev Nov 23, 2025
dff343d
fix null error and extend memory
awong-dev Nov 24, 2025
292e2b8
Use corrent masterkey variable name
awong-dev Nov 24, 2025
11cb552
use MEILISEARCH_MASTER_KEY uniformly in all areas
awong-dev Nov 24, 2025
ac5b557
Call wait from the client
awong-dev Nov 25, 2025
a636077
New version of meilisearch js api
awong-dev Nov 25, 2025
07e66a7
debugging one line at a time
awong-dev Nov 25, 2025
80cd74e
Upgrade to latest react@19 and next@16 to handle CVE-2025-55182
awong-dev Dec 4, 2025
8ac6d7b
Remove old budget stuff
awong-dev Dec 4, 2025
97ae5e5
Fix eslint config
awong-dev Dec 4, 2025
774a24b
Upgrade react and next for CVE-2025-55182
awong-dev Jan 23, 2026
51fa3a0
Fix typos
awong-dev Jan 23, 2026
f3efef1
add @types/langs, csv-parse to root package.json
wonderzombie Jan 23, 2026
d0d2add
Throw an error when receiving an invalid language code.
wonderzombie Jan 27, 2026
1775d07
use firebase `logger` for to start/stop transcript instances
wonderzombie Jan 26, 2026
2bd673e
More logging
awong-dev Mar 7, 2026
ac09dad
Add basic robots.txt and fix sitemap
awong-dev Mar 7, 2026
4695faa
Update constants.
awong-dev Mar 12, 2026
e87e186
Add in speaker timing information into transcripts
awong-dev Mar 13, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion app/[category]/v/[videoId]/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@ import IndexWithLang from './[lang]/page';

import type { VideoParams } from './[lang]/page';

export { generateMetadata, dynamic, revalidate } from './[lang]/page';
export { generateMetadata } from './[lang]/page';

export const dynamic = 'force-static';
export const revalidate = 3600;

export default async function NoLangVideo(props: {params: Promise<VideoParams>}) {
const params = await props.params;
Expand Down
2 changes: 1 addition & 1 deletion app/sitemap.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ type SiteMapEntry = {
};

function buildUrl(relativePath): string {
return `${Constants.SITE_ROOT_URL}/${relativePath}`;
return `${Constants.SITE_ROOT_URL}${relativePath}`;
}

export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
Expand Down
3 changes: 2 additions & 1 deletion common/transcript.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ export class DiarizedTranscript {
// Wait for everything.
const [transcriptDataResult, ...sentenceTableResult] = await Promise.allSettled(loadPromises);

console.log("Reading from Transcript " + videoId);
if (transcriptDataResult.status === 'rejected') {
return new DiarizedTranscript(category, videoId, EMPTY_TRASCRIPT_DATA, {},
["Cannout load transcript Data"]);
Expand Down Expand Up @@ -330,7 +331,7 @@ function toSentences(speaker : SpeakerId, firstId : number, words : string[], wo

function tsvToSentenceTable(tsv : string) : SentenceTable {
const sentenceTableRows : [string, string][] =
parse(tsv, { delimiter: '\t', trim: true });
parse(tsv, { delimiter: '\t', trim: true }) as [string, string][];
const sentenceTable : SentenceTable = {};
sentenceTableRows.forEach(row => sentenceTable[row[0]] = row[1]);
return sentenceTable;
Expand Down
6 changes: 5 additions & 1 deletion common/whisperx.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,11 @@ export function makeWhisperXTranscriptsPath(
category: CategoryId,
videoId: VideoId,
language: Iso6393Code): string {
const iso6391Code = langs.where('3', language)['1'];
const iso6391Code = langs.where('3', language)?.['1'];
if (!iso6391Code) {
throw new Error(`Unsupported language code: ${language}`);
}

return makePublicPath(
category,
Constants.WHISPERX_ARCHIVE_SUBDIR,
Expand Down
2 changes: 2 additions & 0 deletions components/SearchResult.tsx
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
'use client';

import * as Constants from 'config/constants';
import Box from '@mui/material/Box';
import Divider from '@mui/material/Divider';
Expand Down
6 changes: 4 additions & 2 deletions components/SpeakerBubble.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,15 @@ import { toSpeakerColorClass } from 'utilities/client/css';

type SpeakerBubbleParams = {
speakerNum : number;
start: number;
end: number;
children: React.ReactNode[];
};

export default function SpeakerBubble({speakerNum, children} : SpeakerBubbleParams) {
export default function SpeakerBubble({speakerNum, start, end, children} : SpeakerBubbleParams) {
return (
<Paper component="article" className={`b ${toSpeakerColorClass(speakerNum)}`}>
<SpeakerBubbleTitle speakerNum={speakerNum} />
<SpeakerBubbleTitle speakerNum={speakerNum} start={start} end={end} />
{ children }
</Paper>
);
Expand Down
23 changes: 22 additions & 1 deletion components/SpeakerBubbleTitle.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,24 @@ import { useContext } from 'react'

type SpeakerBubbleTitleParams = {
speakerNum : number;
start: number;
end: number;
};

export default function SpeakerBubbleTitle({speakerNum} : SpeakerBubbleTitleParams) {
function formatTime(totalSeconds : number) {
const minutes = Math.floor(totalSeconds / 60);
const seconds = totalSeconds % 60;

if (totalSeconds < 60) {
return `${seconds.toFixed(0)}s`;
}

// Format as MM:SS
return `${String(minutes)}m${String(seconds.toFixed(0)).padStart(2, '0')}s`
}

export default function SpeakerBubbleTitle({speakerNum, start, end} : SpeakerBubbleTitleParams) {
const durationSecs = end - start;
const annotationsContext = useAnnotations();
const { name, tags } = getSpeakerAttributes(speakerNum,
annotationsContext.speakerInfo);
Expand Down Expand Up @@ -47,6 +62,12 @@ export default function SpeakerBubbleTitle({speakerNum} : SpeakerBubbleTitlePara
})
}
</Stack>
<Box sx={{
flex: "0 0 auto",
maxWidth: "45%"
}}>
<Typography className="d" data-timing={`${start}-${end}`} variant="body2">[{formatTime(durationSecs)}]</Typography>
</Box>
<Box>
<Typography>
<IconButton
Expand Down
26 changes: 17 additions & 9 deletions components/Transcript.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -54,17 +54,25 @@ export default function Transcript({
// Merge all segments from the same speaker to produce speaking divs.
const speakerBubbles = diarizedTranscript.groupSentenceInfoBySpeaker().map((bubble, i) => {
speakerNums.add(bubble.speaker);
let bubbleStart = Number.MAX_VALUE;
let bubbleEnd = 0;
const sentenceElements = new Array<ReactNode>;
for (const si of bubble.sentenceInfo) {
const [segmentId, speakerId, start, end] = si;
bubbleStart = Math.min(bubbleStart, start);
bubbleEnd = Math.max(bubbleEnd, end);

sentenceElements.push(
<p key={ `${i}-${segmentId}` }
className={ toTimeClassName(start) }>
{ textLines( segmentId, languageOrder, diarizedTranscript) }
</p>
);
}

return (
<SpeakerBubble key={i} speakerNum={ bubble.speaker }>
{
bubble.sentenceInfo.map(([segmentId, speakerId, start]) => (
<p key={ `${i}-${segmentId}` }
className={ toTimeClassName(start) }>
{ textLines( segmentId, languageOrder, diarizedTranscript) }
</p>
))
}
<SpeakerBubble key={i} speakerNum={ bubble.speaker } start={bubbleStart} end={bubbleEnd}>
{sentenceElements}
</SpeakerBubble>
);
});
Expand Down
98 changes: 98 additions & 0 deletions components/TranscriptControlBar.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,14 @@ import FormGroup from '@mui/material/FormGroup';
import IconButton from '@mui/material/IconButton';
import LanguageNav from 'components/LanguageNav';
import Paper from '@mui/material/Paper';
import DownloadIcon from '@mui/icons-material/Download';
import PublishIcon from '@mui/icons-material/Publish';
import Stack from '@mui/material/Stack';
import Switch from '@mui/material/Switch';
import Tooltip from '@mui/material/Tooltip';
import { fromTimeClassName } from 'utilities/client/css';
import { useContext, useState } from 'react';
import { stringify } from 'csv-stringify/browser/esm/sync'
import { VideoControlContext } from 'components/providers/VideoControlProvider';
import { useActionDialog } from 'components/providers/ActionDialogProvider'
import { useAnnotations } from 'components/providers/AnnotationsProvider'
Expand All @@ -24,6 +27,86 @@ type TranscriptControlBarProps = {
sx?: SxProps<Theme>;
};

type CsvEntry = {
start: number;
end: number;
speakerName: string;
speakerTags: string;
sentences: string;
};

function downloadTranscriptCsv(csvContent, fileName) {
// Create a Blob with the CSV data and type
const blob = new Blob([csvContent], { type: 'text/csv;charset=utf-8;' });

// Create a URL for the Blob
const url = URL.createObjectURL(blob);

// Create an anchor tag for downloading
const a = document.createElement('a');
a.href = url;
a.download = fileName || 'download.csv'; // Set the file name

// Append the anchor to the body, click it, and remove it
document.body.appendChild(a); // Required for Firefox
a.click();
document.body.removeChild(a);

// Revoke the object URL to free up memory (optional but good practice)
URL.revokeObjectURL(url);
}

function extractStartEnd(headerSection) : [number, number] {
const timingElement = headerSection.getElementsByClassName('d')[0];
if (timingElement instanceof HTMLElement) {
const timing = timingElement.dataset['timing'];
if (timing) {
const splits = timing.split('-').map(x => parseInt(x));
if (splits.length === 2) {
return [splits[0], splits[1]];
}
}
}

return [-1, -1];
}

function generateTranscriptCsv() {
const entries = new Array<CsvEntry>;

const speakerBubbles = document.getElementsByTagName('main')[0].getElementsByTagName('article');
for (const bubble of speakerBubbles) {
// Parse the headers.
const headerSection = bubble.firstElementChild;
if (!headerSection) {
continue;
}
const speakerName = headerSection.getElementsByTagName('h5')[0]?.textContent ?? "unknown";
const [start, end] = extractStartEnd(headerSection);

const tagList = new Array<string>;
for (const s of headerSection.getElementsByTagName('span')) {
tagList.push(s.textContent);
}

// Parse the sentences. First paragraph is buttons.
const paragraphs = bubble.getElementsByTagName('p') ?? [];
const segments = new Array<string>;
for (let i = 1; i < paragraphs.length; i++) {
segments.push(paragraphs[i].textContent);
}
entries.push({
start,
end,
speakerName,
speakerTags: tagList.join(', '),
sentences: segments.join('\n'),
});
}

return stringify(entries, {header: true});
}

export default function TranscriptControlBar(
{ curLang, sx = [] }: TranscriptControlBarProps) {
const [ autoscroll, setAutoscroll ] = useState<boolean>(true);
Expand Down Expand Up @@ -77,6 +160,21 @@ export default function TranscriptControlBar(
curLang={curLang}
sx={{width: "100%"}}
/>
<Tooltip title="Download Transcript">
<span>
<Button
variant="contained"
aria-label="download-transcript"
color="primary"
onClick={() => downloadTranscriptCsv(
generateTranscriptCsv(),
`${window.location.pathname.split('/').pop()}.csv`)}
sx={{width: "100%", height: "100%"}}
>
<DownloadIcon />
</Button>
</span>
</Tooltip>
<Tooltip title="Publish changes">
<span>
<Button
Expand Down
2 changes: 1 addition & 1 deletion config/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ export const STORAGE_BUCKET = "sps-by-the-numbers.appspot.com";

// Firebase configuration for web clients. This data is public.
export const FIREBASE_CLIENT_CONFIG = {
apiKey: "AIzaSyD30a3gVbP-7PgTvTqCjW4xx-GlLMBQ5Ns",
apiKey: "AIzaSyDN7oBGINczK2mosHWdpDRCCl1AMjFrV4k",
authDomain: "sps-by-the-numbers.firebaseapp.com",
databaseURL: "https://sps-by-the-numbers-default-rtdb.firebaseio.com",
projectId: "sps-by-the-numbers",
Expand Down
25 changes: 25 additions & 0 deletions eslint.config.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import { defineConfig, globalIgnores } from "eslint/config";
import path from "node:path";
import { fileURLToPath } from "node:url";
import js from "@eslint/js";
import { FlatCompat } from "@eslint/eslintrc";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const compat = new FlatCompat({
baseDirectory: __dirname,
recommendedConfig: js.configs.recommended,
allConfig: js.configs.all
});

export default defineConfig([
globalIgnores(["tools/**/*", "functions/**/*", "functions-python/**/*", "node_modules/**/*", "coverage", ".*/**/*"]),
{
extends: compat.extends("next/core-web-vitals"),

rules: {
"@next/next/no-img-element": "off",
"@next/next/no-page-custom-font": "off",
},
},
]);
Loading
Loading