feat: add chord recognition

components/ChordRecognition/UploadProcessing.tsx

@@ -0,0 +1,71 @@
+import { Button, Progress, Upload } from "antd";
+import UploadOutlined from "@ant-design/icons/UploadOutlined";
+import { useTranslation } from "react-i18next";
+
+interface UploadProcessingProps {
+  isConnected: boolean;
+  isProcessing: boolean;
+  progressText: string;
+  progressPercent: number;
+  errorMessage: string;
+  onSelectFile: (file: File) => void;
+}
+
+export function UploadProcessing({
+  isConnected,
+  isProcessing,
+  progressText,
+  progressPercent,
+  errorMessage,
+  onSelectFile,
+}: UploadProcessingProps) {
+  const { t } = useTranslation("common");
+  return (
+    <>
+      <div className="flex items-center justify-center gap-3 flex-wrap">
+        <Upload
+          accept="audio/*"
+          showUploadList={false}
+          disabled={!isConnected || isProcessing}
+          beforeUpload={(file) => {
+            onSelectFile(file as File);
+            return false;
+          }}
+        >
+          <Button
+            type="default"
+            icon={<UploadOutlined />}
+            loading={isProcessing}
+            disabled={!isConnected || isProcessing}
+            className="border-gray-400 text-gray-800"
+          >
+            {t("上传音频文件")}
+          </Button>
+        </Upload>
+      </div>
+
+      {(isProcessing || progressText) && (
+        <div className="flex flex-col gap-2 w-full">
+          {progressText && (
+            <div className="text-xs text-gray-500 text-center">
+              {progressText}
+            </div>
+          )}
+          <Progress
+            percent={progressPercent}
+            status={
+              errorMessage ? "exception" : isProcessing ? "active" : "normal"
+            }
+            strokeColor="#6b7280"
+            trailColor="#f3f4f6"
+            showInfo={false}
+          />
+        </div>
+      )}
+
+      {errorMessage && (
+        <div className="text-red-600 text-xs text-center">{errorMessage}</div>
+      )}
+    </>
+  );
+}

components/ChordRecognition/api.ts

@@ -0,0 +1,163 @@
+import { useEffect, useRef, useState } from "react";
+import { wrap, proxy } from "comlink";
+
+export interface ChordSegment {
+  /**
+   * In seconds
+   */
+  start: number;
+  /**
+   * In seconds
+   */
+  end: number;
+  /**
+   * Possible labels (with root note C as example):
+   * - C:min/b7
+   * - C:min/2
+   * - C:maj/b7
+   * - C:maj/2
+   * - C:sus4(b7)
+   * - C:sus2
+   * - C:sus4
+   * - C:13
+   * - C:11
+   * - C:min9
+   * - C:9
+   * - C:maj9
+   * - C:dim7
+   * - C:hdim7
+   * - C:min7
+   * - C:7
+   * - C:maj7
+   * - C:min/5
+   * - C:min/b3
+   * - C:maj/5
+   * - C:maj/3
+   * - C:dim
+   * - C:aug
+   * - C:min
+   * - C:maj
+   * - N
+   */
+  label: string;
+}
+
+export type MeteredBeatEvent = {
+  time: number;
+  beatInBar: number;
+  meter: number;
+  isDownbeat: boolean;
+};
+
+interface RecognizeChordsArgs {
+  audio: File;
+}
+
+interface RecognizeBeatsArgs {
+  audio: File;
+}
+
+interface ChordMiniApi {
+  recognizeChords: (
+    args: RecognizeChordsArgs,
+    onLogProgress?: (message: string, stepProgress?: number) => void
+  ) => Promise<ChordSegment[]>;
+  recognizeBeats: (
+    args: RecognizeBeatsArgs,
+    onLogProgress?: (message: string, stepProgress?: number) => void
+  ) => Promise<MeteredBeatEvent[]>;
+  forceUpdate: () => void;
+  version: () => string;
+}
+
+export interface RecognitionApi {
+  recognizeChords: (
+    args: RecognizeChordsArgs,
+    onLogProgress?: (
+      message: string,
+      /** 0 ~ 100 */ stepProgress?: number
+    ) => void
+  ) => Promise<ChordSegment[]>;
+  recognizeBeats: (
+    args: RecognizeBeatsArgs,
+    onLogProgress?: (message: string, stepProgress?: number) => void
+  ) => Promise<MeteredBeatEvent[]>;
+}
+
+export function useChordMiniApi() {
+  const [recognitionApi, setRecognitionApi] = useState<RecognitionApi | null>(
+    null
+  );
+  const iframeRef = useRef<HTMLIFrameElement>(null);
+
+  useEffect(() => {
+    const iframe = iframeRef.current;
+    if (!iframe) {
+      console.warn("[ChordMiniApi]: iframeRef is not set");
+      return;
+    }
+
+    let disposed = false;
+    let connected = false;
+
+    const connect = () => {
+      if (disposed || connected || !iframe.contentWindow) {
+        return;
+      }
+
+      const channel = new MessageChannel();
+      const api = wrap<ChordMiniApi>(channel.port1);
+
+      console.log("[ChordMiniApi]: Connecting to ChordMini Api...");
+      iframe.contentWindow.postMessage(
+        {
+          type: "chordmini-api",
+          port: channel.port2,
+        },
+        "*",
+        [channel.port2]
+      );
+
+      channel.port1.addEventListener(
+        "message",
+        (event) => {
+          if (disposed) {
+            return;
+          }
+          connected = true;
+          console.log("[ChordMiniApi]: Connected to ChordMini Api", event.data);
+          setRecognitionApi({
+            recognizeChords: (args, callback) => {
+              return api.recognizeChords(
+                args, // cloned (lightweight file handle)
+                callback ? proxy(callback) : undefined // proxied (callable function)
+              );
+            },
+            recognizeBeats: (args, callback) => {
+              return api.recognizeBeats(
+                args, // cloned (lightweight file handle)
+                callback ? proxy(callback) : undefined // proxied (callable function)
+              );
+            },
+          });
+        },
+        { once: true }
+      );
+
+      channel.port1.start();
+    };
+
+    const interval = setInterval(connect, 1400);
+
+    return () => {
+      disposed = true;
+      clearInterval(interval);
+      console.log("[ChordMiniApi]: Disposed");
+    };
+  }, []);
+
+  return {
+    recognitionApi,
+    iframeRef,
+  };
+}

components/ChordRecognition/index.tsx

@@ -0,0 +1,160 @@
+import { useEffect, useRef, useState } from "react";
+import { Spin } from "antd";
+import { ChordSegment, MeteredBeatEvent, useChordMiniApi } from "./api";
+import {
+  EstimatedChordSegment,
+  estimateKey,
+} from "./key-estimation/estimate-key";
+import { UploadProcessing } from "./UploadProcessing";
+import { RecognitionResult } from "./RecognitionResult";
+import { useTranslation } from "react-i18next";
+
+interface ConnectionStatusProps {
+  isConnected: boolean;
+}
+
+function ConnectionStatus({ isConnected }: ConnectionStatusProps) {
+  const { t } = useTranslation("common");
+
+  if (isConnected) {
+    return null;
+  }
+
+  return (
+    <div className="flex items-center justify-center gap-2 text-gray-500 text-sm">
+      <Spin size="small" />
+      <span>{t("加载中")}</span>
+    </div>
+  );
+}
+
+export function ChordRecognition() {
+  const { iframeRef, recognitionApi } = useChordMiniApi();
+
+  const [isProcessing, setIsProcessing] = useState(false);
+  const [progressText, setProgressText] = useState<string>("");
+  const [progressPercent, setProgressPercent] = useState<number>(0);
+  const [errorMessage, setErrorMessage] = useState<string>("");
+  const [audioUrl, setAudioUrl] = useState<string>("");
+  const [segments, setSegments] = useState<EstimatedChordSegment[]>([]);
+  const [beats, setBeats] = useState<MeteredBeatEvent[]>([]);
+  const objectUrlRef = useRef<string>("");
+
+  useEffect(() => {
+    return () => {
+      if (objectUrlRef.current) {
+        URL.revokeObjectURL(objectUrlRef.current);
+      }
+    };
+  }, []);
+
+  const startRecognition = async (audioFile: File) => {
+    if (!recognitionApi) {
+      return;
+    }
+
+    if (objectUrlRef.current) {
+      URL.revokeObjectURL(objectUrlRef.current);
+      objectUrlRef.current = "";
+    }
+
+    const nextUrl = URL.createObjectURL(audioFile);
+    objectUrlRef.current = nextUrl;
+    setAudioUrl(nextUrl);
+    setSegments([]);
+    setBeats([]);
+    setErrorMessage("");
+    setIsProcessing(true);
+    setProgressPercent(0);
+    setProgressText("Uploading audio...");
+
+    try {
+      const beats = await recognitionApi.recognizeBeats(
+        {
+          audio: audioFile,
+        },
+        (message, stepProgress) => {
+          setProgressText(message || "Processing audio...");
+          if (typeof stepProgress === "number") {
+            setProgressPercent(stepProgress);
+          }
+        }
+      );
+      setBeats(beats);
+
+      const result = await recognitionApi.recognizeChords(
+        {
+          audio: audioFile,
+        },
+        (message, stepProgress) => {
+          setProgressText(message || "Processing audio...");
+          if (typeof stepProgress === "number") {
+            setProgressPercent(stepProgress);
+          }
+        }
+      );
+
+      const estimatedSegments = estimateKey(result);
+      setSegments(estimatedSegments);
+      setProgressPercent(100);
+      setProgressText("Recognition complete.");
+    } catch (error) {
+      console.error("[ChordRecognition]: recognizeChords failed", error);
+      setErrorMessage("Chord recognition failed. Please try another file.");
+      setProgressText("Recognition failed.");
+    } finally {
+      setIsProcessing(false);
+    }
+  };
+
+  return (
+    <div className="w-full max-w-6xl mx-auto p-4 flex flex-col gap-4">
+      <iframe
+        ref={iframeRef}
+        src="https://chordmini-web.pages.dev/"
+        width="0"
+        height="0"
+        referrerPolicy="no-referrer"
+        allow="cross-origin-isolated"
+        style={{
+          border: 0,
+          opacity: 0,
+          pointerEvents: "none",
+          position: "absolute",
+        }}
+        title="ChordMini Api Frame"
+      />
+
+      <div className="p-4 flex flex-col gap-3 max-w-md w-full mx-auto items-center text-center">
+        <ConnectionStatus isConnected={!!recognitionApi} />
+        <UploadProcessing
+          isConnected={!!recognitionApi}
+          isProcessing={isProcessing}
+          progressText={progressText}
+          progressPercent={progressPercent}
+          errorMessage={errorMessage}
+          onSelectFile={(file) => {
+            void startRecognition(file);
+          }}
+        />
+      </div>
+
+      <RecognitionResult
+        audioUrl={audioUrl}
+        segments={segments}
+        beats={beats}
+      />
+
+      <div className="text-xs text-gray-500 text-center opacity-40">
+        Recognition powered by{" "}
+        <a
+          href="https://chordmini-web.pages.dev/"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          ChordMini Web
+        </a>
+      </div>
+    </div>
+  );
+}