feat: add canvas paintaings to media dialog

src/plugin/Panel/MediaDialog/index.tsx

@@ -1,8 +1,9 @@
 import { Dialog, Heading, ImageSelect } from "@components";
 import { usePlugin } from "@context";
-import type { CanvasNormalized, ContentResourceNormalized } from "@iiif/presentation-3-normalized";
+import { serializeConfigPresentation3, Traverse } from "@iiif/parser";
+import type { Canvas, ContentResource } from "@iiif/presentation-3";
 import type { Media } from "@types";
-import { getLabelByUserLanguage } from "@utils";
+import { getLabelByUserLanguage, updateIIIFImageRequestURI } from "@utils";
 import { FC, useEffect, useRef } from "react";
 import style from "./style.module.css";
 
@@ -23,6 +24,37 @@ function handleSelectedMedia(
   onUpdate(resources);
 }
 
+// Though the resource may have `width` and `height` properties
+// it's type does not include them.
+// To ensure we can access them safely, without excessive type checking,
+// cast the resource to `any` and then try to access them.
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function getDimensions(resource: any): { height: number; width: number } {
+  return {
+    height: resource?.height || 0,
+    width: resource?.width || 0,
+  };
+}
+
+/**
+ * Format a caption string for a IIIF resource including its dimensions if available.
+ *
+ * @param topline the top line of the caption, e.g. "Thumbnail"
+ * @param resource the IIIF resource to get dimensions from
+ * @returns a formatted caption string
+ *
+ * @example
+ * ```ts
+ * formatCaption("Thumbnail", resource);
+ * // "Thumbnail\n(100 x 200)"
+ * ```
+ */
+function formatCaption(topline: string, resource: ContentResource) {
+  const { width, height } = getDimensions(resource);
+  const dimensions = width && height ? `\n(${width} x ${height})` : "";
+  return topline + dimensions;
+}
+
 const CurrentView = () => {
   const { state, dispatch } = usePlugin();
 
@@ -88,134 +120,165 @@ const CurrentView = () => {
   );
 };
 
-const Placeholder = () => {
+const Paintings: FC<{ canvas: Canvas }> = ({ canvas }) => {
   const { state, dispatch } = usePlugin();
 
   function handleAddMedia(selected: boolean, media: Media) {
+    // when adding the media, update the size to be max
+    media.src = updateIIIFImageRequestURI(media.src, {
+      size: `max`,
+    });
     handleSelectedMedia(selected, media, state.selectedMedia, (resources) =>
       dispatch({ type: "setSelectedMedia", selectedMedia: resources }),
     );
   }
 
-  // Though the resource may have `width` and `height` properties
-  // it's type, ContentResource, does not include them.
-  // To ensure we can access them safely,
-  // cast the resource to `any` and then try to access them.
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  function getDimensions(resource: any): { height: number; width: number } {
-    return {
-      height: resource.height || 0,
-      width: resource.width || 0,
-    };
-  }
+  const paintings: ContentResource[] = [];
+  const traverse = new Traverse({
+    contentResource: [
+      (resource) => {
+        if (resource.type === "Image") {
+          paintings.push(resource);
+        }
+      },
+    ],
+  });
+  traverse.traverseCanvasItems(canvas);
 
-  function formatCaption(resource: ContentResourceNormalized) {
-    const { width, height } = getDimensions(resource);
-    const dimensions = width && height ? `\n(${width} x ${height})` : "";
-    return `Placeholder${dimensions}`;
+  if (!paintings.length) {
+    return <></>;
   }
 
-  const activeCanvas = state.activeCanvas;
+  const media: Media[] = paintings.reduce((acc, painting, i) => {
+    if (!painting.id) {
+      return acc;
+    }
 
-  if (!activeCanvas.placeholderCanvas) {
-    return <></>;
+    // @ts-expect-error - I don't want to go down the rabbit hole of type checking here
+    const label = getLabelByUserLanguage(painting?.label);
+    acc.push({
+      type: "image",
+      id: `painting-${i}`,
+      // use a smaller size so as to not load large images
+      // update the size when the media is added
+      src: updateIIIFImageRequestURI(painting.id, {
+        size: "!300,300",
+      }),
+      caption: formatCaption(label[0] ? label[0] : "Painting", painting),
+    });
+
+    return acc;
+  }, [] as Media[]);
+
+  return (
+    <>
+      {media.map((m) => (
+        <ImageSelect
+          figcaption={m.caption}
+          imgObjectFit="cover"
+          initialState={isMediaInSelectedMedia(m, state.selectedMedia) ? "selected" : "unselected"}
+          key={`painting-${m.id}`}
+          src={m.src}
+          onSelectionChange={(selected) => handleAddMedia(selected, m)}
+        />
+      ))}
+    </>
+  );
+};
+
+const Placeholder: FC<{ placeholder: NonNullable<Canvas["placeholderCanvas"]> }> = ({
+  placeholder,
+}) => {
+  const { state, dispatch } = usePlugin();
+
+  function handleAddMedia(selected: boolean, media: Media) {
+    handleSelectedMedia(selected, media, state.selectedMedia, (resources) =>
+      dispatch({ type: "setSelectedMedia", selectedMedia: resources }),
+    );
   }
 
-  const placeholderResource = state.vault.get({
-    type: "Canvas",
-    id: activeCanvas.placeholderCanvas.id,
+  const paintings: ContentResource[] = [];
+  const traverse = new Traverse({
+    contentResource: [
+      (resource) => {
+        if (resource.type === "Image") {
+          paintings.push(resource);
+        }
+      },
+    ],
   });
+  traverse.traverseCanvasItems(placeholder);
 
-  const annotationItems = state.vault.get({
-    type: "AnnotationPage",
-    id: placeholderResource.items.map((item) => item.id),
-  });
+  if (!paintings.length) {
+    return <></>;
+  }
 
-  const annotationPageItems = state.vault.get({
-    type: "Annotation",
-    id: annotationItems.items.map((item) => item.id),
-  });
+  const media: Media[] = paintings.reduce((acc, painting, i) => {
+    if (!painting.id) {
+      return acc;
+    }
 
-  const body = state.vault.get({
-    type: "ContentResource",
-    id: annotationPageItems.body.map((b) => b.id),
-  });
+    acc.push({
+      type: "image",
+      id: `placeholder-${i}`,
+      src: painting.id,
+      caption: formatCaption("Placeholder", painting),
+    });
 
-  const id = body.id;
-  const paintingMedia: Media = {
-    type: "image",
-    id: id || "",
-    src: id || "",
-    caption: formatCaption(body),
-  };
+    return acc;
+  }, [] as Media[]);
 
   return (
     <>
-      {id && (
+      {media.map((m) => (
         <ImageSelect
-          figcaption={paintingMedia.caption}
-          imgObjectFit="contain"
-          src={id}
-          initialState={
-            isMediaInSelectedMedia(paintingMedia, state.selectedMedia) ? "selected" : "unselected"
-          }
-          onSelectionChange={(selected) => handleAddMedia(selected, paintingMedia)}
+          figcaption={m.caption}
+          imgObjectFit="cover"
+          initialState={isMediaInSelectedMedia(m, state.selectedMedia) ? "selected" : "unselected"}
+          key={`placeholder-${m.id}`}
+          src={m.src}
+          onSelectionChange={(selected) => handleAddMedia(selected, m)}
         />
-      )}
+      ))}
     </>
   );
 };
 
-const Thumbnail: FC<{
-  thumbnail: CanvasNormalized["thumbnail"][0];
-}> = ({ thumbnail }) => {
+const Thumbnails: FC<{ thumbnails: ContentResource[] }> = ({ thumbnails }) => {
   const { state, dispatch } = usePlugin();
-  const resource = state.vault.get({ type: "ContentResource", id: thumbnail.id });
 
   function handleAddMedia(selected: boolean, media: Media) {
     handleSelectedMedia(selected, media, state.selectedMedia, (resources) =>
       dispatch({ type: "setSelectedMedia", selectedMedia: resources }),
     );
   }
 
-  // Though the resource may have `width` and `height` properties
-  // it's type, ContentResource, does not include them.
-  // To ensure we can access them safely,
-  // cast the resource to `any` and then try to access them.
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  function getDimensions(resource: any): { height: number; width: number } {
-    return {
-      height: resource.height || 0,
-      width: resource.width || 0,
-    };
-  }
-
-  function formatCaption(resource: ContentResourceNormalized) {
-    const { width, height } = getDimensions(resource);
-    const dimensions = width && height ? `\n(${width} x ${height})` : "";
-    return `Thumbnail${dimensions}`;
-  }
-
-  const id = thumbnail.id;
-  const thumbnailMedia: Media = {
-    type: "image",
-    id: id || "",
-    src: id || "",
-    caption: formatCaption(resource),
-  };
+  const media: Media[] = thumbnails.reduce((acc, thumbnail, i) => {
+    if (!thumbnail.id) {
+      return acc;
+    }
+    acc.push({
+      type: "image",
+      id: `thumbnail-${i}`,
+      src: thumbnail.id,
+      caption: formatCaption("Thumbnail", thumbnail),
+    });
+    return acc;
+  }, [] as Media[]);
 
   return (
     <>
-      {id && (
+      {media.map((thumbnail, index) => (
         <ImageSelect
-          figcaption={thumbnailMedia.caption}
-          src={thumbnailMedia.src}
+          figcaption={thumbnail.caption}
+          key={`thumbnail-${index}`}
+          src={thumbnail.src}
           initialState={
-            isMediaInSelectedMedia(thumbnailMedia, state.selectedMedia) ? "selected" : "unselected"
+            isMediaInSelectedMedia(thumbnail, state.selectedMedia) ? "selected" : "unselected"
           }
-          onSelectionChange={(selected) => handleAddMedia(selected, thumbnailMedia)}
+          onSelectionChange={(selected) => handleAddMedia(selected, thumbnail)}
         />
-      )}
+      ))}
     </>
   );
 };
@@ -224,7 +287,7 @@ export const MediaDialog = () => {
   const { state, dispatch } = usePlugin();
   const dialogRef = useRef<HTMLDialogElement>(null);
   const initialFocusRef = useRef<HTMLDivElement>(null);
-  const canvasTitle = getLabelByUserLanguage(state.activeCanvas.label)[0];
+  const canvasTitle = getLabelByUserLanguage(state.activeCanvas?.label ?? undefined)[0];
 
   function closeDialog() {
     dispatch({ type: "setMediaDialogState", state: "closed" });
@@ -260,6 +323,14 @@ export const MediaDialog = () => {
     };
   }, [state.mediaDialogState, dispatch]);
 
+  // serialize the Canvas so it can traversed
+  const canvas: Canvas = state.vault.serialize(
+    { type: "Canvas", id: state.activeCanvas.id },
+    serializeConfigPresentation3,
+  );
+
+  // NOTE: It is assumed that all media resources are images hosted on the internet
+  // and that they can be used directly as the `src` for an `<img>` element.
   return (
     <Dialog
       aria-modal="true"
@@ -284,14 +355,9 @@ export const MediaDialog = () => {
           <div className={style.contentContainer}>
             <div className={style.content}>
               <CurrentView />
-              <Placeholder />
-              {state.activeCanvas.thumbnail.length > 0 && (
-                <>
-                  {state.activeCanvas.thumbnail.map((thumbnail, i) => (
-                    <Thumbnail key={`thumbnail-${i}`} thumbnail={thumbnail} />
-                  ))}
-                </>
-              )}
+              <Paintings canvas={canvas} />
+              {canvas.placeholderCanvas && <Placeholder placeholder={canvas.placeholderCanvas} />}
+              {canvas.thumbnail?.length && <Thumbnails thumbnails={canvas.thumbnail} />}
             </div>
           </div>
         </>

src/plugin/Panel/MediaDialog/style.module.css

@@ -16,10 +16,9 @@
 }
 
 .content {
-  /* display: grid; */
-  /* grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); */
   display: flex;
   gap: var(--clover-ai-sizes-2);
+  flex-wrap: wrap;
 
   figcaption {
     --figure-caption-font-size: var(--clover-ai-sizes-3);

src/plugin/Panel/index.test.tsx

@@ -70,6 +70,23 @@ const mockClover = {
           };
         return null;
       }),
+      serialize: vi.fn((query) => {
+        if (query.type === "Canvas")
+          return {
+            id: "canvas-1",
+            type: "Canvas",
+            label: { en: ["Test Canvas"] },
+            thumbnail: [
+              {
+                id: "thumbnail-1",
+                type: "Image",
+                format: "image/jpeg",
+              },
+            ],
+            items: [],
+          };
+        return null;
+      }),
     },
     activeManifest: "manifest-1",
     activeCanvas: "canvas-1",

src/plugin/Panel/index.tsx

@@ -56,7 +56,7 @@ export function PluginPanelComponent(props: CloverPlugin & PluginProps) {
         type: "setSystemPrompt",
         systemPrompt: `You are a helpful assistant that can answer questions about the item in the viewer. Here is the manifest data for the item:\n\n${JSON.stringify(state.manifest["metadata"], null, 2)}`,
       });
-      const label = state.manifest.label;
+      const label = state.manifest?.label ?? undefined;
       const title = getLabelByUserLanguage(label);
       setItemTitle(title.length > 0 ? title[0] : "this item");
     }