Add long screenshot utilities

packages/electron-screenshots/src/index.ts

@@ -57,6 +57,9 @@ export default class Screenshots extends Events {
 
   private singleWindow: boolean;
 
+  private longTimer: NodeJS.Timeout | null = null;
+  private longDisplay: Display | null = null;
+
   private isReady = new Promise<void>((resolve) => {
     ipcMain.once('SCREENSHOTS:ready', () => {
       this.logger('SCREENSHOTS:ready');
@@ -396,5 +399,30 @@ export default class Screenshots extends Events {
         this.endCapture();
       },
     );
+
+    ipcMain.on('SCREENSHOTS:long-start', async (_e, data: ScreenshotsData) => {
+      this.logger('SCREENSHOTS:long-start');
+      this.longDisplay = data.display;
+      if (this.$win) {
+        this.$win.setIgnoreMouseEvents(true, { forward: true });
+      }
+      this.longTimer = setInterval(async () => {
+        if (!this.longDisplay) return;
+        const url = await this.capture(this.longDisplay);
+        this.$view.webContents.send('SCREENSHOTS:long-add', url);
+      }, 500);
+    });
+
+    ipcMain.on('SCREENSHOTS:long-stop', () => {
+      this.logger('SCREENSHOTS:long-stop');
+      if (this.$win) {
+        this.$win.setIgnoreMouseEvents(false);
+      }
+      if (this.longTimer) {
+        clearInterval(this.longTimer);
+        this.longTimer = null;
+      }
+      this.$view.webContents.send('SCREENSHOTS:long-end');
+    });
   }
 }

packages/electron-screenshots/src/preload.ts

@@ -48,6 +48,14 @@ contextBridge.exposeInMainWorld('screenshots', {
 
     ipcRenderer.send('SCREENSHOTS:ok', Buffer.from(arrayBuffer), data);
   },
+  longStart: (data: ScreenshotsData) => {
+    console.log('contextBridge longStart', data);
+    ipcRenderer.send('SCREENSHOTS:long-start', data);
+  },
+  longStop: () => {
+    console.log('contextBridge longStop');
+    ipcRenderer.send('SCREENSHOTS:long-stop');
+  },
   on: (channel: string, fn: ScreenshotsListener) => {
     console.log('contextBridge on', fn);
 

packages/react-screenshots/package.json

@@ -38,6 +38,9 @@
   "publishConfig": {
     "registry": "https://registry.npmjs.org/"
   },
+  "dependencies": {
+    "@techstark/opencv-js": "^4.10.0-release.1"
+  },
   "peerDependencies": {
     "react": ">=16.8",
     "react-dom": ">=16.8"

packages/react-screenshots/src/Screenshots/ScreenshotsContext.ts

@@ -1,12 +1,14 @@
 import React, { Dispatch, SetStateAction } from 'react'
 import { EmiterRef, History, Bounds, CanvasContextRef } from './types'
+import { Display } from '../electron/app'
 import zhCN, { Lang } from './zh_CN'
 
 export interface ScreenshotsContextStore {
   url?: string
   image: HTMLImageElement | null
   width: number
   height: number
+  display?: Display
   lang: Lang
   emiterRef: EmiterRef
   canvasContextRef: CanvasContextRef
@@ -35,6 +37,7 @@ export default React.createContext<ScreenshotsContextValue>({
     image: null,
     width: 0,
     height: 0,
+    display: undefined,
     lang: zhCN,
     emiterRef: { current: {} },
     canvasContextRef: { current: null },

packages/react-screenshots/src/Screenshots/exports.ts

@@ -1,3 +1,4 @@
 export { Bounds } from './types'
 export { Lang } from './zh_CN'
 export { default, ScreenshotsProps } from './'
+export * from './longScreenshot'

packages/react-screenshots/src/Screenshots/index.tsx

@@ -7,19 +7,21 @@ import ScreenshotsCanvas from './ScreenshotsCanvas'
 import ScreenshotsContext from './ScreenshotsContext'
 import ScreenshotsOperations from './ScreenshotsOperations'
 import { Bounds, Emiter, History } from './types'
+import { Display } from '../electron/app'
 import useGetLoadedImage from './useGetLoadedImage'
 import zhCN, { Lang } from './zh_CN'
 
 export interface ScreenshotsProps {
   url?: string
   width: number
   height: number
+  display?: Display
   lang?: Partial<Lang>
   className?: string
   [key: string]: unknown
 }
 
-export default function Screenshots ({ url, width, height, lang, className, ...props }: ScreenshotsProps): ReactElement {
+export default function Screenshots ({ url, width, height, display, lang, className, ...props }: ScreenshotsProps): ReactElement {
   const image = useGetLoadedImage(url)
   const canvasContextRef = useRef<CanvasRenderingContext2D>(null)
   const emiterRef = useRef<Emiter>({})
@@ -35,6 +37,7 @@ export default function Screenshots ({ url, width, height, lang, className, ...p
     url,
     width,
     height,
+    display,
     image,
     lang: {
       ...zhCN,

packages/react-screenshots/src/Screenshots/longScreenshot.ts

@@ -0,0 +1,150 @@
+// Long screenshot utilities inspired by eSearch
+// This module provides simple helpers to stitch multiple screenshot
+// canvases into a single long image using OpenCV.js template matching.
+
+export type LongMode = 'y' | 'xy'
+
+let cvReady: Promise<void> | null = null
+
+/**
+ * Dynamically load OpenCV.js only when required.
+ */
+export function loadOpenCV (): Promise<void> {
+  if (!cvReady) {
+    cvReady = new Promise((resolve) => {
+      // biome-ignore lint: external library
+      const cv = require('@techstark/opencv-js') as typeof import('@techstark/opencv-js')
+      ;(cv as any).onRuntimeInitialized = () => resolve()
+    })
+  }
+  return cvReady
+}
+
+/**
+ * Match two canvases and calculate the offset of the second image
+ * relative to the first one. The algorithm clips the centre area
+ * of the second image then performs template matching against the
+ * first image to find the best alignment.
+ */
+export function matchCanvas (
+  img0: HTMLCanvasElement,
+  img1: HTMLCanvasElement,
+  mode: LongMode = 'y'
+): { dx: number, dy: number, srcDX: number, srcDY: number, clipped: HTMLCanvasElement } {
+  const cv = require('@techstark/opencv-js') as typeof import('@techstark/opencv-js')
+  const clip = (v: number) => Math.floor(Math.max(v - Math.max((v / 3) * 1, 50), 0) / 2)
+  const dw = mode === 'xy' ? clip(img1.width) : 0
+  const dh = clip(img1.height)
+
+  const clipCanvas = document.createElement('canvas')
+  clipCanvas.width = img1.width - dw * 2
+  clipCanvas.height = img1.height - dh * 2
+  clipCanvas.getContext('2d')?.drawImage(img1, -dw, -dh)
+
+  const src = cv.imread(img0)
+  const templ = cv.imread(clipCanvas)
+  const dst = new cv.Mat()
+  const mask = new cv.Mat()
+  cv.matchTemplate(src, templ, dst, cv.TM_CCOEFF, mask)
+  const result = cv.minMaxLoc(dst, mask)
+  const maxPoint = result.maxLoc
+  const dx = maxPoint.x
+  const dy = maxPoint.y
+  src.delete(); dst.delete(); mask.delete(); templ.delete()
+
+  const ndx = dx - dw
+  const ndy = dy - dh
+
+  const clip2 = document.createElement('canvas')
+  clip2.width = ndx !== 0 ? img1.width - dw : img1.width
+  clip2.height = ndy !== 0 ? img1.height - dh : img1.height
+  clip2.getContext('2d')?.drawImage(img1, ndx > 0 ? -dw : 0, ndy > 0 ? -dh : 0)
+
+  return { dx: ndx > 0 ? dx : ndx, dy: ndy > 0 ? dy : ndy, srcDX: ndx, srcDY: ndy, clipped: clip2 }
+}
+
+interface LongState {
+  img: HTMLCanvasElement | null
+  imgXY: { x: number, y: number }
+  lastImg: HTMLCanvasElement | null
+  lastXY: { x: number, y: number }
+}
+
+/**
+ * Stitch a sequence of canvases into a single canvas.
+ * Images should be provided in capture order.
+ */
+export async function stitchSequence (
+  canvases: HTMLCanvasElement[],
+  mode: LongMode = 'y'
+): Promise<HTMLCanvasElement> {
+  if (canvases.length === 0) throw new Error('no canvas to stitch')
+  await loadOpenCV()
+
+  const state: LongState = {
+    img: canvases[0],
+    imgXY: { x: 0, y: 0 },
+    lastImg: canvases[0],
+    lastXY: { x: 0, y: 0 }
+  }
+
+  for (let i = 1; i < canvases.length; i++) {
+    const match = matchCanvas(state.lastImg!, canvases[i], mode)
+    const dx = mode === 'xy' ? match.dx : 0
+    const dy = match.dy
+    state.img = putCanvas(state.img!, match.clipped, dx + state.lastXY.x, dy + state.lastXY.y, state.imgXY)
+    state.lastImg = canvases[i]
+    state.lastXY.x += mode === 'xy' ? match.srcDX : 0
+    state.lastXY.y += match.srcDY
+  }
+
+  return state.img!
+}
+
+function putCanvas (
+  base: HTMLCanvasElement,
+  img: HTMLCanvasElement,
+  x: number,
+  y: number,
+  state: { x: number, y: number } = { x: 0, y: 0 }
+): HTMLCanvasElement {
+  const newCanvas = document.createElement('canvas')
+  const ctx = newCanvas.getContext('2d') as CanvasRenderingContext2D
+
+  const srcW = base.width
+  const srcH = base.height
+  const minX = state.x
+  const minY = state.y
+  const maxX = minX + srcW
+  const maxY = minY + srcH
+
+  let srcDx = 0
+  let srcDy = 0
+
+  if (x < minX) {
+    srcDx = minX - x
+    newCanvas.width = srcDx + srcW
+    state.x -= srcDx
+  } else if (x + img.width > maxX) {
+    newCanvas.width = x + img.width - maxX + srcW
+  } else {
+    newCanvas.width = srcW
+  }
+
+  if (y < minY) {
+    srcDy = minY - y
+    newCanvas.height = srcDy + srcH
+    state.y -= srcDy
+  } else if (y + img.height > maxY) {
+    newCanvas.height = y + img.height - maxY + srcH
+  } else {
+    newCanvas.height = srcH
+  }
+
+  ctx.drawImage(base, srcDx, srcDy)
+  const nx = x - state.x
+  const ny = y - state.y
+  ctx.drawImage(img, nx, ny)
+
+  return newCanvas
+}

packages/react-screenshots/src/Screenshots/operations/Long/index.tsx

@@ -0,0 +1,106 @@
+import React, { ReactElement, useCallback, useEffect, useRef, useState } from 'react'
+import useStore from '../../hooks/useStore'
+import useCall from '../../hooks/useCall'
+import useReset from '../../hooks/useReset'
+import ScreenshotsButton from '../../ScreenshotsButton'
+import { stitchSequence } from '../../longScreenshot'
+
+function loadImage (url: string): Promise<HTMLImageElement> {
+  return new Promise((resolve) => {
+    const img = new Image()
+    img.onload = () => resolve(img)
+    img.src = url
+  })
+}
+
+export default function Long (): ReactElement | null {
+  const { bounds, width, height, lang, display } = useStore()
+  const call = useCall()
+  const reset = useReset()
+  const [running, setRunning] = useState(false)
+  const framesRef = useRef<string[]>([])
+
+  const handleAdd = useCallback((url: string) => {
+    framesRef.current.push(url)
+  }, [])
+
+  useEffect(() => {
+    if (!running) return
+    window.screenshots.on('long-add', handleAdd)
+    return () => {
+      window.screenshots.off('long-add', handleAdd)
+    }
+  }, [running, handleAdd])
+
+  useEffect(() => {
+    if (!running) return
+    const handleEnd = () => {
+      setRunning(false)
+      finish()
+    }
+    window.screenshots.on('long-end', handleEnd)
+    return () => {
+      window.screenshots.off('long-end', handleEnd)
+    }
+  }, [running, finish])
+
+  const cropImage = async (url: string): Promise<HTMLCanvasElement> => {
+    const img = await loadImage(url)
+    const canvas = document.createElement('canvas')
+    canvas.width = bounds!.width
+    canvas.height = bounds!.height
+    const ctx = canvas.getContext('2d')!
+    const rx = img.naturalWidth / width
+    const ry = img.naturalHeight / height
+    ctx.drawImage(
+      img,
+      bounds!.x * rx,
+      bounds!.y * ry,
+      bounds!.width * rx,
+      bounds!.height * ry,
+      0,
+      0,
+      bounds!.width,
+      bounds!.height,
+    )
+    return canvas
+  }
+
+  const finish = useCallback(async () => {
+    const canvases = [] as HTMLCanvasElement[]
+    for (const url of framesRef.current) {
+      canvases.push(await cropImage(url))
+    }
+    if (!canvases.length) return
+    const result = await stitchSequence(canvases)
+    result.toBlob(async (blob) => {
+      if (!blob) return
+      call('onOk', blob, bounds!)
+      reset()
+    }, 'image/png')
+  }, [call, bounds, reset])
+
+  const onClick = useCallback(() => {
+    if (!bounds || !display) return
+    if (!running) {
+      framesRef.current = []
+      window.screenshots.longStart({ bounds, display })
+      setRunning(true)
+    } else {
+      window.screenshots.longStop()
+      setRunning(false)
+      finish()
+    }
+  }, [bounds, display, running, finish])
+
+  if (!bounds) return null
+
+  return (
+    <ScreenshotsButton
+      title={lang.operation_long_title || 'Long'}
+      icon='icon-rectangle'
+      checked={running}
+      onClick={onClick}
+    />
+  )
+}

packages/react-screenshots/src/Screenshots/operations/index.ts

@@ -9,5 +9,6 @@ import Brush from './Brush'
 import Arrow from './Arrow'
 import Ellipse from './Ellipse'
 import Rectangle from './Rectangle'
+import Long from './Long'
 
-export default [Rectangle, Ellipse, Arrow, Brush, Text, Mosaic, '|', Undo, Redo, '|', Save, Cancel, Ok]
+export default [Rectangle, Ellipse, Arrow, Brush, Text, Mosaic, '|', Undo, Redo, '|', Long, Save, Cancel, Ok]