@@ -300,6 +302,86 @@ export function MediaDetailClient(props: MediaDetailClientProps): ReactElement {
);
}
+/**
+ * Derivatives panel — surfaces the asynchronously-produced artifacts
+ * the heavy-media worker writes back to the row. Currently:
+ *
+ * * HLS playlist URL (issue #52) — shown when `hls_url` is set;
+ * used by the public player to stream video over HLS instead of
+ * pulling the entire mp4.
+ * * Extracted text link (issue #60) — shown when
+ * `has_extracted_text` is true; opens the read-only extracted
+ * text view in a new tab so the operator can verify what was
+ * indexed.
+ * * Proxied / source URL (issue #187) — shown for migration rows
+ * in proxy mode; surfaces the origin URL with an "open" link so
+ * an operator can trace the row back to the source site.
+ *
+ * The panel renders nothing when none of the fields are set — most
+ * assets (images uploaded directly) have no derivatives and the
+ * panel stays out of the layout.
+ */
+function DerivativesPanel({ asset }: { asset: MediaAsset }): ReactElement | null {
+ const hasHls = Boolean(asset.hls_url);
+ const hasText = Boolean(asset.has_extracted_text);
+ const hasProxy = Boolean(asset.is_proxied && asset.source_url);
+ if (!hasHls && !hasText && !hasProxy) {
+ return null;
+ }
+ return (
+
+
+ Derivatives
+
+
+ {hasHls && (
+
+
+ )}
+ {hasText && (
+
+
+ View extracted text
+
+ )}
+ {hasProxy && (
+
+ )}
+
+
+ );
+}
+
/**
* Storage URL panel — the operator's primary "give me the link"
* surface. Shows the URL in Geist Mono inside a paper-3 sunken pill
diff --git a/apps/admin/src/app/(authenticated)/media/types.ts b/apps/admin/src/app/(authenticated)/media/types.ts
index 9fc207bf..fc7c2a2e 100644
--- a/apps/admin/src/app/(authenticated)/media/types.ts
+++ b/apps/admin/src/app/(authenticated)/media/types.ts
@@ -88,6 +88,32 @@ export interface BulkResult {
op: string;
succeeded: number;
failed?: Record
;
+
+ /**
+ * HLS playlist URL for video assets — populated by the
+ * media.video.transcode worker (#52). The video player picks HLS
+ * over the raw mp4 source when this is set.
+ */
+ hls_url?: string;
+
+ /**
+ * True when the media_text row exists for this asset. The detail
+ * page surfaces a "View extracted text" link based on this flag.
+ * Issue #60.
+ */
+ has_extracted_text?: boolean;
+
+ /**
+ * True for assets registered in proxy mode by the migration
+ * importer (#187). The grid shows a "proxied" badge so operators
+ * can tell at a glance which assets live remotely.
+ */
+ is_proxied?: boolean;
+
+ /**
+ * Origin URL for proxied assets. Empty for locally-stored assets.
+ */
+ source_url?: string;
}
/**
diff --git a/apps/api/internal/admin/media/handler.go b/apps/api/internal/admin/media/handler.go
index 97e2c4fb..3cbbcf7d 100644
--- a/apps/api/internal/admin/media/handler.go
+++ b/apps/api/internal/admin/media/handler.go
@@ -41,6 +41,17 @@ type Deps struct {
// passes the taskspec-backed adapter.
Processor ProcessEnqueuer
+ // VideoProcessor enqueues media.video.transcode for video/* mime
+ // types. Optional — nil means no transcoding is fired (the upload
+ // row is still committed and queryable; the player will fall back
+ // to the raw video src). Issue #52.
+ VideoProcessor ProcessEnqueuer
+
+ // PDFProcessor enqueues media.pdf.process for application/pdf
+ // mime types. Optional — nil means no thumbnail/text extraction
+ // is fired. Issue #60.
+ PDFProcessor ProcessEnqueuer
+
// Logger receives structured log lines. nil falls back to
// slog.Default — useful for tests; production wiring should always
// pass a service logger.
@@ -72,13 +83,15 @@ func (d Deps) validate() error {
}
type handlers struct {
- store Store
- putter ObjectPutter
- policy policy.Policy
- processor ProcessEnqueuer
- logger *slog.Logger
- now func() time.Time
- maxBytes int64
+ store Store
+ putter ObjectPutter
+ policy policy.Policy
+ processor ProcessEnqueuer
+ videoProcessor ProcessEnqueuer
+ pdfProcessor ProcessEnqueuer
+ logger *slog.Logger
+ now func() time.Time
+ maxBytes int64
}
// Mount wires the media routes onto mux under base (typically
@@ -106,13 +119,15 @@ func Mount(mux *http.ServeMux, base string, deps Deps) error {
}
h := &handlers{
- store: deps.Store,
- putter: deps.Putter,
- policy: deps.Policy,
- processor: deps.Processor,
- logger: deps.Logger,
- now: deps.Now,
- maxBytes: maxBytes,
+ store: deps.Store,
+ putter: deps.Putter,
+ policy: deps.Policy,
+ processor: deps.Processor,
+ videoProcessor: deps.VideoProcessor,
+ pdfProcessor: deps.PDFProcessor,
+ logger: deps.Logger,
+ now: deps.Now,
+ maxBytes: maxBytes,
}
base = strings.TrimRight(base, "/")
mux.Handle("POST "+base, h.gate(policy.CapMediaUpload, h.upload))
@@ -295,6 +310,30 @@ func (h *handlers) upload(w http.ResponseWriter, r *http.Request, pr policy.Prin
}
}
+ // MIME-routed pipelines. Video uploads go through the HLS
+ // transcoder (#52); PDFs go through pdftoppm + pdftotext (#60).
+ // Both follow the same "log on enqueue failure, never fail the
+ // upload" policy as the image pipeline — the row is the user-
+ // visible artifact, derivatives are a follow-up.
+ if h.videoProcessor != nil && strings.HasPrefix(strings.ToLower(asset.MimeType), "video/") {
+ if err := h.videoProcessor.Enqueue(r.Context(), asset.ID, asset.StorageKey, asset.MimeType); err != nil {
+ h.logger.WarnContext(r.Context(), "admin/media: enqueue video transcode failed",
+ slog.String("asset_id", asset.ID),
+ slog.String("storage_key", asset.StorageKey),
+ slog.Any("err", err),
+ )
+ }
+ }
+ if h.pdfProcessor != nil && strings.EqualFold(strings.TrimSpace(asset.MimeType), "application/pdf") {
+ if err := h.pdfProcessor.Enqueue(r.Context(), asset.ID, asset.StorageKey, asset.MimeType); err != nil {
+ h.logger.WarnContext(r.Context(), "admin/media: enqueue pdf process failed",
+ slog.String("asset_id", asset.ID),
+ slog.String("storage_key", asset.StorageKey),
+ slog.Any("err", err),
+ )
+ }
+ }
+
router.WriteJSON(w, http.StatusCreated, asset)
}
diff --git a/apps/api/internal/admin/media/model.go b/apps/api/internal/admin/media/model.go
index c5e885c1..99404c45 100644
--- a/apps/api/internal/admin/media/model.go
+++ b/apps/api/internal/admin/media/model.go
@@ -77,6 +77,31 @@ type Asset struct {
// not yet completed; clients should treat absence as "fall back
// to the original via PublicURL".
Variants []Variant `json:"variants,omitempty"`
+
+ // HLSURL is the public URL of the HLS playlist produced by the
+ // media.video.transcode task (#52). Empty for non-video assets
+ // and for video assets whose transcode hasn't completed yet; the
+ // public player should fall back to PublicURL when this is
+ // empty.
+ HLSURL string `json:"hls_url,omitempty"`
+
+ // HasExtractedText is true when the media_text table has a row
+ // for this asset (#60). The detail page surfaces a "View
+ // extracted text" link based on this flag; the full payload
+ // lives behind a separate endpoint to keep the list response
+ // from ballooning on long documents.
+ HasExtractedText bool `json:"has_extracted_text,omitempty"`
+
+ // IsProxied is true when the row represents a remotely-hosted
+ // asset registered in proxy mode by the migration importer
+ // (#187). The image proxy serves the bytes via SourceURL; the
+ // admin grid surfaces a "proxied" badge so an operator can tell
+ // at a glance which assets are local vs remote.
+ IsProxied bool `json:"is_proxied,omitempty"`
+
+ // SourceURL is the origin URL for proxied assets. Empty for
+ // locally-stored assets.
+ SourceURL string `json:"source_url,omitempty"`
}
// Variant is one rendition produced by packages/go/media/imageproc.
diff --git a/apps/api/internal/admin/media/processing_test.go b/apps/api/internal/admin/media/processing_test.go
index f3b5891f..b32d7331 100644
--- a/apps/api/internal/admin/media/processing_test.go
+++ b/apps/api/internal/admin/media/processing_test.go
@@ -258,3 +258,160 @@ func TestStore_SetVariantsOnDeletedRow(t *testing.T) {
t.Errorf("SetVariants on deleted row: err = %v, want ErrNotFound", err)
}
}
+
+// newMuxWithAllProcessors mirrors newMuxWithProcessor but wires three
+// independent enqueuers: the image processor (always fired), the
+// video processor (fired only on video/*), and the PDF processor
+// (fired only on application/pdf). Used to assert the MIME routing
+// of the upload handler.
+func newMuxWithAllProcessors(t *testing.T, image, video, pdf ProcessEnqueuer) (*http.ServeMux, *MemoryStore, *MemoryPutter) {
+ t.Helper()
+ var idSeq int
+ idGen := func() string {
+ idSeq++
+ return "asset-" + strings.Repeat("0", 4-len(itoa(idSeq))) + itoa(idSeq)
+ }
+ base := time.Date(2026, 1, 15, 12, 0, 0, 0, time.UTC)
+ store := NewMemoryStore(func() time.Time { return base }, idGen)
+ putter := NewMemoryPutter()
+ mux := http.NewServeMux()
+ if err := Mount(mux, "/api/v1/admin/media", Deps{
+ Store: store,
+ Putter: putter,
+ Policy: policy.NewBasicPolicy(policy.DefaultRoleCapabilities()),
+ Processor: image,
+ VideoProcessor: video,
+ PDFProcessor: pdf,
+ Now: func() time.Time { return base },
+ MaxBytes: 1024 * 1024,
+ }); err != nil {
+ t.Fatalf("Mount: %v", err)
+ }
+ return mux, store, putter
+}
+
+// mp4Bytes returns bytes that http.DetectContentType sniffs as a
+// video file. The 'ftyp' box at offset 4 with brand 'isom' is the
+// canonical MP4/ISO BMFF signature.
+func mp4Bytes() []byte {
+ // Box size (8 bytes), 'ftyp' (4 bytes), brand 'isom' (4 bytes),
+ // minor version (4 bytes), compatible brands. Total 20 bytes is
+ // enough for http.DetectContentType to recognise this as video.
+ return []byte{
+ 0x00, 0x00, 0x00, 0x18, 'f', 't', 'y', 'p',
+ 'i', 's', 'o', 'm', 0x00, 0x00, 0x00, 0x01,
+ 'i', 's', 'o', 'm', 'm', 'p', '4', '1',
+ }
+}
+
+// pdfBytes returns bytes that http.DetectContentType sniffs as a PDF
+// document.
+func pdfBytes() []byte {
+ return []byte("%PDF-1.4\n%\xE2\xE3\xCF\xD3\n1 0 obj\n<<>>\nendobj\n%%EOF\n")
+}
+
+// TestUpload_RoutesVideoToVideoProcessor confirms a video upload
+// fires the video processor and NOT the PDF processor.
+func TestUpload_RoutesVideoToVideoProcessor(t *testing.T) {
+ image := &recordingEnqueuer{}
+ video := &recordingEnqueuer{}
+ pdf := &recordingEnqueuer{}
+ mux, _, _ := newMuxWithAllProcessors(t, image, video, pdf)
+
+ body, ct := buildMultipart(t, "clip.mp4", mp4Bytes())
+ req := withAuth(httptest.NewRequest(http.MethodPost, "/api/v1/admin/media", body), authedPrincipal())
+ req.Header.Set("Content-Type", ct)
+ w := httptest.NewRecorder()
+ mux.ServeHTTP(w, req)
+ if w.Code != http.StatusCreated {
+ t.Fatalf("upload status = %d, body = %s", w.Code, w.Body.String())
+ }
+
+ if len(video.Calls()) != 1 {
+ t.Errorf("video processor calls = %d, want 1", len(video.Calls()))
+ }
+ if len(pdf.Calls()) != 0 {
+ t.Errorf("pdf processor should not fire for video upload; got %d calls", len(pdf.Calls()))
+ }
+ // Image processor (the existing media.process path) still fires
+ // regardless of MIME — the image pipeline decides internally
+ // whether to skip non-images.
+ if len(image.Calls()) != 1 {
+ t.Errorf("image processor calls = %d, want 1", len(image.Calls()))
+ }
+}
+
+// TestUpload_RoutesPDFToPDFProcessor confirms a PDF upload fires the
+// PDF processor and NOT the video processor.
+func TestUpload_RoutesPDFToPDFProcessor(t *testing.T) {
+ image := &recordingEnqueuer{}
+ video := &recordingEnqueuer{}
+ pdf := &recordingEnqueuer{}
+ mux, _, _ := newMuxWithAllProcessors(t, image, video, pdf)
+
+ body, ct := buildMultipart(t, "doc.pdf", pdfBytes())
+ req := withAuth(httptest.NewRequest(http.MethodPost, "/api/v1/admin/media", body), authedPrincipal())
+ req.Header.Set("Content-Type", ct)
+ w := httptest.NewRecorder()
+ mux.ServeHTTP(w, req)
+ if w.Code != http.StatusCreated {
+ t.Fatalf("upload status = %d, body = %s", w.Code, w.Body.String())
+ }
+
+ if len(pdf.Calls()) != 1 {
+ t.Errorf("pdf processor calls = %d, want 1", len(pdf.Calls()))
+ }
+ if len(video.Calls()) != 0 {
+ t.Errorf("video processor should not fire for PDF upload; got %d calls", len(video.Calls()))
+ }
+}
+
+// TestUpload_ImageDoesNotFireVideoOrPDF confirms an image upload only
+// hits the image processor.
+func TestUpload_ImageDoesNotFireVideoOrPDF(t *testing.T) {
+ image := &recordingEnqueuer{}
+ video := &recordingEnqueuer{}
+ pdf := &recordingEnqueuer{}
+ mux, _, _ := newMuxWithAllProcessors(t, image, video, pdf)
+
+ body, ct := buildMultipart(t, "logo.png", pngBytes())
+ req := withAuth(httptest.NewRequest(http.MethodPost, "/api/v1/admin/media", body), authedPrincipal())
+ req.Header.Set("Content-Type", ct)
+ w := httptest.NewRecorder()
+ mux.ServeHTTP(w, req)
+ if w.Code != http.StatusCreated {
+ t.Fatalf("upload status = %d", w.Code)
+ }
+
+ if len(image.Calls()) != 1 {
+ t.Errorf("image processor calls = %d, want 1", len(image.Calls()))
+ }
+ if len(video.Calls()) != 0 {
+ t.Errorf("video processor should not fire for image; got %d calls", len(video.Calls()))
+ }
+ if len(pdf.Calls()) != 0 {
+ t.Errorf("pdf processor should not fire for image; got %d calls", len(pdf.Calls()))
+ }
+}
+
+// TestUpload_VideoEnqueueErrorDoesNotFailUpload pins the same contract
+// as TestUpload_EnqueueErrorDoesNotFailUpload but for the video path:
+// a worker outage at video-enqueue time must not lose the upload.
+func TestUpload_VideoEnqueueErrorDoesNotFailUpload(t *testing.T) {
+ image := &recordingEnqueuer{}
+ video := &recordingEnqueuer{err: errors.New("queue down")}
+ mux, store, _ := newMuxWithAllProcessors(t, image, video, nil)
+
+ body, ct := buildMultipart(t, "clip.mp4", mp4Bytes())
+ req := withAuth(httptest.NewRequest(http.MethodPost, "/api/v1/admin/media", body), authedPrincipal())
+ req.Header.Set("Content-Type", ct)
+ w := httptest.NewRecorder()
+ mux.ServeHTTP(w, req)
+ if w.Code != http.StatusCreated {
+ t.Fatalf("upload should succeed even with video enqueue failure: status = %d", w.Code)
+ }
+ page, _ := store.List(context.Background(), ListFilter{})
+ if len(page.Data) != 1 {
+ t.Errorf("row not committed after video enqueue failure: rows = %d", len(page.Data))
+ }
+}
diff --git a/apps/worker/cmd/worker/main.go b/apps/worker/cmd/worker/main.go
index 534e0a29..635a6cde 100644
--- a/apps/worker/cmd/worker/main.go
+++ b/apps/worker/cmd/worker/main.go
@@ -24,6 +24,7 @@ import (
"github.com/hibiken/asynq"
+ workermedia "github.com/Singleton-Solution/GoNext/apps/worker/internal/media"
"github.com/Singleton-Solution/GoNext/packages/go/buildinfo"
"github.com/Singleton-Solution/GoNext/packages/go/config"
jobsasynq "github.com/Singleton-Solution/GoNext/packages/go/jobs/asynq"
@@ -109,7 +110,7 @@ func run(ctx context.Context) error {
return fmt.Errorf("parse REDIS_URL: %w", err)
}
- srv, _, err := jobsasynq.New(redisOpt, jobsasynq.Config{
+ srv, mux, err := jobsasynq.New(redisOpt, jobsasynq.Config{
Logger: logger,
Metrics: mreg.Prometheus(),
})
@@ -117,6 +118,20 @@ func run(ctx context.Context) error {
return fmt.Errorf("jobs/asynq: %w", err)
}
+ // Heavy-media tasks. Registered in stub mode for the boot-time
+ // skeleton — the package consults the PATH and the wired storage
+ // handles to decide between the real handler and the stub.
+ // Production wiring (when the worker grows S3 access) replaces
+ // the zero-value Deps with real Source/Sink implementations.
+ //
+ // See apps/worker/internal/media for the dispatch contract.
+ mediaTaskRegistry := taskspec.NewRegistry()
+ if _, err := workermedia.Register(mux, mediaTaskRegistry, workermedia.Deps{
+ Logger: logger,
+ }); err != nil {
+ return fmt.Errorf("worker/media: register: %w", err)
+ }
+
// Registration order (locked in by issue #112):
//
// 1. db.pool (registered first → drains last) — future
diff --git a/apps/worker/internal/media/wire.go b/apps/worker/internal/media/wire.go
new file mode 100644
index 00000000..bda044b7
--- /dev/null
+++ b/apps/worker/internal/media/wire.go
@@ -0,0 +1,223 @@
+// Package media wires the heavy-media task handlers (HLS video
+// transcoding and PDF thumbnail/text extraction) onto the worker's
+// asynq mux.
+//
+// The package lives inside apps/worker (not in packages/go) because
+// the wiring is binary-specific: it depends on the storage layer the
+// worker chooses, the logger the worker spins up, and the registry
+// the worker keeps. The handlers themselves live in packages/go/media/
+// (videoproc, pdfproc) — only the boot-time wiring lives here.
+//
+// # Skip-graceful binary checks
+//
+// Both pipelines need on-PATH binaries (ffmpeg for video; pdftoppm
+// /pdftotext for PDF). At boot the package probes the PATH and picks
+// between the production handler and a stub. The stub still
+// registers on the mux so Enqueue calls from the API don't error with
+// "unknown task"; the stub just logs and returns nil for every
+// payload. That keeps a deployment without the binaries running
+// healthy — uploads succeed, rows commit, the derivative pipeline
+// is a no-op.
+//
+// # Wiring contract
+//
+// Register accepts a Deps bag, registers the appropriate specs onto
+// the worker's taskspec registry, and dispatches them onto the
+// asynq mux. It returns a Report describing what was wired ("real"
+// vs "stub") so the worker can log the decision and operators have a
+// visible record of why a deployment isn't transcoding.
+package media
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+
+ "github.com/hibiken/asynq"
+
+ "github.com/Singleton-Solution/GoNext/packages/go/jobs/taskspec"
+ "github.com/Singleton-Solution/GoNext/packages/go/media/pdfproc"
+ "github.com/Singleton-Solution/GoNext/packages/go/media/videoproc"
+)
+
+// Deps is the dependency bag for Register. Storage handles are
+// optional in the boot-time skeleton: when nil, Register installs the
+// stub handlers (matching the case where ffmpeg/pdftoppm are missing
+// on PATH). Operators wiring a real worker pass real Source/Sink
+// implementations backed by their S3 layer.
+type Deps struct {
+ // VideoSource pulls original bytes for the transcoder. Optional;
+ // nil triggers stub registration for media.video.transcode.
+ VideoSource videoproc.Source
+
+ // VideoSink writes the HLS playlist + segments. Optional.
+ VideoSink videoproc.Sink
+
+ // VideoHLSWriter updates the media row's hls_url column.
+ // Optional.
+ VideoHLSWriter videoproc.HLSWriter
+
+ // VideoWorkDir is the scratch directory for the video pipeline.
+ // Empty falls back to os.TempDir() inside the handler.
+ VideoWorkDir string
+
+ // PDFSource pulls original PDF bytes. Optional.
+ PDFSource pdfproc.Source
+
+ // PDFSink writes the page-1 thumbnail. Optional.
+ PDFSink pdfproc.Sink
+
+ // PDFTextWriter persists the extracted text to media_text.
+ // Optional.
+ PDFTextWriter pdfproc.TextWriter
+
+ // PDFThumbnailWriter records the thumbnail storage key. Optional.
+ PDFThumbnailWriter pdfproc.ThumbnailWriter
+
+ // PDFWorkDir scratch directory. Empty falls back to os.TempDir().
+ PDFWorkDir string
+
+ // Logger receives structured log lines from the wiring layer.
+ // nil falls back to slog.Default.
+ Logger *slog.Logger
+}
+
+// Report describes what Register actually wired.
+type Report struct {
+ // VideoMode is "real" when ffmpeg was found on PATH and all
+ // dependencies were wired, "stub" otherwise.
+ VideoMode string
+
+ // PDFMode is "real" or "stub" — same semantics as VideoMode.
+ PDFMode string
+
+ // FFmpegPath is the absolute path to ffmpeg when found, "".
+ FFmpegPath string
+
+ // PDFAvailability reports which PDF binaries were found on PATH.
+ PDFAvailability pdfproc.Availability
+}
+
+// Register wires both heavy-media task handlers onto mux, falling
+// back to stubs when the host's binaries or the caller's storage
+// handles are missing.
+//
+// Logs at info for each task ("registered media.video.transcode in
+// real mode" / "...in stub mode"). The log line is the operator's
+// signal that the deployment supports HLS — a missing or stub line
+// in the boot log is the first place to look when a video upload
+// doesn't transcode.
+func Register(mux *asynq.ServeMux, reg *taskspec.Registry, deps Deps) (Report, error) {
+ if mux == nil {
+ return Report{}, fmt.Errorf("worker/media: nil asynq mux")
+ }
+ if reg == nil {
+ return Report{}, fmt.Errorf("worker/media: nil taskspec registry")
+ }
+ logger := deps.Logger
+ if logger == nil {
+ logger = slog.Default()
+ }
+
+ report := Report{}
+
+ // ──────────────────────────────────────────────────────────────
+ // Video transcoding (media.video.transcode)
+ // ──────────────────────────────────────────────────────────────
+ ffmpegPath, ffmpegOK := videoproc.IsAvailable()
+ report.FFmpegPath = ffmpegPath
+
+ videoSpec, vmode, err := buildVideoSpec(deps, ffmpegOK, logger)
+ if err != nil {
+ return report, fmt.Errorf("worker/media: build video spec: %w", err)
+ }
+ if err := reg.Register(videoSpec); err != nil {
+ return report, fmt.Errorf("worker/media: register video spec: %w", err)
+ }
+ mux.HandleFunc(videoSpec.Name, asynqAdapter(videoSpec.Handler))
+ report.VideoMode = vmode
+ logger.Info("worker/media: video task registered",
+ slog.String("task", videoSpec.Name),
+ slog.String("mode", vmode),
+ slog.String("ffmpeg_path", ffmpegPath),
+ )
+
+ // ──────────────────────────────────────────────────────────────
+ // PDF processing (media.pdf.process)
+ // ──────────────────────────────────────────────────────────────
+ pdfAvail := pdfproc.Probe()
+ report.PDFAvailability = pdfAvail
+
+ pdfSpec, pmode, err := buildPDFSpec(deps, pdfAvail, logger)
+ if err != nil {
+ return report, fmt.Errorf("worker/media: build pdf spec: %w", err)
+ }
+ if err := reg.Register(pdfSpec); err != nil {
+ return report, fmt.Errorf("worker/media: register pdf spec: %w", err)
+ }
+ mux.HandleFunc(pdfSpec.Name, asynqAdapter(pdfSpec.Handler))
+ report.PDFMode = pmode
+ logger.Info("worker/media: pdf task registered",
+ slog.String("task", pdfSpec.Name),
+ slog.String("mode", pmode),
+ slog.String("pdftoppm_path", pdfAvail.PDFToPPMPath),
+ slog.String("pdftotext_path", pdfAvail.PDFToTextPath),
+ slog.String("pdfcpu_path", pdfAvail.PDFCPUPath),
+ )
+ return report, nil
+}
+
+// buildVideoSpec picks between the real and stub video spec based on
+// ffmpeg availability and dependency wiring.
+//
+// The stub fires when EITHER ffmpeg is missing on PATH OR the storage
+// handles haven't been wired.
+func buildVideoSpec(deps Deps, ffmpegOK bool, logger *slog.Logger) (taskspec.TaskSpec, string, error) {
+ canRunReal := ffmpegOK && deps.VideoSource != nil && deps.VideoSink != nil
+ if !canRunReal {
+ spec, err := videoproc.NewStubSpec(logger)
+ return spec, "stub", err
+ }
+ spec, err := videoproc.NewSpec(videoproc.HandlerDeps{
+ Source: deps.VideoSource,
+ Sink: deps.VideoSink,
+ HLSWriter: deps.VideoHLSWriter,
+ Runner: videoproc.ExecRunner{},
+ WorkDir: deps.VideoWorkDir,
+ Logger: logger,
+ })
+ return spec, "real", err
+}
+
+// buildPDFSpec picks between the real and stub PDF spec.
+func buildPDFSpec(deps Deps, avail pdfproc.Availability, logger *slog.Logger) (taskspec.TaskSpec, string, error) {
+ canRunReal := (avail.CanRender() || avail.CanExtractText()) && deps.PDFSource != nil && deps.PDFSink != nil
+ if !canRunReal {
+ spec, err := pdfproc.NewStubSpec(logger)
+ return spec, "stub", err
+ }
+ spec, err := pdfproc.NewSpec(pdfproc.HandlerDeps{
+ Source: deps.PDFSource,
+ Sink: deps.PDFSink,
+ TextWriter: deps.PDFTextWriter,
+ ThumbnailWriter: deps.PDFThumbnailWriter,
+ Runner: pdfproc.ExecRunner{},
+ Availability: avail,
+ WorkDir: deps.PDFWorkDir,
+ Logger: logger,
+ })
+ return spec, "real", err
+}
+
+// asynqAdapter wraps a (ctx, []byte) taskspec.Handler in the asynq
+// handler signature mux.HandleFunc expects.
+//
+// We duplicate the adapter that lives in taskspec.Dispatch here
+// because the wiring layer wants per-task control: a future task
+// may need bespoke registration (priority hints, middleware) that
+// the all-or-nothing Dispatch helper doesn't offer.
+func asynqAdapter(h func(context.Context, []byte) error) func(context.Context, *asynq.Task) error {
+ return func(ctx context.Context, t *asynq.Task) error {
+ return h(ctx, t.Payload())
+ }
+}
diff --git a/apps/worker/internal/media/wire_test.go b/apps/worker/internal/media/wire_test.go
new file mode 100644
index 00000000..b5665680
--- /dev/null
+++ b/apps/worker/internal/media/wire_test.go
@@ -0,0 +1,129 @@
+package media
+
+import (
+ "context"
+ "testing"
+
+ "github.com/hibiken/asynq"
+
+ "github.com/Singleton-Solution/GoNext/packages/go/jobs/taskspec"
+ "github.com/Singleton-Solution/GoNext/packages/go/media/pdfproc"
+ "github.com/Singleton-Solution/GoNext/packages/go/media/videoproc"
+)
+
+// fakeVideoSource implements videoproc.Source. The map is intentionally
+// empty — these tests only assert registration succeeds; they don't
+// exercise the handler body.
+type fakeVideoSource struct{}
+
+func (fakeVideoSource) GetObject(ctx context.Context, key string) ([]byte, error) {
+ return nil, nil
+}
+
+type fakeVideoSink struct{}
+
+func (fakeVideoSink) PutObject(ctx context.Context, key string, body []byte, mime string) error {
+ return nil
+}
+func (fakeVideoSink) PublicURL(key string) string { return "https://x/" + key }
+
+type fakePDFSource struct{}
+
+func (fakePDFSource) GetObject(ctx context.Context, key string) ([]byte, error) {
+ return nil, nil
+}
+
+type fakePDFSink struct{}
+
+func (fakePDFSink) PutObject(ctx context.Context, key string, body []byte, mime string) error {
+ return nil
+}
+func (fakePDFSink) PublicURL(key string) string { return "https://x/" + key }
+
+// TestRegister_StubMode confirms that the boot path with no
+// dependencies wires both tasks as stubs without erroring. This is
+// the "fresh skeleton" case the worker boots into until the storage
+// wiring lands.
+func TestRegister_StubMode(t *testing.T) {
+ reg := taskspec.NewRegistry()
+ mux := asynq.NewServeMux()
+ rep, err := Register(mux, reg, Deps{})
+ if err != nil {
+ t.Fatalf("Register: %v", err)
+ }
+ if rep.VideoMode != "stub" {
+ t.Errorf("VideoMode = %q, want stub", rep.VideoMode)
+ }
+ if rep.PDFMode != "stub" {
+ t.Errorf("PDFMode = %q, want stub", rep.PDFMode)
+ }
+ // Both names should be on the registry now.
+ if _, ok := reg.Get(videoproc.TaskName); !ok {
+ t.Errorf("video task not registered")
+ }
+ if _, ok := reg.Get(pdfproc.TaskName); !ok {
+ t.Errorf("pdf task not registered")
+ }
+}
+
+// TestRegister_WithStorageMode confirms that supplying storage
+// handles selects the "real" mode — independent of whether ffmpeg
+// is actually on PATH (the test machine may not have it; the report
+// flips back to stub in that case which is also fine, but we assert
+// that providing storage does NOT itself produce an error).
+func TestRegister_WithStorageWiring(t *testing.T) {
+ reg := taskspec.NewRegistry()
+ mux := asynq.NewServeMux()
+ _, err := Register(mux, reg, Deps{
+ VideoSource: fakeVideoSource{},
+ VideoSink: fakeVideoSink{},
+ PDFSource: fakePDFSource{},
+ PDFSink: fakePDFSink{},
+ })
+ if err != nil {
+ t.Fatalf("Register: %v", err)
+ }
+}
+
+// TestRegister_NilMux rejects nil dependencies loudly.
+func TestRegister_NilMux(t *testing.T) {
+ reg := taskspec.NewRegistry()
+ if _, err := Register(nil, reg, Deps{}); err == nil {
+ t.Fatal("Register(nil mux): expected error")
+ }
+ mux := asynq.NewServeMux()
+ if _, err := Register(mux, nil, Deps{}); err == nil {
+ t.Fatal("Register(nil registry): expected error")
+ }
+}
+
+// TestRegister_Idempotency confirms a second call with the same
+// registry returns the "already registered" error.
+func TestRegister_Idempotency(t *testing.T) {
+ reg := taskspec.NewRegistry()
+ mux := asynq.NewServeMux()
+ if _, err := Register(mux, reg, Deps{}); err != nil {
+ t.Fatalf("first Register: %v", err)
+ }
+ // Use a fresh mux for the second call so asynq doesn't panic
+ // on the duplicate pattern; the registry should still reject.
+ mux2 := asynq.NewServeMux()
+ if _, err := Register(mux2, reg, Deps{}); err == nil {
+ t.Fatal("second Register: expected error from duplicate spec")
+ }
+}
+
+// TestRegister_PDFAvailabilityReported just exercises the field on
+// the report — the contents are environment-dependent.
+func TestRegister_PDFAvailabilityReported(t *testing.T) {
+ reg := taskspec.NewRegistry()
+ mux := asynq.NewServeMux()
+ rep, err := Register(mux, reg, Deps{})
+ if err != nil {
+ t.Fatalf("Register: %v", err)
+ }
+ // Just confirm the struct is populated (zero-value when nothing
+ // is on PATH is also a valid Availability state).
+ var avail pdfproc.Availability = rep.PDFAvailability
+ _ = avail
+}
diff --git a/cli/gonext/cmd/migrate/wp.go b/cli/gonext/cmd/migrate/wp.go
index ec15cbaa..69c4a0d0 100644
--- a/cli/gonext/cmd/migrate/wp.go
+++ b/cli/gonext/cmd/migrate/wp.go
@@ -28,6 +28,7 @@ Flags:
--on-conflict string Conflict policy: skip | update | fail. Default "skip".
--batch-size int Number of post rows per transaction. Default 100.
--skip-comments Do not import comment threads.
+ --media-mode string Media migration mode: '' (off, default), 'copy', or 'proxy'. See issue #187.
Environment:
DATABASE_URL Required (unless --dry-run). Postgres DSN.
@@ -54,12 +55,14 @@ func runWP(args []string, stdout, stderr io.Writer) int {
conflictFlag string
batchSizeFlag int
skipCommentsFlag bool
+ mediaModeFlag string
)
fs.StringVar(&fileFlag, "file", "", "Path to WXR XML export (required)")
fs.BoolVar(&dryFlag, "dry-run", false, "Walk the WXR but write no rows")
fs.StringVar(&conflictFlag, "on-conflict", "skip", "Conflict policy: skip | update | fail")
fs.IntVar(&batchSizeFlag, "batch-size", 100, "Posts per transaction")
fs.BoolVar(&skipCommentsFlag, "skip-comments", false, "Skip comment import")
+ fs.StringVar(&mediaModeFlag, "media-mode", "", "Media migration mode: '' (off), 'copy' (download bytes), 'proxy' (proxy via image proxy). See issue #187.")
fs.Usage = func() { fmt.Fprintln(stderr, wpUsage) }
if err := fs.Parse(args); err != nil {
@@ -81,6 +84,20 @@ func runWP(args []string, stdout, stderr io.Writer) int {
return ExitUsage
}
+ // Media migration mode is opt-in: an empty flag value disables
+ // the MediaMigrator entirely (post bodies retain their source
+ // URLs; the imported site hot-links to the source). Operators
+ // pick 'copy' or 'proxy' explicitly when they want migration.
+ // The CLI does not (yet) wire the MediaMigrator's storage
+ // backend — that requires DB + bucket configuration the wizard
+ // UI in #234 will supply.
+ if mediaModeFlag != "" {
+ if _, perr := importer.ParseMediaMode(mediaModeFlag); perr != nil {
+ fmt.Fprintf(stderr, "gonext migrate wp: %v\n", perr)
+ return ExitUsage
+ }
+ }
+
opts := importer.Options{
Dryrun: dryFlag,
OnConflict: policy,
@@ -163,6 +180,14 @@ func printReport(w io.Writer, r *importer.Report, dryrun bool) {
fmt.Fprintf(w, " posts: %d\n", r.Posts)
fmt.Fprintf(w, " attachments: %d\n", r.Attachments)
fmt.Fprintf(w, " comments: %d\n", r.Comments)
+ if r.MediaCopied > 0 || r.MediaProxied > 0 || r.MediaSkipped > 0 {
+ fmt.Fprintf(w, " media copied: %d\n", r.MediaCopied)
+ fmt.Fprintf(w, " media proxied: %d\n", r.MediaProxied)
+ fmt.Fprintf(w, " media skipped: %d\n", r.MediaSkipped)
+ if r.MediaBytesFetched > 0 {
+ fmt.Fprintf(w, " media bytes: %d\n", r.MediaBytesFetched)
+ }
+ }
fmt.Fprintf(w, " errors: %d\n", len(r.Errors))
fmt.Fprintf(w, " took: %s\n", r.Took)
}
diff --git a/migrations/000036_media_hls.down.sql b/migrations/000036_media_hls.down.sql
new file mode 100644
index 00000000..473b70bc
--- /dev/null
+++ b/migrations/000036_media_hls.down.sql
@@ -0,0 +1,8 @@
+-- 000036_media_hls.down.sql
+--
+-- Reverse of 000036_media_hls.up.sql. Drops the HLS playlist column
+-- from the media table. The bytes on disk (m3u8 + segments) are NOT
+-- swept by this migration — that's a separate purge cron concern;
+-- this only releases the row-side reference.
+
+ALTER TABLE media DROP COLUMN IF EXISTS hls_url;
diff --git a/migrations/000036_media_hls.up.sql b/migrations/000036_media_hls.up.sql
new file mode 100644
index 00000000..bb79d043
--- /dev/null
+++ b/migrations/000036_media_hls.up.sql
@@ -0,0 +1,37 @@
+-- 000036_media_hls.up.sql
+--
+-- Adds the HLS playlist URL column to the media table — backs the
+-- video transcoding pipeline (issue #52). When the worker's
+-- media.video.transcode task lands, it writes the resulting
+-- index.m3u8 URL here so the public player can pick HLS over the
+-- raw mp4 source.
+--
+-- Design notes
+--
+-- * NULLABLE on purpose. Every existing video row predates the
+-- pipeline and the playlist won't exist for them until an
+-- operator triggers a reprocess; nullable means "not yet
+-- transcoded" without a sentinel value. New uploads also flow
+-- through nullable: the row is committed at upload-time, the
+-- HLS URL fills in asynchronously once the worker finishes.
+--
+-- * TEXT (not bytea / json) because the URL is exactly what the
+-- attribute consumes; serialising or compressing
+-- it would add overhead on the read hot-path.
+--
+-- * No FK to a playlist-segment table. The HLS output is many
+-- small files (index.m3u8 + 6-second .ts segments) — tracking
+-- each one as a row would blow up the table for no read-side
+-- benefit. The bucket itself is the authoritative manifest;
+-- this column points at the playlist that knows how to walk
+-- them.
+--
+-- Depends on:
+-- * 000024_media — the media table this column belongs to.
+
+ALTER TABLE media
+ ADD COLUMN hls_url TEXT
+ CHECK (hls_url IS NULL OR length(hls_url) <= 2048);
+
+COMMENT ON COLUMN media.hls_url IS
+ 'Public URL of the HLS index.m3u8 produced by the media.video.transcode task. NULL until the worker writes it.';
diff --git a/migrations/000037_media_text.down.sql b/migrations/000037_media_text.down.sql
new file mode 100644
index 00000000..7d41c9dc
--- /dev/null
+++ b/migrations/000037_media_text.down.sql
@@ -0,0 +1,5 @@
+-- 000037_media_text.down.sql
+--
+-- Reverse of 000037_media_text.up.sql.
+
+DROP TABLE IF EXISTS media_text;
diff --git a/migrations/000037_media_text.up.sql b/migrations/000037_media_text.up.sql
new file mode 100644
index 00000000..da9a6ab5
--- /dev/null
+++ b/migrations/000037_media_text.up.sql
@@ -0,0 +1,66 @@
+-- 000037_media_text.up.sql
+--
+-- Storage for extracted full-text content of PDF (and other text-
+-- bearing) media assets. Backs issue #60: the worker's
+-- media.pdf.process task runs pdftotext on a freshly uploaded PDF
+-- and stores the result here so the admin search index can target
+-- the file's contents alongside the post bodies.
+--
+-- One row per media asset (1:1 — the media.id is the PK). The
+-- relationship is enforced by the FK; deleting the parent media row
+-- cascades through and frees the text row.
+--
+-- Design notes
+--
+-- * full_text is the verbatim extraction. We keep it (rather than
+-- just the tsvector) so a future re-indexing pass with a
+-- different language config can rebuild the tsvector from the
+-- stored text without re-running pdftotext on the original PDF.
+--
+-- * content is a generated tsvector column. Storing it lets the
+-- GIN index back the `media_text.content @@ tsquery` lookup
+-- directly — no on-read coerce, no functional-index gotchas.
+--
+-- * extracted_at is wall-clock at the moment the worker wrote the
+-- row. Used by the admin UI to surface "indexed 5 minutes ago"
+-- and by a future re-extract trigger that wants to skip rows
+-- that are already fresh.
+--
+-- Depends on:
+-- * 000024_media — parent table.
+
+CREATE TABLE media_text (
+ -- 1:1 with media. Cascade because the text is a derivative
+ -- artifact, not independent data — orphaning it would be a
+ -- pure leak.
+ media_id UUID PRIMARY KEY
+ REFERENCES media(id) ON DELETE CASCADE,
+
+ -- Raw extracted text. Capped at 16 MiB — a PDF this big is
+ -- almost certainly a scanned image stack, and the text payload
+ -- would dwarf the actual signal. Operators with larger PDFs
+ -- can disable the cap on a per-deployment basis via the
+ -- worker's PDF text size limit env var.
+ full_text TEXT NOT NULL DEFAULT ''
+ CHECK (length(full_text) <= 16 * 1024 * 1024),
+
+ -- Generated full-text-search vector. STORED so the GIN index
+ -- below covers it without a functional index dance.
+ -- to_tsvector('simple', ...) — we use the 'simple' config
+ -- because the documents are arbitrary user content where the
+ -- language is unknown; pluggable per-deployment via a future
+ -- options key.
+ content TSVECTOR GENERATED ALWAYS AS (to_tsvector('simple', full_text)) STORED,
+
+ extracted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX media_text_content_idx
+ ON media_text
+ USING GIN (content);
+
+CREATE INDEX media_text_extracted_at_idx
+ ON media_text (extracted_at DESC);
+
+COMMENT ON TABLE media_text IS
+ 'Extracted full-text content for media assets (PDF, etc). Written by the media.pdf.process worker task. See issue #60.';
diff --git a/migrations/000038_media_proxy.down.sql b/migrations/000038_media_proxy.down.sql
new file mode 100644
index 00000000..f2583603
--- /dev/null
+++ b/migrations/000038_media_proxy.down.sql
@@ -0,0 +1,12 @@
+-- 000038_media_proxy.down.sql
+--
+-- Reverse of 000038_media_proxy.up.sql. DROP COLUMN cascades to the
+-- CHECK constraint and the partial index automatically; the data
+-- migration is destructive (proxied rows lose their origin URL),
+-- which is acceptable since this is the development-reversibility
+-- path — production rollbacks go through a forward migration.
+
+ALTER TABLE media
+ DROP CONSTRAINT IF EXISTS media_proxy_url_consistent,
+ DROP COLUMN IF EXISTS source_url,
+ DROP COLUMN IF EXISTS is_proxied;
diff --git a/migrations/000038_media_proxy.up.sql b/migrations/000038_media_proxy.up.sql
new file mode 100644
index 00000000..14d61ed7
--- /dev/null
+++ b/migrations/000038_media_proxy.up.sql
@@ -0,0 +1,57 @@
+-- 000038_media_proxy.up.sql
+--
+-- Adds proxy-mode columns to the media table — backs issue #187.
+-- When a WP migration runs in proxy mode, the importer registers
+-- each remote attachment as a media row without copying the bytes;
+-- requests to the asset's URL pass through the existing image
+-- proxy (#37), which fetches from source_url on first hit and
+-- caches the response.
+--
+-- Design notes
+--
+-- * is_proxied DEFAULTs FALSE so every existing media row (which
+-- was uploaded directly) retains its current semantics. The
+-- column flips to TRUE only for rows the migrator inserts in
+-- proxy mode.
+--
+-- * source_url is NULL by default and only populated for proxied
+-- rows. The CHECK keeps the two columns coupled: a proxied row
+-- MUST have a source URL, and a non-proxied row MUST NOT (the
+-- latter half guards against an operator accidentally setting
+-- source_url and forgetting to flip the flag).
+--
+-- * The storage_key column on a proxied row is left as a synthetic
+-- placeholder (e.g. "proxy/") so the UNIQUE
+-- constraint still applies and the rest of the codebase can
+-- treat the row uniformly. The proxy handler routes off
+-- is_proxied, not the key shape.
+--
+-- Depends on:
+-- * 000024_media — parent table.
+
+ALTER TABLE media
+ ADD COLUMN is_proxied BOOLEAN NOT NULL DEFAULT FALSE,
+ ADD COLUMN source_url TEXT
+ CHECK (source_url IS NULL OR length(source_url) <= 2048);
+
+-- The two columns are coupled: proxied rows MUST carry a source_url,
+-- and non-proxied rows MUST NOT. Keeping the invariant in the schema
+-- so a buggy importer can't half-write a row.
+ALTER TABLE media
+ ADD CONSTRAINT media_proxy_url_consistent
+ CHECK (
+ (is_proxied = TRUE AND source_url IS NOT NULL) OR
+ (is_proxied = FALSE AND source_url IS NULL)
+ );
+
+-- Index proxied rows for the migration audit view ("which assets
+-- are we proxying and how many?"). Partial-on so the index stays
+-- tiny on a non-migrated deployment.
+CREATE INDEX media_is_proxied_idx
+ ON media (created_at DESC)
+ WHERE is_proxied = TRUE;
+
+COMMENT ON COLUMN media.is_proxied IS
+ 'TRUE when the media row references a remote source served via the image proxy (issue #187). FALSE for normally uploaded assets.';
+COMMENT ON COLUMN media.source_url IS
+ 'Origin URL for proxied media rows. Read-through cached by the proxy on first hit. NULL for non-proxied rows.';
diff --git a/packages/go/media/pdfproc/doc.go b/packages/go/media/pdfproc/doc.go
new file mode 100644
index 00000000..28fe19d9
--- /dev/null
+++ b/packages/go/media/pdfproc/doc.go
@@ -0,0 +1,46 @@
+// Package pdfproc is the upload-time PDF processing pipeline for the
+// GoNext media library — closes issue #60.
+//
+// # What it does
+//
+// When an operator uploads a PDF through the admin Media Library, two
+// derivative artifacts are useful:
+//
+// 1. A first-page thumbnail — the same grid that shows an image
+// preview should show a recognisable cover for documents, not a
+// generic file icon.
+// 2. Extracted full text — used by the admin search index so an
+// operator can find a document by its contents, not just by
+// filename.
+//
+// The package produces both. Page-1 thumbnail rendering goes through
+// pdftoppm (or pdfcpu when pdftoppm is missing — pdfcpu is a pure-Go
+// fallback that doesn't need poppler installed). Text extraction goes
+// through pdftotext, with a configurable byte cap on the result.
+//
+// # External-binary policy
+//
+// poppler-utils (pdftoppm, pdftotext) is the canonical Unix toolchain
+// for PDF processing. We don't reimplement it in Go because the
+// existing tools handle the long tail of malformed PDFs better than a
+// from-scratch implementation could in a release cycle, and shelling
+// adds <50ms of overhead per invocation. The trade-off is that the
+// worker container has to ship the binaries on PATH; a deployment
+// without them must degrade gracefully — the upload still succeeds,
+// the row commits, and the admin UI shows a generic-document icon
+// for the asset.
+//
+// # Skip-graceful when poppler is missing
+//
+// IsAvailable probes the PATH at worker boot. The worker's task
+// registration uses the flag: if pdftoppm/pdftotext are not present,
+// the spec is registered with a stub handler that logs at warn and
+// returns nil for every payload. Boot does NOT fail.
+//
+// # Testability
+//
+// The package uses an injectable Runner interface for both binary
+// invocations. Production wires it to os/exec; tests substitute a
+// recording fake that captures the arguments and fabricates the
+// output files. The test path never spawns a subprocess.
+package pdfproc
diff --git a/packages/go/media/pdfproc/process.go b/packages/go/media/pdfproc/process.go
new file mode 100644
index 00000000..9da3f6ae
--- /dev/null
+++ b/packages/go/media/pdfproc/process.go
@@ -0,0 +1,187 @@
+package pdfproc
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "os/exec"
+ "strings"
+)
+
+// Binary names looked up on PATH.
+const (
+ PDFToPPMBinary = "pdftoppm"
+ PDFCPUBinary = "pdfcpu"
+ PDFToTextBinary = "pdftotext"
+)
+
+// Runner is the injection seam between the package and the actual
+// subprocess invocations. A single Runner serves both pdftoppm and
+// pdftotext — the runtime arguments differentiate the call, not the
+// runner identity.
+type Runner interface {
+ // Run invokes binary with args. The returned error wraps the
+ // binary's combined output when the exit code is non-zero;
+ // callers compare against errors.Is(err, ErrBinaryMissing) to
+ // distinguish missing-tool from runtime failures.
+ Run(ctx context.Context, binary string, args []string) error
+}
+
+// ErrBinaryMissing is returned by ExecRunner when the requested
+// binary is not on PATH. The PDF task handler treats this as a
+// permanent failure (no retry) for the per-call binary; the
+// worker-boot path uses IsAvailable to register the stub spec
+// instead of trying-and-failing every job.
+var ErrBinaryMissing = errors.New("pdfproc: binary not found on PATH")
+
+// Availability reports which PDF binaries are reachable on PATH. The
+// renderer is degraded when pdftoppm is missing but pdfcpu is
+// present — the task handler picks the available one at runtime.
+type Availability struct {
+ // PDFToPPMPath is the absolute path to pdftoppm if found, "" if
+ // not.
+ PDFToPPMPath string
+
+ // PDFCPUPath is the absolute path to pdfcpu if found, "" if not.
+ // Used as the rendering fallback when pdftoppm is absent.
+ PDFCPUPath string
+
+ // PDFToTextPath is the absolute path to pdftotext if found, "".
+ PDFToTextPath string
+}
+
+// CanRender reports whether at least one of the supported rendering
+// binaries is available. The handler's thumbnail step short-circuits
+// to "no thumbnail produced" when this is false.
+func (a Availability) CanRender() bool {
+ return a.PDFToPPMPath != "" || a.PDFCPUPath != ""
+}
+
+// CanExtractText reports whether pdftotext is available.
+func (a Availability) CanExtractText() bool {
+ return a.PDFToTextPath != ""
+}
+
+// Probe checks every binary the package can use. Safe to call at boot
+// to gate task registration.
+func Probe() Availability {
+ a := Availability{}
+ if p, err := exec.LookPath(PDFToPPMBinary); err == nil {
+ a.PDFToPPMPath = p
+ }
+ if p, err := exec.LookPath(PDFCPUBinary); err == nil {
+ a.PDFCPUPath = p
+ }
+ if p, err := exec.LookPath(PDFToTextBinary); err == nil {
+ a.PDFToTextPath = p
+ }
+ return a
+}
+
+// ExecRunner is the production Runner backed by os/exec. The runner
+// captures stderr alongside stdout so a failing invocation surfaces
+// the binary's diagnostic output in the worker log.
+type ExecRunner struct{}
+
+// Run implements Runner.
+func (ExecRunner) Run(ctx context.Context, binary string, args []string) error {
+ if _, err := exec.LookPath(binary); err != nil {
+ return fmt.Errorf("%w: %v", ErrBinaryMissing, err)
+ }
+ cmd := exec.CommandContext(ctx, binary, args...)
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ const maxOut = 4 * 1024
+ snippet := string(out)
+ if len(snippet) > maxOut {
+ snippet = snippet[:maxOut] + "...(truncated)"
+ }
+ return fmt.Errorf("pdfproc: %s failed: %w (output: %s)", binary, err, strings.TrimSpace(snippet))
+ }
+ return nil
+}
+
+// RenderOptions controls the thumbnail rendering step.
+type RenderOptions struct {
+ // DPI is the resolution of the rendered page. Higher = sharper +
+ // bigger output. 150 is a good "looks good in the admin grid
+ // without bloating the bucket" default.
+ DPI int
+
+ // OutputPrefix is the filename prefix pdftoppm uses
+ // (it produces "-1.png" for page 1). Defaults to "thumb".
+ OutputPrefix string
+}
+
+// DefaultDPI is the rendering resolution when the caller passes 0.
+const DefaultDPI = 150
+
+// DefaultOutputPrefix is the filename prefix when the caller passes "".
+const DefaultOutputPrefix = "thumb"
+
+// resolved returns a copy of RenderOptions with defaults applied.
+func (o RenderOptions) resolved() RenderOptions {
+ if o.DPI <= 0 {
+ o.DPI = DefaultDPI
+ }
+ if o.OutputPrefix == "" {
+ o.OutputPrefix = DefaultOutputPrefix
+ }
+ return o
+}
+
+// BuildPDFToPPMArgs assembles the argv that produces a PNG of page 1
+// of inputPath in outputDir. The shape is a documented contract — the
+// task brief in #60 names the exact flags.
+//
+// pdftoppm -png -f 1 -l 1 -r input /
+//
+// pdftoppm appends "-1.png" to the prefix, so the resulting file is
+// "/-1.png".
+func BuildPDFToPPMArgs(inputPath, outputDir string, opts RenderOptions) []string {
+ opts = opts.resolved()
+ return []string{
+ "-png",
+ "-f", "1",
+ "-l", "1",
+ "-r", fmt.Sprintf("%d", opts.DPI),
+ inputPath,
+ outputDir + "/" + opts.OutputPrefix,
+ }
+}
+
+// BuildPDFCPUArgs assembles the fallback argv when pdftoppm is missing
+// and pdfcpu is available. pdfcpu's "extract -mode i" pulls images
+// from the document; we then pick page 1 in the handler code.
+// Documented as a fallback because pdftoppm is generally more
+// reliable; pdfcpu serves as the "works without poppler" escape
+// hatch.
+//
+// pdfcpu extract -mode image -pages 1
+func BuildPDFCPUArgs(inputPath, outputDir string) []string {
+ return []string{
+ "extract",
+ "-mode", "image",
+ "-pages", "1",
+ inputPath,
+ outputDir,
+ }
+}
+
+// BuildPDFToTextArgs assembles the argv for pdftotext. The single "-"
+// destination writes to stdout so we don't have to round-trip through
+// a temp file; the task handler captures the runner's output.
+//
+// pdftotext -enc UTF-8 -nopgbrk
+//
+// The handler passes a real file path rather than "-" so the size
+// cap can be enforced on disk (avoid loading multi-MiB into memory
+// from a stdout pipe).
+func BuildPDFToTextArgs(inputPath, outputPath string) []string {
+ return []string{
+ "-enc", "UTF-8",
+ "-nopgbrk",
+ inputPath,
+ outputPath,
+ }
+}
diff --git a/packages/go/media/pdfproc/task.go b/packages/go/media/pdfproc/task.go
new file mode 100644
index 00000000..24d5e571
--- /dev/null
+++ b/packages/go/media/pdfproc/task.go
@@ -0,0 +1,381 @@
+package pdfproc
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/Singleton-Solution/GoNext/packages/go/jobs/taskspec"
+ "github.com/Singleton-Solution/GoNext/packages/go/jsonschemautil"
+)
+
+// TaskName is the on-wire identifier for the PDF processing task.
+const TaskName = "media.pdf.process"
+
+// DefaultQueue is the queue PDF tasks land on.
+const DefaultQueue = "media"
+
+// DefaultMaxRetry caps how many times asynq will re-run a failing
+// PDF task. 2 is intentionally low — most failures (corrupt PDF,
+// encrypted file) are permanent, and pdftoppm/pdftotext are
+// deterministic.
+const DefaultMaxRetry = 2
+
+// DefaultTimeout bounds a single PDF processing invocation.
+const DefaultTimeout = 3 * time.Minute
+
+// DefaultMaxTextBytes caps the size of the extracted text. A PDF this
+// big is almost certainly a scanned image stack; the extracted text
+// for such a file is mostly OCR noise. Operators with larger PDFs
+// can raise the cap via HandlerDeps.MaxTextBytes.
+const DefaultMaxTextBytes = 4 * 1024 * 1024
+
+// Payload is the JSON shape the upload handler enqueues.
+type Payload struct {
+ AssetID string `json:"asset_id"`
+ StorageKey string `json:"storage_key"`
+ MIMEType string `json:"mime_type"`
+}
+
+var payloadSchemaRaw = []byte(`{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "type": "object",
+ "properties": {
+ "asset_id": {"type": "string", "minLength": 1},
+ "storage_key": {"type": "string", "minLength": 1},
+ "mime_type": {"type": "string"}
+ },
+ "required": ["asset_id", "storage_key"],
+ "additionalProperties": false
+}`)
+
+// Source reads original bytes from storage.
+type Source interface {
+ GetObject(ctx context.Context, key string) ([]byte, error)
+}
+
+// Sink writes derivative artifacts (the thumbnail PNG) back to
+// storage.
+type Sink interface {
+ PutObject(ctx context.Context, key string, body []byte, mimeType string) error
+ PublicURL(key string) string
+}
+
+// TextWriter persists the extracted text on the media_text table.
+type TextWriter interface {
+ // SetText stores the full extracted text for assetID. The store
+ // implementation is responsible for updating the generated
+ // tsvector column. Idempotent — re-running on the same asset
+ // replaces the previous extraction.
+ SetText(ctx context.Context, assetID, fullText string) error
+}
+
+// ThumbnailWriter persists the thumbnail storage key on the media
+// row. Optional — when nil, the handler still uploads the thumbnail
+// but the row's variants list won't reference it.
+type ThumbnailWriter interface {
+ // SetPDFThumbnail records the storage key (and PublicURL via the
+ // usual path-translation) for the page-1 thumbnail of assetID.
+ SetPDFThumbnail(ctx context.Context, assetID, thumbnailStorageKey string) error
+}
+
+// HandlerDeps is the dependency bag for NewHandler.
+type HandlerDeps struct {
+ Source Source
+ Sink Sink
+ TextWriter TextWriter
+ ThumbnailWriter ThumbnailWriter
+ Runner Runner
+
+ // Availability pre-computed at boot. Used to pick pdftoppm vs.
+ // pdfcpu, and to decide whether text extraction runs at all.
+ Availability Availability
+
+ // WorkDir for per-job scratch. Defaults to os.TempDir().
+ WorkDir string
+
+ // KeyPrefix for the thumbnail upload. Defaults to "pdf-thumbs/".
+ KeyPrefix string
+
+ // MaxTextBytes overrides DefaultMaxTextBytes. Zero falls back to
+ // the default.
+ MaxTextBytes int
+
+ // RenderOptions pass through to the thumbnail step.
+ RenderOptions RenderOptions
+
+ Logger *slog.Logger
+}
+
+// NewHandler returns a TaskSpec.Handler closure that runs the
+// pipeline end-to-end. The handler is tolerant of partial
+// availability: if pdftoppm is missing but pdftotext is present,
+// text extraction still runs and the thumbnail step is skipped.
+//
+// On a fully-unavailable host (neither rendering nor text binaries
+// present), the handler logs at warn and returns nil — the same
+// behaviour as NewStubSpec, but selected at runtime based on Probe.
+func NewHandler(deps HandlerDeps) func(context.Context, []byte) error {
+ if deps.Logger == nil {
+ deps.Logger = slog.Default()
+ }
+ if deps.KeyPrefix == "" {
+ deps.KeyPrefix = "pdf-thumbs/"
+ }
+ if deps.WorkDir == "" {
+ deps.WorkDir = os.TempDir()
+ }
+ if deps.MaxTextBytes <= 0 {
+ deps.MaxTextBytes = DefaultMaxTextBytes
+ }
+ return func(ctx context.Context, raw []byte) error {
+ var p Payload
+ if err := json.Unmarshal(raw, &p); err != nil {
+ return fmt.Errorf("pdfproc.task: parse payload: %w", err)
+ }
+ if p.StorageKey == "" || p.AssetID == "" {
+ return errors.New("pdfproc.task: storage_key and asset_id are required")
+ }
+ if p.MIMEType != "" && !IsSupportedMIME(p.MIMEType) {
+ deps.Logger.InfoContext(ctx,
+ "pdfproc.task: skipping non-PDF MIME",
+ slog.String("asset_id", p.AssetID),
+ slog.String("storage_key", p.StorageKey),
+ slog.String("mime_type", p.MIMEType),
+ )
+ return nil
+ }
+ if deps.Source == nil || deps.Sink == nil || deps.Runner == nil {
+ return errors.New("pdfproc.task: Source, Sink, and Runner must be wired")
+ }
+
+ // Skip-graceful: nothing to do if both binaries are absent.
+ if !deps.Availability.CanRender() && !deps.Availability.CanExtractText() {
+ deps.Logger.WarnContext(ctx,
+ "pdfproc.task: no PDF binaries on PATH, skipping",
+ slog.String("asset_id", p.AssetID),
+ slog.String("storage_key", p.StorageKey),
+ )
+ return nil
+ }
+
+ body, err := deps.Source.GetObject(ctx, p.StorageKey)
+ if err != nil {
+ return fmt.Errorf("pdfproc.task: fetch source %q: %w", p.StorageKey, err)
+ }
+
+ jobDir, err := os.MkdirTemp(deps.WorkDir, "pdfproc-"+sanitize(p.AssetID)+"-*")
+ if err != nil {
+ return fmt.Errorf("pdfproc.task: mkdir scratch: %w", err)
+ }
+ defer os.RemoveAll(jobDir)
+
+ inputPath := filepath.Join(jobDir, "input.pdf")
+ if err := os.WriteFile(inputPath, body, 0o600); err != nil {
+ return fmt.Errorf("pdfproc.task: write input: %w", err)
+ }
+
+ // 1. Thumbnail
+ if deps.Availability.CanRender() {
+ if err := renderThumbnail(ctx, deps, p, inputPath, jobDir); err != nil {
+ // A thumbnail failure does NOT abort the job — text
+ // extraction may still succeed and that's a useful
+ // degraded outcome. Log and continue.
+ deps.Logger.WarnContext(ctx,
+ "pdfproc.task: thumbnail render failed",
+ slog.String("asset_id", p.AssetID),
+ slog.Any("err", err),
+ )
+ }
+ }
+
+ // 2. Text extraction
+ if deps.Availability.CanExtractText() && deps.TextWriter != nil {
+ if err := extractText(ctx, deps, p, inputPath, jobDir); err != nil {
+ return fmt.Errorf("pdfproc.task: extract text: %w", err)
+ }
+ }
+
+ deps.Logger.InfoContext(ctx,
+ "pdfproc.task: processed PDF",
+ slog.String("asset_id", p.AssetID),
+ slog.String("storage_key", p.StorageKey),
+ )
+ return nil
+ }
+}
+
+// renderThumbnail runs pdftoppm (preferred) or pdfcpu (fallback) to
+// produce a PNG of page 1, uploads it to the sink, and writes the
+// storage key to the thumbnail writer if one is wired.
+func renderThumbnail(ctx context.Context, deps HandlerDeps, p Payload, inputPath, jobDir string) error {
+ thumbDir := filepath.Join(jobDir, "thumb")
+ if err := os.MkdirAll(thumbDir, 0o700); err != nil {
+ return fmt.Errorf("mkdir thumb dir: %w", err)
+ }
+ opts := deps.RenderOptions.resolved()
+
+ var producedFile string
+ if deps.Availability.PDFToPPMPath != "" {
+ args := BuildPDFToPPMArgs(inputPath, thumbDir, opts)
+ if err := deps.Runner.Run(ctx, PDFToPPMBinary, args); err != nil {
+ return fmt.Errorf("pdftoppm: %w", err)
+ }
+ // pdftoppm writes "-1.png"
+ producedFile = filepath.Join(thumbDir, opts.OutputPrefix+"-1.png")
+ } else {
+ args := BuildPDFCPUArgs(inputPath, thumbDir)
+ if err := deps.Runner.Run(ctx, PDFCPUBinary, args); err != nil {
+ return fmt.Errorf("pdfcpu: %w", err)
+ }
+ // pdfcpu writes an image file per page; pick the first one.
+ entries, err := os.ReadDir(thumbDir)
+ if err != nil {
+ return fmt.Errorf("read pdfcpu output: %w", err)
+ }
+ for _, e := range entries {
+ if !e.IsDir() {
+ producedFile = filepath.Join(thumbDir, e.Name())
+ break
+ }
+ }
+ if producedFile == "" {
+ return errors.New("pdfcpu produced no image")
+ }
+ }
+
+ thumbBytes, err := os.ReadFile(producedFile)
+ if err != nil {
+ return fmt.Errorf("read thumbnail %q: %w", producedFile, err)
+ }
+ thumbKey := deps.KeyPrefix + p.AssetID + "/thumb.png"
+ if err := deps.Sink.PutObject(ctx, thumbKey, thumbBytes, "image/png"); err != nil {
+ return fmt.Errorf("put thumbnail: %w", err)
+ }
+ if deps.ThumbnailWriter != nil {
+ if err := deps.ThumbnailWriter.SetPDFThumbnail(ctx, p.AssetID, thumbKey); err != nil {
+ return fmt.Errorf("record thumbnail key: %w", err)
+ }
+ }
+ return nil
+}
+
+// extractText runs pdftotext and persists the result via the TextWriter.
+func extractText(ctx context.Context, deps HandlerDeps, p Payload, inputPath, jobDir string) error {
+ outputPath := filepath.Join(jobDir, "text.txt")
+ args := BuildPDFToTextArgs(inputPath, outputPath)
+ if err := deps.Runner.Run(ctx, PDFToTextBinary, args); err != nil {
+ return fmt.Errorf("pdftotext: %w", err)
+ }
+ textBytes, err := os.ReadFile(outputPath)
+ if err != nil {
+ return fmt.Errorf("read extracted text: %w", err)
+ }
+ // Cap the stored payload. Truncation is preferred over rejection
+ // because a multi-hundred-page PDF that exceeds the cap still has
+ // useful first-N-MB of indexable content.
+ if len(textBytes) > deps.MaxTextBytes {
+ textBytes = textBytes[:deps.MaxTextBytes]
+ deps.Logger.InfoContext(ctx,
+ "pdfproc.task: extracted text truncated",
+ slog.String("asset_id", p.AssetID),
+ slog.Int("max_bytes", deps.MaxTextBytes),
+ )
+ }
+ if err := deps.TextWriter.SetText(ctx, p.AssetID, string(textBytes)); err != nil {
+ return fmt.Errorf("persist text: %w", err)
+ }
+ return nil
+}
+
+// NewSpec returns the TaskSpec ready to register into a Registry.
+func NewSpec(deps HandlerDeps) (taskspec.TaskSpec, error) {
+ schema, err := jsonschemautil.Compile("https://gonext.example/media-pdf-process.json", payloadSchemaRaw)
+ if err != nil {
+ return taskspec.TaskSpec{}, fmt.Errorf("pdfproc: compile payload schema: %w", err)
+ }
+ return taskspec.TaskSpec{
+ Name: TaskName,
+ Queue: DefaultQueue,
+ MaxRetry: DefaultMaxRetry,
+ Timeout: DefaultTimeout,
+ PayloadSchema: schema,
+ Handler: NewHandler(deps),
+ }, nil
+}
+
+// NewStubSpec returns a TaskSpec whose handler logs and returns nil
+// for every payload. Used at worker boot when neither pdftoppm nor
+// pdftotext is on PATH.
+func NewStubSpec(logger *slog.Logger) (taskspec.TaskSpec, error) {
+ schema, err := jsonschemautil.Compile("https://gonext.example/media-pdf-process.json", payloadSchemaRaw)
+ if err != nil {
+ return taskspec.TaskSpec{}, fmt.Errorf("pdfproc: compile payload schema: %w", err)
+ }
+ if logger == nil {
+ logger = slog.Default()
+ }
+ return taskspec.TaskSpec{
+ Name: TaskName,
+ Queue: DefaultQueue,
+ MaxRetry: DefaultMaxRetry,
+ Timeout: DefaultTimeout,
+ PayloadSchema: schema,
+ Handler: func(ctx context.Context, raw []byte) error {
+ var p Payload
+ _ = json.Unmarshal(raw, &p)
+ logger.WarnContext(ctx,
+ "pdfproc.task: PDF binaries not on PATH, skipping",
+ slog.String("asset_id", p.AssetID),
+ slog.String("storage_key", p.StorageKey),
+ )
+ return nil
+ },
+ }, nil
+}
+
+// IsSupportedMIME reports whether mime is a PDF. We do NOT extend
+// this to arbitrary application/* — the only thing pdftoppm/pdftotext
+// can do is PDF.
+func IsSupportedMIME(mime string) bool {
+ return strings.EqualFold(strings.TrimSpace(mime), "application/pdf")
+}
+
+// PayloadSchema returns the compiled schema for tests.
+func PayloadSchema() ([]byte, error) {
+ out := make([]byte, len(payloadSchemaRaw))
+ copy(out, payloadSchemaRaw)
+ return out, nil
+}
+
+// sanitize returns a filesystem-safe slug of s for use inside a
+// MkdirTemp pattern.
+func sanitize(s string) string {
+ if s == "" {
+ return "anon"
+ }
+ out := make([]byte, 0, len(s))
+ for i := 0; i < len(s); i++ {
+ c := s[i]
+ switch {
+ case c >= 'a' && c <= 'z':
+ out = append(out, c)
+ case c >= 'A' && c <= 'Z':
+ out = append(out, c+32)
+ case c >= '0' && c <= '9':
+ out = append(out, c)
+ case c == '-' || c == '_':
+ out = append(out, c)
+ }
+ }
+ if len(out) == 0 {
+ return "anon"
+ }
+ return string(out)
+}
diff --git a/packages/go/media/pdfproc/task_test.go b/packages/go/media/pdfproc/task_test.go
new file mode 100644
index 00000000..aa632f46
--- /dev/null
+++ b/packages/go/media/pdfproc/task_test.go
@@ -0,0 +1,380 @@
+package pdfproc
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+)
+
+// recordingRunner captures every Run invocation and optionally
+// fabricates output via onRun.
+type recordingRunner struct {
+ mu sync.Mutex
+ calls []recordedCall
+ err error
+ onRun func(binary string, args []string) error
+}
+
+type recordedCall struct {
+ binary string
+ args []string
+}
+
+func (r *recordingRunner) Run(ctx context.Context, binary string, args []string) error {
+ r.mu.Lock()
+ r.calls = append(r.calls, recordedCall{binary: binary, args: append([]string(nil), args...)})
+ r.mu.Unlock()
+ if r.onRun != nil {
+ if err := r.onRun(binary, args); err != nil {
+ return err
+ }
+ }
+ return r.err
+}
+
+// fakeSource is a Source backed by an in-memory map.
+type fakeSource struct{ objects map[string][]byte }
+
+func (f *fakeSource) GetObject(ctx context.Context, key string) ([]byte, error) {
+ if b, ok := f.objects[key]; ok {
+ return b, nil
+ }
+ return nil, errors.New("not found")
+}
+
+// fakeSink captures uploads.
+type fakeSink struct {
+ mu sync.Mutex
+ uploads []sinkUpload
+}
+
+type sinkUpload struct {
+ key string
+ body []byte
+ mime string
+}
+
+func (f *fakeSink) PutObject(ctx context.Context, key string, body []byte, mime string) error {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.uploads = append(f.uploads, sinkUpload{key: key, body: bytes.Clone(body), mime: mime})
+ return nil
+}
+
+func (f *fakeSink) PublicURL(key string) string { return "https://cdn.example/" + key }
+
+// fakeTextWriter captures the persisted text for assertion.
+type fakeTextWriter struct {
+ assetID string
+ text string
+ err error
+}
+
+func (f *fakeTextWriter) SetText(ctx context.Context, id, text string) error {
+ if f.err != nil {
+ return f.err
+ }
+ f.assetID = id
+ f.text = text
+ return nil
+}
+
+// fakeThumbnailWriter captures the thumbnail key.
+type fakeThumbnailWriter struct {
+ assetID string
+ key string
+}
+
+func (f *fakeThumbnailWriter) SetPDFThumbnail(ctx context.Context, id, key string) error {
+ f.assetID = id
+ f.key = key
+ return nil
+}
+
+// onRunFabricate emits the pdftoppm-style "-1.png" image and
+// the pdftotext-style "text.txt" file. The job directory is inferred
+// from the args (last positional arg for both binaries is the
+// output target).
+func onRunFabricate(t *testing.T, withText, withImage bool) func(binary string, args []string) error {
+ t.Helper()
+ return func(binary string, args []string) error {
+ switch binary {
+ case PDFToPPMBinary:
+ if !withImage {
+ return nil
+ }
+ // Last arg is "/"; append "-1.png"
+ dest := args[len(args)-1] + "-1.png"
+ return os.WriteFile(dest, []byte("FAKE_PNG"), 0o600)
+ case PDFCPUBinary:
+ if !withImage {
+ return nil
+ }
+ outDir := args[len(args)-1]
+ return os.WriteFile(filepath.Join(outDir, "page1.png"), []byte("FAKE_PNG"), 0o600)
+ case PDFToTextBinary:
+ if !withText {
+ return nil
+ }
+ // Last arg is the output path.
+ return os.WriteFile(args[len(args)-1], []byte("EXTRACTED TEXT CONTENT"), 0o600)
+ }
+ return nil
+ }
+}
+
+// TestHandler_FullyAvailable drives the handler with both binaries
+// "installed" and both outputs fabricated. Asserts both pipelines
+// fire.
+func TestHandler_FullyAvailable(t *testing.T) {
+ src := &fakeSource{objects: map[string][]byte{
+ "uploads/doc.pdf": []byte("FAKE_PDF_BYTES"),
+ }}
+ sink := &fakeSink{}
+ text := &fakeTextWriter{}
+ thumb := &fakeThumbnailWriter{}
+ runner := &recordingRunner{onRun: onRunFabricate(t, true, true)}
+
+ h := NewHandler(HandlerDeps{
+ Source: src,
+ Sink: sink,
+ TextWriter: text,
+ ThumbnailWriter: thumb,
+ Runner: runner,
+ Availability: Availability{
+ PDFToPPMPath: "/usr/bin/pdftoppm",
+ PDFToTextPath: "/usr/bin/pdftotext",
+ },
+ KeyPrefix: "pdf-thumbs/",
+ })
+
+ payload, _ := json.Marshal(Payload{
+ AssetID: "asset-001",
+ StorageKey: "uploads/doc.pdf",
+ MIMEType: "application/pdf",
+ })
+ if err := h(context.Background(), payload); err != nil {
+ t.Fatalf("handler: unexpected error: %v", err)
+ }
+
+ if len(runner.calls) != 2 {
+ t.Fatalf("runner called %d times, want 2 (pdftoppm + pdftotext)", len(runner.calls))
+ }
+ if len(sink.uploads) != 1 {
+ t.Fatalf("sink got %d uploads, want 1 (thumbnail)", len(sink.uploads))
+ }
+ if !strings.HasSuffix(sink.uploads[0].key, "/thumb.png") {
+ t.Errorf("thumbnail key = %q, want suffix /thumb.png", sink.uploads[0].key)
+ }
+ if sink.uploads[0].mime != "image/png" {
+ t.Errorf("thumbnail mime = %q, want image/png", sink.uploads[0].mime)
+ }
+ if text.assetID != "asset-001" {
+ t.Errorf("text.assetID = %q, want asset-001", text.assetID)
+ }
+ if text.text != "EXTRACTED TEXT CONTENT" {
+ t.Errorf("text.text = %q, want EXTRACTED TEXT CONTENT", text.text)
+ }
+ if thumb.assetID != "asset-001" {
+ t.Errorf("thumb.assetID = %q, want asset-001", thumb.assetID)
+ }
+}
+
+// TestHandler_PDFCPUFallback confirms that when pdftoppm is absent but
+// pdfcpu is present, the runner is invoked with the pdfcpu argv.
+func TestHandler_PDFCPUFallback(t *testing.T) {
+ src := &fakeSource{objects: map[string][]byte{
+ "uploads/doc.pdf": []byte("FAKE_PDF"),
+ }}
+ sink := &fakeSink{}
+ runner := &recordingRunner{onRun: onRunFabricate(t, false, true)}
+
+ h := NewHandler(HandlerDeps{
+ Source: src, Sink: sink, Runner: runner,
+ Availability: Availability{
+ PDFCPUPath: "/usr/bin/pdfcpu",
+ // No pdftoppm and no pdftotext
+ },
+ })
+ payload, _ := json.Marshal(Payload{
+ AssetID: "id-1",
+ StorageKey: "uploads/doc.pdf",
+ MIMEType: "application/pdf",
+ })
+ if err := h(context.Background(), payload); err != nil {
+ t.Fatalf("handler: %v", err)
+ }
+ if len(runner.calls) != 1 {
+ t.Fatalf("runner calls = %d, want 1", len(runner.calls))
+ }
+ if runner.calls[0].binary != PDFCPUBinary {
+ t.Errorf("first call binary = %q, want %q", runner.calls[0].binary, PDFCPUBinary)
+ }
+ if len(sink.uploads) != 1 {
+ t.Errorf("sink uploads = %d, want 1", len(sink.uploads))
+ }
+}
+
+// TestHandler_TextOnly confirms that when no rendering binary is
+// available but pdftotext is, only the text extraction step runs.
+func TestHandler_TextOnly(t *testing.T) {
+ src := &fakeSource{objects: map[string][]byte{
+ "uploads/doc.pdf": []byte("PDF"),
+ }}
+ sink := &fakeSink{}
+ text := &fakeTextWriter{}
+ runner := &recordingRunner{onRun: onRunFabricate(t, true, false)}
+
+ h := NewHandler(HandlerDeps{
+ Source: src, Sink: sink, Runner: runner, TextWriter: text,
+ Availability: Availability{PDFToTextPath: "/usr/bin/pdftotext"},
+ })
+ payload, _ := json.Marshal(Payload{AssetID: "x", StorageKey: "uploads/doc.pdf", MIMEType: "application/pdf"})
+ if err := h(context.Background(), payload); err != nil {
+ t.Fatalf("handler: %v", err)
+ }
+ if len(sink.uploads) != 0 {
+ t.Errorf("sink uploads = %d, want 0 (no rendering)", len(sink.uploads))
+ }
+ if text.text == "" {
+ t.Error("text writer not invoked")
+ }
+}
+
+// TestHandler_FullyUnavailable confirms a host with no binaries
+// returns nil (skip-graceful) without spawning anything or writing.
+func TestHandler_FullyUnavailable(t *testing.T) {
+ src := &fakeSource{objects: map[string][]byte{"k": []byte("X")}}
+ sink := &fakeSink{}
+ runner := &recordingRunner{}
+
+ h := NewHandler(HandlerDeps{
+ Source: src, Sink: sink, Runner: runner,
+ Availability: Availability{}, // empty: nothing on PATH
+ })
+ payload, _ := json.Marshal(Payload{AssetID: "id", StorageKey: "k", MIMEType: "application/pdf"})
+ if err := h(context.Background(), payload); err != nil {
+ t.Fatalf("handler: %v", err)
+ }
+ if len(runner.calls) != 0 {
+ t.Errorf("runner should not be called when no binaries available; got %d calls", len(runner.calls))
+ }
+ if len(sink.uploads) != 0 {
+ t.Errorf("sink should not see uploads; got %d", len(sink.uploads))
+ }
+}
+
+// TestHandler_SkipsNonPDF confirms non-PDF MIME types short-circuit.
+func TestHandler_SkipsNonPDF(t *testing.T) {
+ runner := &recordingRunner{}
+ h := NewHandler(HandlerDeps{
+ Source: &fakeSource{}, Sink: &fakeSink{}, Runner: runner,
+ Availability: Availability{PDFToPPMPath: "/x", PDFToTextPath: "/y"},
+ })
+ payload, _ := json.Marshal(Payload{AssetID: "id", StorageKey: "k", MIMEType: "image/png"})
+ if err := h(context.Background(), payload); err != nil {
+ t.Fatalf("handler: %v", err)
+ }
+ if len(runner.calls) != 0 {
+ t.Errorf("non-PDF should not invoke runner")
+ }
+}
+
+// TestHandler_TextTruncation confirms the size cap is enforced.
+func TestHandler_TextTruncation(t *testing.T) {
+ big := strings.Repeat("a", 1024)
+ src := &fakeSource{objects: map[string][]byte{"k": []byte("PDF")}}
+ sink := &fakeSink{}
+ text := &fakeTextWriter{}
+ runner := &recordingRunner{onRun: func(binary string, args []string) error {
+ if binary == PDFToTextBinary {
+ return os.WriteFile(args[len(args)-1], []byte(big), 0o600)
+ }
+ return nil
+ }}
+ h := NewHandler(HandlerDeps{
+ Source: src, Sink: sink, Runner: runner, TextWriter: text,
+ Availability: Availability{PDFToTextPath: "/x"},
+ MaxTextBytes: 100,
+ })
+ payload, _ := json.Marshal(Payload{AssetID: "id", StorageKey: "k", MIMEType: "application/pdf"})
+ if err := h(context.Background(), payload); err != nil {
+ t.Fatalf("handler: %v", err)
+ }
+ if len(text.text) != 100 {
+ t.Errorf("text length = %d, want 100 (truncated)", len(text.text))
+ }
+}
+
+// TestHandler_RejectsInvalidPayload confirms missing fields error.
+func TestHandler_RejectsInvalidPayload(t *testing.T) {
+ h := NewHandler(HandlerDeps{
+ Source: &fakeSource{}, Sink: &fakeSink{}, Runner: &recordingRunner{},
+ })
+ if err := h(context.Background(), []byte(`{}`)); err == nil {
+ t.Fatal("expected error for empty payload")
+ }
+}
+
+// TestProbe just exercises the function — it doesn't assert truth or
+// falsity (the test machine may or may not have any of the binaries).
+func TestProbe(t *testing.T) {
+ a := Probe()
+ _ = a.CanRender()
+ _ = a.CanExtractText()
+}
+
+// TestNewSpec confirms the spec compiles cleanly with the bundled
+// schema.
+func TestNewSpec(t *testing.T) {
+ spec, err := NewSpec(HandlerDeps{
+ Source: &fakeSource{}, Sink: &fakeSink{}, Runner: &recordingRunner{},
+ })
+ if err != nil {
+ t.Fatalf("NewSpec: %v", err)
+ }
+ if spec.Name != TaskName {
+ t.Errorf("spec.Name = %q, want %q", spec.Name, TaskName)
+ }
+ if spec.PayloadSchema == nil {
+ t.Error("schema is nil")
+ }
+}
+
+// TestNewStubSpec confirms the stub handler returns nil and the
+// spec validates as expected.
+func TestNewStubSpec(t *testing.T) {
+ spec, err := NewStubSpec(nil)
+ if err != nil {
+ t.Fatalf("NewStubSpec: %v", err)
+ }
+ payload, _ := json.Marshal(Payload{AssetID: "id", StorageKey: "k"})
+ if err := spec.Handler(context.Background(), payload); err != nil {
+ t.Errorf("stub handler error: %v", err)
+ }
+}
+
+// TestIsSupportedMIME pins the accepted MIME types.
+func TestIsSupportedMIME(t *testing.T) {
+ for _, tt := range []struct {
+ mime string
+ want bool
+ }{
+ {"application/pdf", true},
+ {"APPLICATION/PDF", true},
+ {" application/pdf ", true},
+ {"application/x-pdf", false},
+ {"image/jpeg", false},
+ {"", false},
+ } {
+ if got := IsSupportedMIME(tt.mime); got != tt.want {
+ t.Errorf("IsSupportedMIME(%q) = %v, want %v", tt.mime, got, tt.want)
+ }
+ }
+}
diff --git a/packages/go/media/videoproc/doc.go b/packages/go/media/videoproc/doc.go
new file mode 100644
index 00000000..74c931d8
--- /dev/null
+++ b/packages/go/media/videoproc/doc.go
@@ -0,0 +1,53 @@
+// Package videoproc is the upload-time video transcoding pipeline for
+// the GoNext media library.
+//
+// # What it does
+//
+// When an operator uploads a video file through the admin Media
+// Library, the original bytes are a one-shot, single-bitrate,
+// container-and-codec-of-the-uploader's-choosing blob. Serving that
+// blob directly works for small clips but is a poor fit for the public
+// web: every viewer pulls the entire file before they can seek to a
+// timestamp, mobile networks can't keep up with the source bitrate,
+// and the browser may not even support the upload's codec.
+//
+// videoproc closes that gap by transcoding the source into an HLS
+// (HTTP Live Streaming) playlist — an index.m3u8 manifest pointing at
+// 6-second .ts segments. The playlist is what the
+// attribute consumes; the segments are what the player streams as
+// the user scrubs. HLS is supported natively by Safari and via
+// hls.js / Media Source Extensions everywhere else.
+//
+// For v1 we ship a single 720p rendition (issue #52). A multi-bitrate
+// ladder (240p / 480p / 720p / 1080p) is a follow-up; the Transcoder
+// interface is shaped so the ladder lands as a config-driven variant
+// list rather than a code change.
+//
+// # Why ffmpeg
+//
+// ffmpeg is the de-facto open-source video toolchain. We could shell
+// to a managed transcoding service (AWS MediaConvert, Mux, etc.) but
+// that would either pin operators to a single cloud or require a
+// pluggable encoder layer this surface doesn't need yet. The price of
+// shelling to ffmpeg is a runtime dependency on the host: the worker
+// container has to ship the binary on PATH, and a deployment without
+// it must degrade gracefully (the upload still succeeds, the row
+// commits, and the player falls back to the original mp4).
+//
+// # Skip-graceful when ffmpeg is missing
+//
+// IsAvailable probes the PATH at worker boot. The worker's task
+// registration consults this flag: if ffmpeg isn't present, the spec
+// is registered with a stub handler that logs at warn and returns nil
+// for every payload. Boot does NOT fail, because that would prevent
+// the rest of the worker (email, webhooks, image processing) from
+// running on a deployment that doesn't care about video.
+//
+// # Testability
+//
+// The package uses an injectable Runner interface for the actual
+// ffmpeg invocation. Production wires it to the real exec.Command;
+// tests substitute a recording fake that captures the arguments and
+// fabricates the output directory. The test path never spawns a
+// subprocess.
+package videoproc
diff --git a/packages/go/media/videoproc/task.go b/packages/go/media/videoproc/task.go
new file mode 100644
index 00000000..4cb54c9e
--- /dev/null
+++ b/packages/go/media/videoproc/task.go
@@ -0,0 +1,391 @@
+package videoproc
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/Singleton-Solution/GoNext/packages/go/jobs/taskspec"
+ "github.com/Singleton-Solution/GoNext/packages/go/jsonschemautil"
+)
+
+// TaskName is the on-wire identifier for the video transcoding task.
+// Exported so the upload handler can pass the same constant to
+// taskspec.Enqueue, and so a future admin-UI "retranscode this asset"
+// button can target the same handler.
+const TaskName = "media.video.transcode"
+
+// DefaultQueue is the queue name the upload pipeline lands tasks on.
+// "media" is the dedicated queue for media processing; the worker
+// drains it at a lower priority than the critical queue so a backlog
+// of transcodes can't starve email or webhook delivery.
+const DefaultQueue = "media"
+
+// DefaultMaxRetry caps how many times asynq will re-run a failing
+// transcode. 2 is intentionally low — a transcode failure is almost
+// always permanent (unsupported codec, corrupt input) rather than
+// transient. The retry covers the case where ffmpeg's own download
+// of an external font fails mid-run.
+const DefaultMaxRetry = 2
+
+// DefaultTimeout bounds a single transcode invocation. 10 minutes
+// covers a multi-hundred-MB 720p source on a modest CPU; longer
+// videos either need a beefier worker or a follow-up that splits the
+// transcode into per-segment jobs.
+const DefaultTimeout = 10 * time.Minute
+
+// Payload is the JSON shape the upload handler enqueues. AssetID and
+// StorageKey both ride the wire so the worker can fetch the original
+// bytes by key and attribute the resulting playlist back to the row
+// by id.
+type Payload struct {
+ AssetID string `json:"asset_id"`
+ StorageKey string `json:"storage_key"`
+ MIMEType string `json:"mime_type"`
+}
+
+// payloadSchemaRaw is the JSON Schema validated by taskspec.Enqueue
+// before a payload reaches the queue. Kept close to the struct
+// definition so the schema and the Go type drift only when a
+// maintainer touches both files.
+var payloadSchemaRaw = []byte(`{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "type": "object",
+ "properties": {
+ "asset_id": {"type": "string", "minLength": 1},
+ "storage_key": {"type": "string", "minLength": 1},
+ "mime_type": {"type": "string"}
+ },
+ "required": ["asset_id", "storage_key"],
+ "additionalProperties": false
+}`)
+
+// Source is the read side of the storage layer the handler uses to
+// pull original bytes. Wire surface kept tiny so an in-memory test
+// fake stays cheap.
+type Source interface {
+ GetObject(ctx context.Context, key string) ([]byte, error)
+}
+
+// Sink is the write side. The handler writes the playlist + every
+// segment file the encoder produced. KeyPrefix on the handler config
+// determines where the playlist lives (e.g. "hls//").
+type Sink interface {
+ PutObject(ctx context.Context, key string, body []byte, mimeType string) error
+
+ // PublicURL returns the externally addressable URL for key. The
+ // handler stores this on media.hls_url for the player to consume.
+ PublicURL(key string) string
+}
+
+// HLSWriter is the optional hook that records "asset X now has an
+// HLS playlist at URL Y" on the media row. Nil writer is allowed —
+// the handler still produces a playlist, but the row's hls_url
+// column stays NULL and the public player will fall back to the
+// original mp4.
+type HLSWriter interface {
+ SetHLSURL(ctx context.Context, assetID, hlsURL string) error
+}
+
+// HandlerDeps is the dependency bag for NewHandler.
+type HandlerDeps struct {
+ // Source pulls original bytes. Required.
+ Source Source
+
+ // Sink writes the playlist + segment files. Required.
+ Sink Sink
+
+ // HLSWriter persists the playlist URL on the media row. Optional.
+ HLSWriter HLSWriter
+
+ // Runner spawns the actual ffmpeg subprocess. Required for the
+ // production path; tests substitute a recording fake.
+ Runner Runner
+
+ // WorkDir is a writable directory the handler uses for the
+ // per-job scratch space. The handler creates a subdirectory per
+ // invocation and removes it on completion. Empty means
+ // os.TempDir().
+ WorkDir string
+
+ // KeyPrefix is the storage-key prefix for HLS output. Defaults to
+ // "hls/" — the per-job prefix is "/".
+ KeyPrefix string
+
+ // Logger receives structured log lines. nil falls back to
+ // slog.Default.
+ Logger *slog.Logger
+
+ // Options pass through to Transcode. See TranscodeOptions.
+ Options TranscodeOptions
+}
+
+// NewHandler returns a TaskSpec.Handler closure that runs the
+// pipeline end-to-end.
+//
+// The handler:
+//
+// 1. Parses the payload.
+// 2. Skips non-video MIME types (logs and returns nil).
+// 3. Fetches the source bytes via Source.GetObject.
+// 4. Writes them to a scratch file under WorkDir.
+// 5. Invokes Transcode against an output subdirectory.
+// 6. Uploads every file in the output directory to Sink under
+// "/".
+// 7. Computes the playlist's public URL and writes it to the row
+// via HLSWriter.
+// 8. Cleans up the scratch directory.
+//
+// Errors mid-pipeline wrap with the stage so the worker log line
+// reads "videoproc.task: fetch source ...: ..." or "videoproc.task:
+// transcode ...: ...". asynq's retry path is driven by the wrapped
+// error — a missing-binary error from the runner does NOT retry.
+func NewHandler(deps HandlerDeps) func(context.Context, []byte) error {
+ if deps.Logger == nil {
+ deps.Logger = slog.Default()
+ }
+ if deps.KeyPrefix == "" {
+ deps.KeyPrefix = "hls/"
+ }
+ if deps.WorkDir == "" {
+ deps.WorkDir = os.TempDir()
+ }
+ return func(ctx context.Context, raw []byte) error {
+ var p Payload
+ if err := json.Unmarshal(raw, &p); err != nil {
+ return fmt.Errorf("videoproc.task: parse payload: %w", err)
+ }
+ if p.StorageKey == "" || p.AssetID == "" {
+ return errors.New("videoproc.task: storage_key and asset_id are required")
+ }
+ if p.MIMEType != "" && !IsSupportedMIME(p.MIMEType) {
+ deps.Logger.InfoContext(ctx,
+ "videoproc.task: skipping non-video MIME",
+ slog.String("asset_id", p.AssetID),
+ slog.String("storage_key", p.StorageKey),
+ slog.String("mime_type", p.MIMEType),
+ )
+ return nil
+ }
+ if deps.Source == nil || deps.Sink == nil || deps.Runner == nil {
+ return errors.New("videoproc.task: Source, Sink, and Runner must be wired")
+ }
+
+ body, err := deps.Source.GetObject(ctx, p.StorageKey)
+ if err != nil {
+ return fmt.Errorf("videoproc.task: fetch source %q: %w", p.StorageKey, err)
+ }
+
+ // Build the per-job scratch directory. The pattern includes
+ // the asset id so a directory listing in /tmp is diagnosable.
+ jobDir, err := os.MkdirTemp(deps.WorkDir, "videoproc-"+sanitize(p.AssetID)+"-*")
+ if err != nil {
+ return fmt.Errorf("videoproc.task: mkdir scratch: %w", err)
+ }
+ defer os.RemoveAll(jobDir)
+
+ inputPath := filepath.Join(jobDir, "input"+ext(p.StorageKey))
+ if err := os.WriteFile(inputPath, body, 0o600); err != nil {
+ return fmt.Errorf("videoproc.task: write input: %w", err)
+ }
+ outDir := filepath.Join(jobDir, "out")
+ if err := os.MkdirAll(outDir, 0o700); err != nil {
+ return fmt.Errorf("videoproc.task: mkdir out: %w", err)
+ }
+
+ if err := Transcode(ctx, deps.Runner, inputPath, outDir, deps.Options); err != nil {
+ return fmt.Errorf("videoproc.task: transcode: %w", err)
+ }
+
+ // Walk the output directory and upload everything. The
+ // playlist references segments by filename, so we just
+ // re-create the relative structure under the storage prefix.
+ entries, err := os.ReadDir(outDir)
+ if err != nil {
+ return fmt.Errorf("videoproc.task: read out dir: %w", err)
+ }
+ opts := deps.Options.resolved()
+ prefix := deps.KeyPrefix + p.AssetID + "/"
+ playlistKey := prefix + opts.PlaylistName
+ uploaded := 0
+ for _, entry := range entries {
+ if entry.IsDir() {
+ continue
+ }
+ name := entry.Name()
+ b, err := os.ReadFile(filepath.Join(outDir, name))
+ if err != nil {
+ return fmt.Errorf("videoproc.task: read output %q: %w", name, err)
+ }
+ mime := segmentMIME(name)
+ if err := deps.Sink.PutObject(ctx, prefix+name, b, mime); err != nil {
+ return fmt.Errorf("videoproc.task: put output %q: %w", name, err)
+ }
+ uploaded++
+ }
+
+ hlsURL := deps.Sink.PublicURL(playlistKey)
+ if deps.HLSWriter != nil {
+ if err := deps.HLSWriter.SetHLSURL(ctx, p.AssetID, hlsURL); err != nil {
+ return fmt.Errorf("videoproc.task: write hls url: %w", err)
+ }
+ }
+
+ deps.Logger.InfoContext(ctx,
+ "videoproc.task: transcoded asset",
+ slog.String("asset_id", p.AssetID),
+ slog.String("storage_key", p.StorageKey),
+ slog.String("hls_url", hlsURL),
+ slog.Int("files", uploaded),
+ )
+ return nil
+ }
+}
+
+// NewSpec returns the TaskSpec ready to register into a Registry.
+// Callers wiring the worker process call NewSpec at boot; callers
+// building the producer side (the admin upload path) pass TaskName to
+// taskspec.Enqueue.
+func NewSpec(deps HandlerDeps) (taskspec.TaskSpec, error) {
+ schema, err := jsonschemautil.Compile("https://gonext.example/media-video-transcode.json", payloadSchemaRaw)
+ if err != nil {
+ return taskspec.TaskSpec{}, fmt.Errorf("videoproc: compile payload schema: %w", err)
+ }
+ return taskspec.TaskSpec{
+ Name: TaskName,
+ Queue: DefaultQueue,
+ MaxRetry: DefaultMaxRetry,
+ Timeout: DefaultTimeout,
+ PayloadSchema: schema,
+ Handler: NewHandler(deps),
+ }, nil
+}
+
+// NewStubSpec returns a TaskSpec whose handler logs and returns nil
+// for every payload. Used at worker boot when ffmpeg is not on PATH:
+// registering the stub keeps Enqueue calls (from the API upload
+// handler) from erroring out with "unknown task" while the worker
+// gracefully no-ops the work.
+//
+// The stub still validates the payload schema — so a malformed
+// payload from the producer still trips the validator. We only
+// short-circuit the handler body, not the upstream contract.
+func NewStubSpec(logger *slog.Logger) (taskspec.TaskSpec, error) {
+ schema, err := jsonschemautil.Compile("https://gonext.example/media-video-transcode.json", payloadSchemaRaw)
+ if err != nil {
+ return taskspec.TaskSpec{}, fmt.Errorf("videoproc: compile payload schema: %w", err)
+ }
+ if logger == nil {
+ logger = slog.Default()
+ }
+ return taskspec.TaskSpec{
+ Name: TaskName,
+ Queue: DefaultQueue,
+ MaxRetry: DefaultMaxRetry,
+ Timeout: DefaultTimeout,
+ PayloadSchema: schema,
+ Handler: func(ctx context.Context, raw []byte) error {
+ var p Payload
+ _ = json.Unmarshal(raw, &p)
+ logger.WarnContext(ctx,
+ "videoproc.task: ffmpeg not on PATH, skipping transcode",
+ slog.String("asset_id", p.AssetID),
+ slog.String("storage_key", p.StorageKey),
+ )
+ return nil
+ },
+ }, nil
+}
+
+// IsSupportedMIME reports whether mime falls in the family the
+// transcoder accepts. The check is permissive on purpose — ffmpeg
+// can decode almost anything; we only skip non-video types up-front
+// so a stray image upload doesn't burn a worker slot on a guaranteed
+// no-op.
+func IsSupportedMIME(mime string) bool {
+ return strings.HasPrefix(strings.ToLower(mime), "video/")
+}
+
+// PayloadSchema returns the compiled schema, useful for tests that
+// want to validate a payload outside the Enqueue path.
+func PayloadSchema() ([]byte, error) {
+ out := make([]byte, len(payloadSchemaRaw))
+ copy(out, payloadSchemaRaw)
+ return out, nil
+}
+
+// segmentMIME maps an HLS output filename to its Content-Type so the
+// storage bucket records a useful header for HTTP clients.
+//
+// * index.m3u8 → application/vnd.apple.mpegurl
+// * .ts → video/mp2t
+//
+// Anything else gets octet-stream — ffmpeg can be coaxed into
+// producing oddities (init.mp4 for fMP4), but the default HLS output
+// is m3u8 + ts and we cover those.
+func segmentMIME(filename string) string {
+ switch {
+ case strings.HasSuffix(filename, ".m3u8"):
+ return "application/vnd.apple.mpegurl"
+ case strings.HasSuffix(filename, ".ts"):
+ return "video/mp2t"
+ case strings.HasSuffix(filename, ".mp4"):
+ return "video/mp4"
+ default:
+ return "application/octet-stream"
+ }
+}
+
+// ext returns a safe filename extension for the input scratch file.
+// We do not trust the storage key's extension — the on-PATH ffmpeg
+// auto-detects format from content — but giving it the right
+// extension helps with the demuxer's heuristics for unusual files.
+func ext(storageKey string) string {
+ idx := strings.LastIndex(storageKey, ".")
+ if idx < 0 || idx == len(storageKey)-1 {
+ return ".bin"
+ }
+ candidate := strings.ToLower(storageKey[idx:])
+ // Allow only ASCII alnum + dot. A storage key with a weird
+ // extension shouldn't end up as a shell metacharacter on disk.
+ for _, c := range candidate[1:] {
+ if !((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) {
+ return ".bin"
+ }
+ }
+ return candidate
+}
+
+// sanitize returns a filesystem-safe slug of s. The output is used
+// only inside a controlled MkdirTemp pattern, so we only need to
+// strip the characters that would break the pattern (path
+// separators).
+func sanitize(s string) string {
+ if s == "" {
+ return "anon"
+ }
+ out := make([]byte, 0, len(s))
+ for i := 0; i < len(s); i++ {
+ c := s[i]
+ switch {
+ case c >= 'a' && c <= 'z':
+ out = append(out, c)
+ case c >= 'A' && c <= 'Z':
+ out = append(out, c+32)
+ case c >= '0' && c <= '9':
+ out = append(out, c)
+ case c == '-' || c == '_':
+ out = append(out, c)
+ }
+ }
+ if len(out) == 0 {
+ return "anon"
+ }
+ return string(out)
+}
diff --git a/packages/go/media/videoproc/task_test.go b/packages/go/media/videoproc/task_test.go
new file mode 100644
index 00000000..3b04ac27
--- /dev/null
+++ b/packages/go/media/videoproc/task_test.go
@@ -0,0 +1,257 @@
+package videoproc
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+)
+
+// fakeSource is the test-only Source: a map of storage-key to bytes.
+type fakeSource struct {
+ objects map[string][]byte
+ err error
+}
+
+func (f *fakeSource) GetObject(ctx context.Context, key string) ([]byte, error) {
+ if f.err != nil {
+ return nil, f.err
+ }
+ b, ok := f.objects[key]
+ if !ok {
+ return nil, errors.New("not found: " + key)
+ }
+ return b, nil
+}
+
+// fakeSink records uploads in-order and answers PublicURL via a
+// caller-supplied template.
+type fakeSink struct {
+ mu sync.Mutex
+ uploads []sinkUpload
+ baseURL string
+ putErr error
+}
+
+type sinkUpload struct {
+ key string
+ body []byte
+ mimeType string
+}
+
+func (f *fakeSink) PutObject(ctx context.Context, key string, body []byte, mimeType string) error {
+ if f.putErr != nil {
+ return f.putErr
+ }
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.uploads = append(f.uploads, sinkUpload{key: key, body: bytes.Clone(body), mimeType: mimeType})
+ return nil
+}
+
+func (f *fakeSink) PublicURL(key string) string {
+ if f.baseURL == "" {
+ return "https://cdn.example/" + key
+ }
+ return f.baseURL + "/" + key
+}
+
+// fakeHLSWriter captures the (assetID, hlsURL) pair so tests can
+// assert the row update happened.
+type fakeHLSWriter struct {
+ assetID string
+ hlsURL string
+ err error
+}
+
+func (f *fakeHLSWriter) SetHLSURL(ctx context.Context, assetID, hlsURL string) error {
+ if f.err != nil {
+ return f.err
+ }
+ f.assetID = assetID
+ f.hlsURL = hlsURL
+ return nil
+}
+
+// onRunFabricate is a helper that synthesises the ffmpeg output
+// directory (the playlist + a single segment) so the handler's
+// upload step has something to walk. The output directory is the
+// last positional argument's directory.
+func onRunFabricate(t *testing.T) func(args []string, workingDir string) error {
+ t.Helper()
+ return func(args []string, workingDir string) error {
+ // The last argument is the playlist destination
+ // "/index.m3u8". The output directory is its parent.
+ dest := args[len(args)-1]
+ outDir := filepath.Dir(dest)
+ if err := os.WriteFile(dest, []byte("#EXTM3U\n#EXT-X-VERSION:3\n#EXTINF:6.0,\nsegment0.ts\n#EXT-X-ENDLIST\n"), 0o600); err != nil {
+ return err
+ }
+ if err := os.WriteFile(filepath.Join(outDir, "segment0.ts"), []byte("FAKE_TS_BYTES"), 0o600); err != nil {
+ return err
+ }
+ return nil
+ }
+}
+
+// TestHandler_HappyPath drives the handler end-to-end with a faked
+// runner that fabricates the output files. Asserts:
+//
+// * the runner is invoked with an argv containing the right flags
+// * the playlist and the segment both land in the sink
+// * the HLS URL is written to the row writer
+// * the scratch directory is cleaned up
+func TestHandler_HappyPath(t *testing.T) {
+ src := &fakeSource{objects: map[string][]byte{
+ "uploads/2026/05/abc.mp4": []byte("FAKE_MP4_BYTES"),
+ }}
+ sink := &fakeSink{}
+ writer := &fakeHLSWriter{}
+ runner := &recordingRunner{onRun: onRunFabricate(t)}
+
+ h := NewHandler(HandlerDeps{
+ Source: src,
+ Sink: sink,
+ HLSWriter: writer,
+ Runner: runner,
+ KeyPrefix: "hls/",
+ })
+
+ payload, _ := json.Marshal(Payload{
+ AssetID: "media-id-001",
+ StorageKey: "uploads/2026/05/abc.mp4",
+ MIMEType: "video/mp4",
+ })
+ if err := h(context.Background(), payload); err != nil {
+ t.Fatalf("handler: unexpected error: %v", err)
+ }
+
+ if len(runner.calls) != 1 {
+ t.Fatalf("runner called %d times, want 1", len(runner.calls))
+ }
+ // Confirm the argv mentions HLS — load-bearing
+ joined := strings.Join(runner.calls[0].args, " ")
+ if !strings.Contains(joined, "-f hls") {
+ t.Errorf("runner argv missing -f hls: %s", joined)
+ }
+
+ // Two uploads: playlist + segment
+ if len(sink.uploads) != 2 {
+ t.Fatalf("sink got %d uploads, want 2", len(sink.uploads))
+ }
+ gotKeys := map[string]string{}
+ for _, u := range sink.uploads {
+ gotKeys[u.key] = u.mimeType
+ }
+ wantKey := "hls/media-id-001/index.m3u8"
+ if mt, ok := gotKeys[wantKey]; !ok {
+ t.Errorf("missing playlist upload at %s; got keys: %v", wantKey, keys(gotKeys))
+ } else if mt != "application/vnd.apple.mpegurl" {
+ t.Errorf("playlist mime = %q, want application/vnd.apple.mpegurl", mt)
+ }
+ wantSeg := "hls/media-id-001/segment0.ts"
+ if mt, ok := gotKeys[wantSeg]; !ok {
+ t.Errorf("missing segment upload at %s; got keys: %v", wantSeg, keys(gotKeys))
+ } else if mt != "video/mp2t" {
+ t.Errorf("segment mime = %q, want video/mp2t", mt)
+ }
+
+ if writer.assetID != "media-id-001" {
+ t.Errorf("writer.assetID = %q, want media-id-001", writer.assetID)
+ }
+ if !strings.HasSuffix(writer.hlsURL, "/hls/media-id-001/index.m3u8") {
+ t.Errorf("writer.hlsURL = %q, want suffix /hls/media-id-001/index.m3u8", writer.hlsURL)
+ }
+}
+
+// TestHandler_SkipsNonVideoMIME confirms the early-return path for
+// non-video uploads — the handler must not spawn ffmpeg or hit
+// storage.
+func TestHandler_SkipsNonVideoMIME(t *testing.T) {
+ src := &fakeSource{}
+ sink := &fakeSink{}
+ runner := &recordingRunner{}
+
+ h := NewHandler(HandlerDeps{
+ Source: src, Sink: sink, Runner: runner,
+ })
+
+ payload, _ := json.Marshal(Payload{
+ AssetID: "id",
+ StorageKey: "k",
+ MIMEType: "image/jpeg",
+ })
+ if err := h(context.Background(), payload); err != nil {
+ t.Fatalf("handler: unexpected error: %v", err)
+ }
+ if len(runner.calls) != 0 {
+ t.Errorf("non-video upload should not invoke runner; got %d calls", len(runner.calls))
+ }
+ if len(sink.uploads) != 0 {
+ t.Errorf("non-video upload should not write to sink; got %d uploads", len(sink.uploads))
+ }
+}
+
+// TestHandler_RejectsInvalidPayload confirms missing required fields
+// surface as an error.
+func TestHandler_RejectsInvalidPayload(t *testing.T) {
+ h := NewHandler(HandlerDeps{
+ Source: &fakeSource{}, Sink: &fakeSink{}, Runner: &recordingRunner{},
+ })
+ if err := h(context.Background(), []byte(`{}`)); err == nil {
+ t.Fatal("handler: expected error for empty payload")
+ }
+ if err := h(context.Background(), []byte(`not json`)); err == nil {
+ t.Fatal("handler: expected error for malformed JSON")
+ }
+}
+
+// TestNewSpec_RegisteredShape exercises NewSpec end-to-end via the
+// real schema compile. A failure here means the schema is malformed.
+func TestNewSpec_RegisteredShape(t *testing.T) {
+ spec, err := NewSpec(HandlerDeps{
+ Source: &fakeSource{}, Sink: &fakeSink{}, Runner: &recordingRunner{},
+ })
+ if err != nil {
+ t.Fatalf("NewSpec: %v", err)
+ }
+ if spec.Name != TaskName {
+ t.Errorf("spec.Name = %q, want %q", spec.Name, TaskName)
+ }
+ if spec.Queue != DefaultQueue {
+ t.Errorf("spec.Queue = %q, want %q", spec.Queue, DefaultQueue)
+ }
+ if spec.PayloadSchema == nil {
+ t.Error("spec.PayloadSchema is nil")
+ }
+}
+
+// TestNewStubSpec confirms the stub variant compiles and the handler
+// returns nil on every input without touching storage.
+func TestNewStubSpec(t *testing.T) {
+ spec, err := NewStubSpec(nil)
+ if err != nil {
+ t.Fatalf("NewStubSpec: %v", err)
+ }
+ if spec.Name != TaskName {
+ t.Errorf("stub spec.Name = %q, want %q", spec.Name, TaskName)
+ }
+ payload, _ := json.Marshal(Payload{AssetID: "x", StorageKey: "k"})
+ if err := spec.Handler(context.Background(), payload); err != nil {
+ t.Errorf("stub handler returned error: %v", err)
+ }
+}
+
+// keys returns a stable-ordered slice of map keys for error messages.
+func keys(m map[string]string) []string {
+ out := make([]string, 0, len(m))
+ for k := range m {
+ out = append(out, k)
+ }
+ return out
+}
diff --git a/packages/go/media/videoproc/transcode.go b/packages/go/media/videoproc/transcode.go
new file mode 100644
index 00000000..d7edc1ce
--- /dev/null
+++ b/packages/go/media/videoproc/transcode.go
@@ -0,0 +1,225 @@
+package videoproc
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "os/exec"
+ "strings"
+)
+
+// FFmpegBinary is the on-PATH name of the binary the package looks up.
+// Exported so an operator-side override (env var, config) can pin a
+// vendored copy.
+const FFmpegBinary = "ffmpeg"
+
+// Runner is the injection seam between the package and the actual
+// ffmpeg subprocess. The wire surface is intentionally tiny — a single
+// command line plus a working directory — so a test fake can record
+// invocations without rebuilding any complex state.
+//
+// Production wiring uses ExecRunner, which delegates to os/exec.
+// Tests use a recording fake that captures argv, optionally writes
+// fabricated output files to the destination directory, and returns a
+// caller-supplied error.
+type Runner interface {
+ // Run invokes the binary with args. WorkingDir is optional; an
+ // empty value means "inherit from the parent process". The
+ // returned error wraps the binary's stderr when the exit code is
+ // non-zero; callers compare against errors.Is(err, ErrBinaryMissing)
+ // to distinguish "ffmpeg isn't installed" from "ffmpeg ran but
+ // failed". Callers MUST honour ctx for cancellation — a long-
+ // running transcode that exceeds the task timeout has to be
+ // killed cleanly so asynq's retry path sees a cancelled error
+ // rather than a hung worker.
+ Run(ctx context.Context, args []string, workingDir string) error
+}
+
+// ErrBinaryMissing is returned by Runner.Run when the configured binary
+// is not on PATH. The transcode task handler treats this as a permanent
+// failure (no retry): a missing binary is a deployment fact, not a
+// transient hiccup.
+var ErrBinaryMissing = errors.New("videoproc: ffmpeg binary not found on PATH")
+
+// IsAvailable reports whether ffmpeg is reachable on PATH. The check
+// is a single exec.LookPath; safe to call at boot to gate task
+// registration. Returns the empty string for the path component when
+// the binary is not found.
+//
+// Callers should log a clear warning when this returns false so an
+// operator can decide whether the missing binary is intentional
+// (video transcoding not wanted on this deployment) or a config bug.
+func IsAvailable() (string, bool) {
+ p, err := exec.LookPath(FFmpegBinary)
+ if err != nil {
+ return "", false
+ }
+ return p, true
+}
+
+// ExecRunner is the production Runner backed by os/exec. The zero
+// value is ready to use; Binary defaults to FFmpegBinary.
+//
+// The runner captures stderr into the returned error so a failing
+// transcode surfaces ffmpeg's diagnostic output in the worker log,
+// not an opaque "exit status 1".
+type ExecRunner struct {
+ // Binary overrides FFmpegBinary. Empty means "use the default".
+ Binary string
+}
+
+// Run implements Runner.
+func (e ExecRunner) Run(ctx context.Context, args []string, workingDir string) error {
+ bin := e.Binary
+ if bin == "" {
+ bin = FFmpegBinary
+ }
+ if _, err := exec.LookPath(bin); err != nil {
+ return fmt.Errorf("%w: %v", ErrBinaryMissing, err)
+ }
+ cmd := exec.CommandContext(ctx, bin, args...)
+ if workingDir != "" {
+ cmd.Dir = workingDir
+ }
+ // Combined output captures the encoder's stderr (where ffmpeg
+ // writes its progress + errors) alongside stdout so the failure
+ // log lines are informative.
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ // Trim to a reasonable cap so a runaway encoder doesn't write
+ // megabytes into the error log. The first 4 KiB of ffmpeg's
+ // banner is enough to diagnose any real failure.
+ const maxOut = 4 * 1024
+ snippet := string(out)
+ if len(snippet) > maxOut {
+ snippet = snippet[:maxOut] + "...(truncated)"
+ }
+ return fmt.Errorf("videoproc: ffmpeg failed: %w (output: %s)", err, strings.TrimSpace(snippet))
+ }
+ return nil
+}
+
+// TranscodeOptions controls a single Transcode call. The zero value
+// is acceptable; defaults are applied by Transcode.
+type TranscodeOptions struct {
+ // SegmentSeconds is the HLS segment duration. 6s is the
+ // reference value from the HLS spec — small enough that a slow
+ // connection can start playback quickly, large enough that the
+ // per-segment overhead (a TS file header, a manifest entry) is
+ // amortised. Zero falls back to DefaultSegmentSeconds.
+ SegmentSeconds int
+
+ // Height is the target rendition height in pixels. The width is
+ // computed from the source aspect ratio (the -vf scale filter
+ // applies "trunc(oh*a/2)*2:720" to preserve the aspect and keep
+ // the width even — H.264 requires even dimensions). Zero falls
+ // back to DefaultHeight (720).
+ Height int
+
+ // PlaylistName is the manifest filename. Defaults to "index.m3u8";
+ // override only if the public URL routing wants a stable
+ // per-media filename that includes the id.
+ PlaylistName string
+}
+
+// DefaultSegmentSeconds is the HLS segment duration used when the
+// caller passes the zero value. Matches the HLS spec's reference.
+const DefaultSegmentSeconds = 6
+
+// DefaultHeight is the target rendition height in pixels for the v1
+// single-rendition transcode.
+const DefaultHeight = 720
+
+// DefaultPlaylistName is the HLS manifest filename.
+const DefaultPlaylistName = "index.m3u8"
+
+// Transcode invokes the configured Runner to produce an HLS playlist
+// at outputDir/PlaylistName from the source file at inputPath. The
+// runner is responsible for spawning the actual subprocess; this
+// function only assembles the argv and forwards to runner.Run.
+//
+// The argv is deterministic for a given options value — important for
+// tests that assert on the exact flags, and for a future "redo this
+// transcode with the same params" reprocess endpoint.
+//
+// Errors from the runner are wrapped with the input path so a
+// pipeline log line is self-describing without the caller having to
+// add it.
+func Transcode(ctx context.Context, runner Runner, inputPath, outputDir string, opts TranscodeOptions) error {
+ if runner == nil {
+ return errors.New("videoproc: nil runner")
+ }
+ if inputPath == "" {
+ return errors.New("videoproc: empty input path")
+ }
+ if outputDir == "" {
+ return errors.New("videoproc: empty output directory")
+ }
+ opts = opts.resolved()
+ args := BuildArgs(inputPath, outputDir, opts)
+ if err := runner.Run(ctx, args, ""); err != nil {
+ return fmt.Errorf("videoproc: transcode %q: %w", inputPath, err)
+ }
+ return nil
+}
+
+// resolved returns a copy of TranscodeOptions with defaults applied.
+func (o TranscodeOptions) resolved() TranscodeOptions {
+ if o.SegmentSeconds <= 0 {
+ o.SegmentSeconds = DefaultSegmentSeconds
+ }
+ if o.Height <= 0 {
+ o.Height = DefaultHeight
+ }
+ if o.PlaylistName == "" {
+ o.PlaylistName = DefaultPlaylistName
+ }
+ return o
+}
+
+// BuildArgs assembles the ffmpeg argv for a single HLS transcode. The
+// function is exported so tests can assert on the exact flags — the
+// shape is a documented contract, not an implementation detail.
+//
+// The argv was chosen to match the task brief in #52:
+//
+// ffmpeg -i
+// -c:v libx264 -preset veryfast -crf 23
+// -c:a aac -b:a 128k
+// -hls_time
+// -hls_playlist_type vod
+// -f hls
+// /
+//
+// Notes:
+//
+// * "-vf scale=-2:H" preserves aspect and forces an even width.
+// H.264 requires both dimensions even; the trunc(...)*2 trick is
+// replaced here by ffmpeg's own -2 marker which does the same.
+// * "-preset veryfast" trades a bit of bitrate efficiency for a
+// large reduction in CPU time. Worker CPU is the bottleneck on
+// a busy site; the storage-side cost of a slightly larger
+// segment is much cheaper than the latency cost of "slow".
+// * "-crf 23" is the H.264 quality knob; 23 is the ffmpeg default
+// and produces a perceptually transparent rendition for typical
+// 720p web content.
+// * "-hls_playlist_type vod" tags the playlist as VOD (not LIVE)
+// so players cache aggressively and do not poll for updates.
+func BuildArgs(inputPath, outputDir string, opts TranscodeOptions) []string {
+ opts = opts.resolved()
+ return []string{
+ "-y", // overwrite the destination playlist on rerun
+ "-i", inputPath,
+ "-vf", fmt.Sprintf("scale=-2:%d", opts.Height),
+ "-c:v", "libx264",
+ "-preset", "veryfast",
+ "-crf", "23",
+ "-c:a", "aac",
+ "-b:a", "128k",
+ "-hls_time", fmt.Sprintf("%d", opts.SegmentSeconds),
+ "-hls_playlist_type", "vod",
+ "-f", "hls",
+ outputDir + "/" + opts.PlaylistName,
+ }
+}
diff --git a/packages/go/media/videoproc/transcode_test.go b/packages/go/media/videoproc/transcode_test.go
new file mode 100644
index 00000000..9681be75
--- /dev/null
+++ b/packages/go/media/videoproc/transcode_test.go
@@ -0,0 +1,186 @@
+package videoproc
+
+import (
+ "context"
+ "errors"
+ "strings"
+ "testing"
+)
+
+// recordingRunner is the test-only Runner implementation. It captures
+// the argv of every Run call (so assertions can inspect the exact
+// flags ffmpeg would have received) and returns a caller-supplied
+// error so the "ffmpeg failed mid-encode" path can be exercised
+// without a subprocess.
+type recordingRunner struct {
+ calls []recordedCall
+ err error
+ // onRun, when set, is invoked before the runner returns its
+ // caller-supplied error. Tests use it to fabricate the output
+ // directory contents so the handler's upload step has something
+ // to walk.
+ onRun func(args []string, workingDir string) error
+}
+
+type recordedCall struct {
+ args []string
+ workingDir string
+}
+
+func (r *recordingRunner) Run(ctx context.Context, args []string, workingDir string) error {
+ cp := append([]string(nil), args...)
+ r.calls = append(r.calls, recordedCall{args: cp, workingDir: workingDir})
+ if r.onRun != nil {
+ if err := r.onRun(args, workingDir); err != nil {
+ return err
+ }
+ }
+ return r.err
+}
+
+// TestBuildArgs_DefaultShape pins the argv the transcoder hands to
+// ffmpeg. The shape is a contract (the task brief in #52 names the
+// flags); a maintainer who changes it must also bump the test so the
+// change is intentional rather than accidental.
+func TestBuildArgs_DefaultShape(t *testing.T) {
+ args := BuildArgs("/in/source.mp4", "/out", TranscodeOptions{})
+
+ // We don't require an exact slice match because the position of
+ // individual flags is allowed to evolve; we DO require the
+ // load-bearing flags to be present and well-formed.
+ want := []string{
+ "-i", "/in/source.mp4",
+ "-vf", "scale=-2:720",
+ "-c:v", "libx264",
+ "-c:a", "aac",
+ "-hls_time", "6",
+ "-hls_playlist_type", "vod",
+ "-f", "hls",
+ }
+ joined := strings.Join(args, " ")
+ for i := 0; i < len(want); i += 2 {
+ pair := want[i] + " " + want[i+1]
+ if !strings.Contains(joined, pair) {
+ t.Errorf("BuildArgs missing pair %q in %q", pair, joined)
+ }
+ }
+ // The last argument is the playlist destination — load-bearing.
+ if !strings.HasSuffix(args[len(args)-1], "/index.m3u8") {
+ t.Errorf("BuildArgs: last arg = %q, want suffix /index.m3u8", args[len(args)-1])
+ }
+}
+
+// TestBuildArgs_CustomOptions confirms the option overrides flow into
+// the argv. The test also indirectly verifies the resolved() defaults
+// only fire when the caller passes the zero value.
+func TestBuildArgs_CustomOptions(t *testing.T) {
+ args := BuildArgs("/in.mp4", "/out", TranscodeOptions{
+ SegmentSeconds: 4,
+ Height: 1080,
+ PlaylistName: "stream.m3u8",
+ })
+ joined := strings.Join(args, " ")
+ if !strings.Contains(joined, "scale=-2:1080") {
+ t.Errorf("height override missing: %q", joined)
+ }
+ if !strings.Contains(joined, "-hls_time 4") {
+ t.Errorf("segment override missing: %q", joined)
+ }
+ if !strings.HasSuffix(args[len(args)-1], "/stream.m3u8") {
+ t.Errorf("playlist override missing: %q", args[len(args)-1])
+ }
+}
+
+// TestTranscode_InvokesRunner is the happy-path: the runner is called
+// with the expected argv shape and no error bubbles up.
+func TestTranscode_InvokesRunner(t *testing.T) {
+ r := &recordingRunner{}
+ err := Transcode(context.Background(), r, "/in.mp4", "/out", TranscodeOptions{})
+ if err != nil {
+ t.Fatalf("Transcode: unexpected error: %v", err)
+ }
+ if len(r.calls) != 1 {
+ t.Fatalf("recordingRunner: got %d calls, want 1", len(r.calls))
+ }
+ if r.calls[0].args[0] != "-y" {
+ t.Errorf("expected first arg to be -y (overwrite), got %q", r.calls[0].args[0])
+ }
+}
+
+// TestTranscode_RunnerErrorPropagates confirms a non-nil error from
+// the runner surfaces with the input path in the wrapped chain so
+// log lines are diagnosable.
+func TestTranscode_RunnerErrorPropagates(t *testing.T) {
+ sentinel := errors.New("encoder crashed")
+ r := &recordingRunner{err: sentinel}
+ err := Transcode(context.Background(), r, "/in.mp4", "/out", TranscodeOptions{})
+ if err == nil {
+ t.Fatal("Transcode: expected error, got nil")
+ }
+ if !errors.Is(err, sentinel) {
+ t.Errorf("Transcode: error not wrapping sentinel, got %v", err)
+ }
+ if !strings.Contains(err.Error(), "/in.mp4") {
+ t.Errorf("Transcode error should mention input path: %v", err)
+ }
+}
+
+// TestTranscode_NilRunnerRejected ensures the package can't be called
+// without an injected runner — production wires ExecRunner; a nil
+// here is almost certainly a wiring bug we want to surface loudly.
+func TestTranscode_NilRunnerRejected(t *testing.T) {
+ err := Transcode(context.Background(), nil, "/in.mp4", "/out", TranscodeOptions{})
+ if err == nil {
+ t.Fatal("Transcode with nil runner: expected error")
+ }
+}
+
+// TestIsSupportedMIME confirms only video/* gets through. The
+// permissive prefix-match is intentional — the transcoder shouldn't
+// have to know about every codec under video/*.
+func TestIsSupportedMIME(t *testing.T) {
+ for _, tt := range []struct {
+ mime string
+ want bool
+ }{
+ {"video/mp4", true},
+ {"video/quicktime", true},
+ {"VIDEO/MP4", true},
+ {"image/jpeg", false},
+ {"application/pdf", false},
+ {"", false},
+ } {
+ got := IsSupportedMIME(tt.mime)
+ if got != tt.want {
+ t.Errorf("IsSupportedMIME(%q) = %v, want %v", tt.mime, got, tt.want)
+ }
+ }
+}
+
+// TestIsAvailable does not assert truth or falsity — the test machine
+// may or may not have ffmpeg installed. It only ensures the function
+// returns without panicking and that the boolean and path agree.
+func TestIsAvailable(t *testing.T) {
+ path, ok := IsAvailable()
+ if ok && path == "" {
+ t.Error("IsAvailable: ok=true but path is empty")
+ }
+ if !ok && path != "" {
+ t.Error("IsAvailable: ok=false but path is non-empty")
+ }
+}
+
+// TestExecRunner_MissingBinary confirms the runner returns
+// ErrBinaryMissing (and does not panic) when the configured binary
+// cannot be found on PATH. A real test substitutes a binary that
+// definitely doesn't exist.
+func TestExecRunner_MissingBinary(t *testing.T) {
+ r := ExecRunner{Binary: "definitely-not-a-real-binary-fooblat"}
+ err := r.Run(context.Background(), []string{}, "")
+ if err == nil {
+ t.Fatal("ExecRunner.Run: expected error for missing binary")
+ }
+ if !errors.Is(err, ErrBinaryMissing) {
+ t.Errorf("ExecRunner.Run: expected ErrBinaryMissing, got %v", err)
+ }
+}
diff --git a/packages/go/migrate/importer/media.go b/packages/go/migrate/importer/media.go
new file mode 100644
index 00000000..7f457ad5
--- /dev/null
+++ b/packages/go/migrate/importer/media.go
@@ -0,0 +1,545 @@
+package importer
+
+import (
+ "context"
+ "crypto/sha256"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "path"
+ "strings"
+ "time"
+
+ "github.com/google/uuid"
+)
+
+// MediaMode controls how the MediaMigrator handles a remote
+// attachment URL. The two modes embody fundamentally different
+// trade-offs:
+//
+// - Copy mode pulls the bytes once, at migration time, and stores
+// them in the destination's media bucket. The source site can
+// disappear afterwards and the migrated content keeps working.
+// The trade-off is migration runtime (and bandwidth) — a site
+// with thousands of full-resolution photos can take an hour.
+//
+// - Proxy mode leaves the bytes at the source URL and registers
+// the media row with is_proxied=true. The first read-through
+// request goes through the existing image proxy (issue #37),
+// which fetches from source_url and caches the response. The
+// migration is fast and bandwidth-cheap, but the migrated
+// content has a runtime dependency on the source URL staying
+// up; if the operator wants full ownership they can flip a
+// row to copy mode later with a backfill job.
+//
+// A migration MAY mix modes per asset, but the typical case is one
+// mode for the whole run. The MediaMigrator's Config exposes the
+// per-run default; a future refinement may add per-URL overrides.
+type MediaMode uint8
+
+const (
+ // MediaModeCopy downloads each remote URL and stores the bytes
+ // locally.
+ MediaModeCopy MediaMode = iota
+
+ // MediaModeProxy leaves the bytes at the source URL and
+ // registers the row with is_proxied=true.
+ MediaModeProxy
+)
+
+// String returns the canonical CLI form so flag parsing stays
+// symmetric across the importer's CLI surface.
+func (m MediaMode) String() string {
+ switch m {
+ case MediaModeCopy:
+ return "copy"
+ case MediaModeProxy:
+ return "proxy"
+ default:
+ return "unknown"
+ }
+}
+
+// ParseMediaMode turns a CLI string into a MediaMode. The empty
+// string maps to MediaModeCopy so callers can pass the flag value
+// untouched.
+func ParseMediaMode(s string) (MediaMode, error) {
+ switch strings.ToLower(strings.TrimSpace(s)) {
+ case "", "copy":
+ return MediaModeCopy, nil
+ case "proxy":
+ return MediaModeProxy, nil
+ default:
+ return MediaModeCopy, fmt.Errorf("importer: unknown media mode %q", s)
+ }
+}
+
+// MediaConfig configures a MediaMigrator. The zero value is
+// acceptable; defaults are applied at construction time.
+type MediaConfig struct {
+ // Mode selects copy vs proxy semantics. Default: MediaModeCopy.
+ Mode MediaMode
+
+ // MaxBytes caps the per-asset download size in copy mode. Larger
+ // assets fail the migrator's IngestURL call with ErrTooLarge;
+ // the caller decides whether to skip or abort the migration.
+ // Zero falls back to DefaultMediaMaxBytes.
+ MaxBytes int64
+
+ // HTTPClient is the client used to fetch source URLs. nil falls
+ // back to a default with the package's timeout. Tests pin this
+ // to a mock client backed by httptest.NewServer.
+ HTTPClient *http.Client
+
+ // RequestTimeout bounds a single HTTP fetch in copy mode. Zero
+ // falls back to DefaultMediaRequestTimeout.
+ RequestTimeout time.Duration
+
+ // UserAgent is the User-Agent header sent on every fetch. Empty
+ // falls back to DefaultMediaUserAgent — important so a source
+ // site's WAF can identify the importer (and not block it as a
+ // generic Go http.Client).
+ UserAgent string
+}
+
+// DefaultMediaMaxBytes is the per-asset size cap in copy mode.
+// 200 MiB covers nearly every legitimate WordPress upload (the
+// platform's own default cap is 64 MiB or smaller) while keeping a
+// stray multi-gigabyte attachment from wedging the importer.
+const DefaultMediaMaxBytes int64 = 200 * 1024 * 1024
+
+// DefaultMediaRequestTimeout bounds a single HTTP fetch.
+const DefaultMediaRequestTimeout = 60 * time.Second
+
+// DefaultMediaUserAgent is sent on every fetch when the caller
+// hasn't pinned a custom value.
+const DefaultMediaUserAgent = "GoNext-WP-Migrator/1.0 (+https://gonext.dev)"
+
+// ErrTooLarge is returned by MediaMigrator.IngestURL when the source
+// response exceeds MaxBytes. The migrator's caller can treat this
+// as a per-asset skip or a fatal abort, per their migration policy.
+var ErrTooLarge = errors.New("importer: media source exceeds MaxBytes")
+
+// ErrSourceNot200 is returned when the source URL responded with a
+// non-2xx status. The error message includes the status so a CLI
+// caller can render "skipped photo.jpg: source returned 404".
+var ErrSourceNot200 = errors.New("importer: media source non-2xx")
+
+// MediaPutter is the write side of the destination storage layer.
+// Identical surface to the admin upload handler's ObjectPutter —
+// we re-declare here so the importer doesn't pull in the api
+// package.
+type MediaPutter interface {
+ // PutObject uploads body at key with the recorded Content-Type.
+ PutObject(ctx context.Context, key string, body []byte, mimeType string) error
+}
+
+// MediaInserter is the persistence boundary for media row inserts.
+// The migrator never updates rows itself — a re-run of the same
+// migration is expected to be idempotent at the post-rewrite layer,
+// not at the media-row layer (a second insert with the same source
+// URL is silently treated as success via FindBySourceURL).
+type MediaInserter interface {
+ // FindBySourceURL returns an existing media row's id and
+ // storage_key when the row was inserted in proxy mode (or in
+ // copy mode with the same source URL recorded as metadata).
+ // Returns "", "", nil when no row matches. The returned
+ // storage_key is what the migrator uses to rewrite content
+ // references — in copy mode it's the new local key; in proxy
+ // mode it's a synthetic "proxy/" placeholder.
+ FindBySourceURL(ctx context.Context, sourceURL string) (id, storageKey string, found bool, err error)
+
+ // InsertCopied registers a row for an asset whose bytes were
+ // downloaded and stored locally. is_proxied=false; source_url
+ // stays NULL (the row is fully owned by the destination).
+ InsertCopied(ctx context.Context, row MediaRow) (id string, err error)
+
+ // InsertProxied registers a row whose bytes remain at sourceURL.
+ // is_proxied=true; storage_key is a synthetic placeholder so
+ // the UNIQUE constraint applies and the rest of the codebase
+ // can treat the row uniformly.
+ InsertProxied(ctx context.Context, sourceURL string, row MediaRow) (id string, err error)
+}
+
+// MediaRow is the wire shape between the migrator and the inserter.
+// Fields mirror media-table columns rather than the WXR record —
+// the migrator does the per-field derivation (mime sniff, sha256
+// of body for copy mode, slugified filename, storage-key minting)
+// so the inserter is a thin pass-through.
+type MediaRow struct {
+ Filename string
+ MimeType string
+ ByteSize int64
+ StorageKey string
+ SHA256 []byte
+ UploaderID string
+
+ // SourceURL is non-empty for proxied rows. The inserter writes
+ // it to media.source_url and sets is_proxied=true; for copied
+ // rows the field is empty and the inserter leaves source_url
+ // NULL.
+ SourceURL string
+}
+
+// MediaMigrator coordinates per-asset ingestion during a migration.
+// Construct with NewMediaMigrator. The struct is stateless beyond
+// its config; safe for concurrent IngestURL calls.
+type MediaMigrator struct {
+ cfg MediaConfig
+ putter MediaPutter
+ inserter MediaInserter
+
+ // now is the time source for storage-key minting. Pluggable for
+ // tests that need deterministic keys.
+ now func() time.Time
+
+ // keyGen mints the storage key for a copy-mode upload. Tests
+ // pin a deterministic generator.
+ keyGen func(now time.Time, filename string) string
+}
+
+// MediaIngestResult is the per-URL outcome of IngestURL.
+type MediaIngestResult struct {
+ // MediaID is the id of the inserted (or pre-existing) media row.
+ MediaID string
+
+ // StorageKey is the key the migrator wrote (copy mode) or the
+ // synthetic proxy key (proxy mode).
+ StorageKey string
+
+ // Mode is the mode the migrator used for THIS asset. Useful for
+ // per-asset accounting on the Report.
+ Mode MediaMode
+
+ // BytesFetched is the number of bytes the migrator downloaded
+ // in copy mode; zero for proxy mode.
+ BytesFetched int64
+
+ // Reused is true when FindBySourceURL hit — IngestURL did not
+ // re-fetch or re-insert, so the run is idempotent on re-runs.
+ Reused bool
+}
+
+// NewMediaMigrator constructs a MediaMigrator. Either argument may
+// be nil for a constructor-only test, but IngestURL with nil
+// dependencies returns an error.
+//
+// The default storage key layout matches the admin upload handler's:
+// "yyyy/mm/-". A future config knob can pin a
+// custom layout for operators who want the migrated content under
+// a separate prefix.
+func NewMediaMigrator(cfg MediaConfig, putter MediaPutter, inserter MediaInserter) *MediaMigrator {
+ if cfg.MaxBytes <= 0 {
+ cfg.MaxBytes = DefaultMediaMaxBytes
+ }
+ if cfg.RequestTimeout <= 0 {
+ cfg.RequestTimeout = DefaultMediaRequestTimeout
+ }
+ if cfg.UserAgent == "" {
+ cfg.UserAgent = DefaultMediaUserAgent
+ }
+ if cfg.HTTPClient == nil {
+ cfg.HTTPClient = &http.Client{Timeout: cfg.RequestTimeout}
+ }
+ return &MediaMigrator{
+ cfg: cfg,
+ putter: putter,
+ inserter: inserter,
+ now: time.Now,
+ keyGen: defaultMediaKeyGen,
+ }
+}
+
+// SetNow pins the time source for tests.
+func (m *MediaMigrator) SetNow(fn func() time.Time) {
+ if fn != nil {
+ m.now = fn
+ }
+}
+
+// SetKeyGen pins the storage-key generator for tests.
+func (m *MediaMigrator) SetKeyGen(fn func(now time.Time, filename string) string) {
+ if fn != nil {
+ m.keyGen = fn
+ }
+}
+
+// IngestURL processes a single remote attachment URL. The behaviour
+// depends on the configured mode:
+//
+// - Copy: GET sourceURL → sniff mime → PUT bytes at a fresh
+// storage key → InsertCopied. Returns the new media id.
+//
+// - Proxy: skip the fetch; InsertProxied with a synthetic
+// storage_key derived from sha256(sourceURL). The row's
+// is_proxied flag is true; the runtime image proxy serves the
+// bytes on first read.
+//
+// In both modes IngestURL first probes the inserter with
+// FindBySourceURL; if a row already exists the result is returned
+// with Reused=true and no fetch / no row write happens. This makes
+// the function idempotent at the per-URL level — re-running a
+// migration after a partial failure picks up where it left off.
+//
+// uploaderID is the GoNext users.id the migrator attributes the row
+// to (typically the operator who triggered the migration).
+func (m *MediaMigrator) IngestURL(ctx context.Context, sourceURL, uploaderID string) (MediaIngestResult, error) {
+ if m == nil {
+ return MediaIngestResult{}, errors.New("importer: nil MediaMigrator")
+ }
+ if m.putter == nil && m.cfg.Mode == MediaModeCopy {
+ return MediaIngestResult{}, errors.New("importer: nil MediaPutter (required for copy mode)")
+ }
+ if m.inserter == nil {
+ return MediaIngestResult{}, errors.New("importer: nil MediaInserter")
+ }
+ if sourceURL == "" {
+ return MediaIngestResult{}, errors.New("importer: empty source URL")
+ }
+ if _, err := url.ParseRequestURI(sourceURL); err != nil {
+ return MediaIngestResult{}, fmt.Errorf("importer: invalid source URL %q: %w", sourceURL, err)
+ }
+ if uploaderID == "" {
+ return MediaIngestResult{}, errors.New("importer: empty uploaderID")
+ }
+
+ // Idempotency probe. If the migrator already registered this
+ // URL on an earlier run (or earlier within this run), return
+ // the existing row without touching the network or the bucket.
+ if id, key, found, err := m.inserter.FindBySourceURL(ctx, sourceURL); err != nil {
+ return MediaIngestResult{}, fmt.Errorf("importer: FindBySourceURL: %w", err)
+ } else if found {
+ return MediaIngestResult{
+ MediaID: id,
+ StorageKey: key,
+ Mode: m.cfg.Mode,
+ Reused: true,
+ }, nil
+ }
+
+ switch m.cfg.Mode {
+ case MediaModeCopy:
+ return m.ingestCopy(ctx, sourceURL, uploaderID)
+ case MediaModeProxy:
+ return m.ingestProxy(ctx, sourceURL, uploaderID)
+ default:
+ return MediaIngestResult{}, fmt.Errorf("importer: unknown media mode %v", m.cfg.Mode)
+ }
+}
+
+func (m *MediaMigrator) ingestCopy(ctx context.Context, sourceURL, uploaderID string) (MediaIngestResult, error) {
+ body, mime, err := m.fetch(ctx, sourceURL)
+ if err != nil {
+ return MediaIngestResult{}, err
+ }
+ hash := sha256.Sum256(body)
+ filename := filenameFromURL(sourceURL)
+ if filename == "" {
+ filename = "attachment"
+ }
+ key := m.keyGen(m.now(), filename)
+
+ if err := m.putter.PutObject(ctx, key, body, mime); err != nil {
+ return MediaIngestResult{}, fmt.Errorf("importer: PutObject: %w", err)
+ }
+ row := MediaRow{
+ Filename: filename,
+ MimeType: mime,
+ ByteSize: int64(len(body)),
+ StorageKey: key,
+ SHA256: hash[:],
+ UploaderID: uploaderID,
+ // SourceURL is intentionally left empty for copied rows:
+ // the bytes are now fully owned by the destination.
+ }
+ id, err := m.inserter.InsertCopied(ctx, row)
+ if err != nil {
+ return MediaIngestResult{}, fmt.Errorf("importer: InsertCopied: %w", err)
+ }
+ return MediaIngestResult{
+ MediaID: id,
+ StorageKey: key,
+ Mode: MediaModeCopy,
+ BytesFetched: int64(len(body)),
+ }, nil
+}
+
+func (m *MediaMigrator) ingestProxy(ctx context.Context, sourceURL, uploaderID string) (MediaIngestResult, error) {
+ // Synthetic storage key. The proxy handler routes off
+ // is_proxied=true, not the key shape — but the key must be
+ // unique (the column has a UNIQUE constraint), and deterministic
+ // so a re-run resolves to the same row via FindBySourceURL even
+ // when the synthetic key bytes aren't otherwise visible.
+ hash := sha256.Sum256([]byte(sourceURL))
+ key := fmt.Sprintf("proxy/%x", hash[:16])
+ filename := filenameFromURL(sourceURL)
+ if filename == "" {
+ filename = "attachment"
+ }
+ mime := mimeFromFilename(filename)
+ row := MediaRow{
+ Filename: filename,
+ MimeType: mime,
+ ByteSize: 0, // unknown — bytes live remotely
+ StorageKey: key,
+ SHA256: hash[:], // hash of the URL (no body to hash)
+ UploaderID: uploaderID,
+ SourceURL: sourceURL,
+ }
+ id, err := m.inserter.InsertProxied(ctx, sourceURL, row)
+ if err != nil {
+ return MediaIngestResult{}, fmt.Errorf("importer: InsertProxied: %w", err)
+ }
+ return MediaIngestResult{
+ MediaID: id,
+ StorageKey: key,
+ Mode: MediaModeProxy,
+ }, nil
+}
+
+// fetch downloads a source URL in copy mode. Returns the bytes, the
+// sniffed Content-Type, and an error.
+//
+// We do NOT trust the source server's Content-Type header — a
+// migrated photo from a misconfigured site has the wrong header
+// often enough that re-sniffing on our side is the only safe path.
+// http.DetectContentType produces the same value the admin upload
+// path uses, so the round-trip is symmetric.
+func (m *MediaMigrator) fetch(ctx context.Context, sourceURL string) ([]byte, string, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil)
+ if err != nil {
+ return nil, "", fmt.Errorf("importer: build request: %w", err)
+ }
+ req.Header.Set("User-Agent", m.cfg.UserAgent)
+ resp, err := m.cfg.HTTPClient.Do(req)
+ if err != nil {
+ return nil, "", fmt.Errorf("importer: fetch %q: %w", sourceURL, err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ return nil, "", fmt.Errorf("%w: %s -> %d", ErrSourceNot200, sourceURL, resp.StatusCode)
+ }
+ limited := io.LimitReader(resp.Body, m.cfg.MaxBytes+1)
+ body, err := io.ReadAll(limited)
+ if err != nil {
+ return nil, "", fmt.Errorf("importer: read response: %w", err)
+ }
+ if int64(len(body)) > m.cfg.MaxBytes {
+ return nil, "", fmt.Errorf("%w: %s exceeded %d bytes", ErrTooLarge, sourceURL, m.cfg.MaxBytes)
+ }
+ sniffLen := 512
+ if len(body) < sniffLen {
+ sniffLen = len(body)
+ }
+ return body, http.DetectContentType(body[:sniffLen]), nil
+}
+
+// defaultMediaKeyGen is the canonical "yyyy/mm/-"
+// layout, matching the admin upload handler. Exported via the
+// SetKeyGen seam for tests that need determinism.
+func defaultMediaKeyGen(now time.Time, filename string) string {
+ return fmt.Sprintf("%04d/%02d/%s-%s", now.UTC().Year(), int(now.UTC().Month()), uuid.NewString(), filename)
+}
+
+// filenameFromURL pulls the basename out of a URL's path. We don't
+// trust the source URL's filename for storage-key purposes — the
+// migrator slugifies it — but the human-facing column still wants
+// a recognisable name. Returns "" when no recognisable name is
+// embedded.
+func filenameFromURL(rawURL string) string {
+ u, err := url.Parse(rawURL)
+ if err != nil {
+ return ""
+ }
+ base := path.Base(u.Path)
+ if base == "/" || base == "." || base == "" {
+ return ""
+ }
+ // Trim query strings or fragments that snuck in.
+ if i := strings.IndexAny(base, "?#"); i >= 0 {
+ base = base[:i]
+ }
+ // Cap at the media.filename column's limit (255).
+ if len(base) > 255 {
+ base = base[len(base)-255:]
+ }
+ return base
+}
+
+// mimeFromFilename guesses a Content-Type from a filename extension.
+// Used only in proxy mode where we don't have the body to sniff;
+// the runtime image proxy re-sniffs the actual bytes on first
+// fetch, so an incorrect guess here is overridden at runtime.
+func mimeFromFilename(filename string) string {
+ idx := strings.LastIndex(filename, ".")
+ if idx < 0 {
+ return "application/octet-stream"
+ }
+ ext := strings.ToLower(filename[idx:])
+ switch ext {
+ case ".jpg", ".jpeg":
+ return "image/jpeg"
+ case ".png":
+ return "image/png"
+ case ".gif":
+ return "image/gif"
+ case ".webp":
+ return "image/webp"
+ case ".svg":
+ return "image/svg+xml"
+ case ".mp4":
+ return "video/mp4"
+ case ".webm":
+ return "video/webm"
+ case ".pdf":
+ return "application/pdf"
+ default:
+ return "application/octet-stream"
+ }
+}
+
+// RewriteContent walks content (typically the post body HTML) and
+// replaces every occurrence of an old (source) URL with the new
+// destination URL.
+//
+// The replacement is a literal string substitution, not a parsed
+// AST traversal — the WP content body is often a mix of HTML,
+// shortcodes, and raw URLs, and a parser-based pass would miss the
+// non-HTML cases. The migrator's caller is expected to feed
+// RewriteContent a per-asset {source → destination} map; the
+// function applies every replacement to the input string and
+// returns the result.
+//
+// The destination URLs differ between copy and proxy modes:
+//
+// - Copy: destination URL is the public URL of the local storage
+// key (the destination's own CDN).
+//
+// - Proxy: destination URL is the public URL of the proxy
+// endpoint for the row's id, which the runtime proxy then
+// resolves to source_url internally.
+//
+// Both cases are opaque to RewriteContent — the caller passes a
+// pre-computed map.
+func RewriteContent(content string, replacements map[string]string) string {
+ if len(replacements) == 0 || content == "" {
+ return content
+ }
+ // Build a strings.Replacer for one-pass substitution. The
+ // Replacer's longest-match-first behaviour means we don't have
+ // to worry about a source URL being a prefix of another (rare
+ // in WP exports but possible with subdir attachments).
+ pairs := make([]string, 0, len(replacements)*2)
+ for src, dst := range replacements {
+ if src == "" || dst == "" {
+ continue
+ }
+ pairs = append(pairs, src, dst)
+ }
+ if len(pairs) == 0 {
+ return content
+ }
+ return strings.NewReplacer(pairs...).Replace(content)
+}
diff --git a/packages/go/migrate/importer/media_test.go b/packages/go/migrate/importer/media_test.go
new file mode 100644
index 00000000..5926a05a
--- /dev/null
+++ b/packages/go/migrate/importer/media_test.go
@@ -0,0 +1,393 @@
+package importer
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+)
+
+// fakePutter is the test-only MediaPutter. Captures every PUT so a
+// test can assert "the migrator wrote these bytes to this key".
+type fakePutter struct {
+ mu sync.Mutex
+ stored map[string]storedObject
+ err error
+}
+
+type storedObject struct {
+ body []byte
+ mimeType string
+}
+
+func newFakePutter() *fakePutter {
+ return &fakePutter{stored: map[string]storedObject{}}
+}
+
+func (f *fakePutter) PutObject(ctx context.Context, key string, body []byte, mime string) error {
+ if f.err != nil {
+ return f.err
+ }
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ cp := make([]byte, len(body))
+ copy(cp, body)
+ f.stored[key] = storedObject{body: cp, mimeType: mime}
+ return nil
+}
+
+// fakeInserter is the test-only MediaInserter. Records every insert
+// so tests can assert the (mode, key, source_url) outcome.
+type fakeInserter struct {
+ mu sync.Mutex
+ bySourceURL map[string]string // sourceURL → id (proxy only)
+ byKey map[string]string // storage_key → id (copy and proxy)
+ copied []MediaRow
+ proxied []proxyEntry
+ idSeq int
+ findErr error
+ insertErr error
+}
+
+type proxyEntry struct {
+ sourceURL string
+ row MediaRow
+}
+
+func newFakeInserter() *fakeInserter {
+ return &fakeInserter{
+ bySourceURL: map[string]string{},
+ byKey: map[string]string{},
+ }
+}
+
+func (f *fakeInserter) nextID() string {
+ f.idSeq++
+ return fmt.Sprintf("media-%04d", f.idSeq)
+}
+
+func (f *fakeInserter) FindBySourceURL(ctx context.Context, sourceURL string) (string, string, bool, error) {
+ if f.findErr != nil {
+ return "", "", false, f.findErr
+ }
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ if id, ok := f.bySourceURL[sourceURL]; ok {
+ // Find the storage key for the id by reverse lookup.
+ for k, v := range f.byKey {
+ if v == id {
+ return id, k, true, nil
+ }
+ }
+ return id, "", true, nil
+ }
+ return "", "", false, nil
+}
+
+func (f *fakeInserter) InsertCopied(ctx context.Context, row MediaRow) (string, error) {
+ if f.insertErr != nil {
+ return "", f.insertErr
+ }
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ id := f.nextID()
+ f.copied = append(f.copied, row)
+ f.byKey[row.StorageKey] = id
+ return id, nil
+}
+
+func (f *fakeInserter) InsertProxied(ctx context.Context, sourceURL string, row MediaRow) (string, error) {
+ if f.insertErr != nil {
+ return "", f.insertErr
+ }
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ id := f.nextID()
+ f.proxied = append(f.proxied, proxyEntry{sourceURL: sourceURL, row: row})
+ f.bySourceURL[sourceURL] = id
+ f.byKey[row.StorageKey] = id
+ return id, nil
+}
+
+// pinClock returns a clock function that always returns t. Used to
+// make storage-key minting deterministic in tests.
+func pinClock(t time.Time) func() time.Time {
+ return func() time.Time { return t }
+}
+
+// staticKeyGen returns a key generator that ignores its inputs and
+// always returns key. Lets a test assert "the migrator wrote at
+// EXACTLY this key".
+func staticKeyGen(key string) func(time.Time, string) string {
+ return func(time.Time, string) string { return key }
+}
+
+// TestMediaMigrator_CopyMode_DownloadsAndStores spins up a fake
+// source server, points the migrator at it in copy mode, and
+// asserts:
+// - the source server received exactly one request
+// - the putter saw the same bytes
+// - the inserter recorded the row as copied (not proxied)
+// - the resulting row carries no SourceURL (copy mode owns the
+// bytes outright)
+func TestMediaMigrator_CopyMode_DownloadsAndStores(t *testing.T) {
+ var reqCount int
+ body := []byte("FAKE_JPEG_BYTES_FROM_SOURCE")
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ reqCount++
+ w.Header().Set("Content-Type", "image/jpeg")
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write(body)
+ }))
+ defer srv.Close()
+
+ putter := newFakePutter()
+ inserter := newFakeInserter()
+ m := NewMediaMigrator(MediaConfig{Mode: MediaModeCopy}, putter, inserter)
+ m.SetNow(pinClock(time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC)))
+ m.SetKeyGen(staticKeyGen("2026/05/test-photo.jpg"))
+
+ res, err := m.IngestURL(context.Background(), srv.URL+"/uploads/2024/03/photo.jpg", "user-001")
+ if err != nil {
+ t.Fatalf("IngestURL: %v", err)
+ }
+ if reqCount != 1 {
+ t.Errorf("source server hit %d times, want 1", reqCount)
+ }
+ if res.Mode != MediaModeCopy {
+ t.Errorf("result.Mode = %v, want copy", res.Mode)
+ }
+ if res.StorageKey != "2026/05/test-photo.jpg" {
+ t.Errorf("result.StorageKey = %q, want 2026/05/test-photo.jpg", res.StorageKey)
+ }
+ if res.BytesFetched != int64(len(body)) {
+ t.Errorf("result.BytesFetched = %d, want %d", res.BytesFetched, len(body))
+ }
+ if res.Reused {
+ t.Error("result.Reused should be false on first call")
+ }
+ // Putter saw the bytes
+ stored, ok := putter.stored["2026/05/test-photo.jpg"]
+ if !ok {
+ t.Fatalf("putter did not receive the expected key; got keys: %v", keysOf(putter.stored))
+ }
+ if string(stored.body) != string(body) {
+ t.Errorf("putter body mismatch: %q vs %q", stored.body, body)
+ }
+ // Inserter recorded the row as copied
+ if len(inserter.copied) != 1 {
+ t.Fatalf("inserter.copied len = %d, want 1", len(inserter.copied))
+ }
+ if inserter.copied[0].SourceURL != "" {
+ t.Errorf("copied row should have empty SourceURL, got %q", inserter.copied[0].SourceURL)
+ }
+ if len(inserter.proxied) != 0 {
+ t.Errorf("inserter should not have proxied rows in copy mode")
+ }
+}
+
+// TestMediaMigrator_ProxyMode_NoFetchNoUpload confirms proxy mode
+// never hits the source server and never writes to the putter.
+func TestMediaMigrator_ProxyMode_NoFetchNoUpload(t *testing.T) {
+ var reqCount int
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ reqCount++
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer srv.Close()
+
+ putter := newFakePutter()
+ inserter := newFakeInserter()
+ m := NewMediaMigrator(MediaConfig{Mode: MediaModeProxy}, putter, inserter)
+
+ res, err := m.IngestURL(context.Background(), srv.URL+"/uploads/2024/03/photo.jpg", "user-001")
+ if err != nil {
+ t.Fatalf("IngestURL: %v", err)
+ }
+ if reqCount != 0 {
+ t.Errorf("proxy mode should NOT fetch source; got %d requests", reqCount)
+ }
+ if len(putter.stored) != 0 {
+ t.Errorf("proxy mode should NOT write to putter; got %d uploads", len(putter.stored))
+ }
+ if res.Mode != MediaModeProxy {
+ t.Errorf("result.Mode = %v, want proxy", res.Mode)
+ }
+ if res.BytesFetched != 0 {
+ t.Errorf("result.BytesFetched = %d, want 0 for proxy", res.BytesFetched)
+ }
+ if !strings.HasPrefix(res.StorageKey, "proxy/") {
+ t.Errorf("proxy storage key should have proxy/ prefix; got %q", res.StorageKey)
+ }
+ // Inserter recorded the row as proxied
+ if len(inserter.proxied) != 1 {
+ t.Fatalf("inserter.proxied len = %d, want 1", len(inserter.proxied))
+ }
+ if inserter.proxied[0].sourceURL != srv.URL+"/uploads/2024/03/photo.jpg" {
+ t.Errorf("proxied sourceURL = %q", inserter.proxied[0].sourceURL)
+ }
+ if inserter.proxied[0].row.SourceURL == "" {
+ t.Error("proxied row.SourceURL should be set")
+ }
+ if len(inserter.copied) != 0 {
+ t.Errorf("inserter should not have copied rows in proxy mode")
+ }
+}
+
+// TestMediaMigrator_Idempotency confirms a second IngestURL with the
+// same URL returns the existing row without re-fetching or
+// re-inserting.
+func TestMediaMigrator_Idempotency(t *testing.T) {
+ var reqCount int
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ reqCount++
+ w.Write([]byte("BYTES"))
+ }))
+ defer srv.Close()
+
+ putter := newFakePutter()
+ inserter := newFakeInserter()
+ // Pre-seed the inserter with a row for this URL so the
+ // idempotency probe hits on first call too.
+ inserter.bySourceURL[srv.URL+"/x.jpg"] = "preexisting-id"
+ inserter.byKey["preexisting-key"] = "preexisting-id"
+
+ m := NewMediaMigrator(MediaConfig{Mode: MediaModeProxy}, putter, inserter)
+ res, err := m.IngestURL(context.Background(), srv.URL+"/x.jpg", "u")
+ if err != nil {
+ t.Fatalf("IngestURL: %v", err)
+ }
+ if !res.Reused {
+ t.Error("result.Reused should be true when FindBySourceURL hits")
+ }
+ if res.MediaID != "preexisting-id" {
+ t.Errorf("MediaID = %q, want preexisting-id", res.MediaID)
+ }
+ if reqCount != 0 {
+ t.Errorf("idempotent call should not fetch; got %d requests", reqCount)
+ }
+}
+
+// TestMediaMigrator_CopyMode_Non200 confirms a 404 from the source
+// surfaces as ErrSourceNot200.
+func TestMediaMigrator_CopyMode_Non200(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.NotFound(w, r)
+ }))
+ defer srv.Close()
+
+ m := NewMediaMigrator(MediaConfig{Mode: MediaModeCopy}, newFakePutter(), newFakeInserter())
+ _, err := m.IngestURL(context.Background(), srv.URL+"/missing.jpg", "u")
+ if err == nil {
+ t.Fatal("expected error, got nil")
+ }
+ if !errors.Is(err, ErrSourceNot200) {
+ t.Errorf("err = %v, want ErrSourceNot200", err)
+ }
+}
+
+// TestMediaMigrator_CopyMode_TooLarge confirms the size cap is
+// enforced and ErrTooLarge surfaces.
+func TestMediaMigrator_CopyMode_TooLarge(t *testing.T) {
+ big := make([]byte, 200) // larger than the cap below
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Write(big)
+ }))
+ defer srv.Close()
+
+ m := NewMediaMigrator(MediaConfig{Mode: MediaModeCopy, MaxBytes: 50}, newFakePutter(), newFakeInserter())
+ _, err := m.IngestURL(context.Background(), srv.URL+"/big.jpg", "u")
+ if err == nil {
+ t.Fatal("expected error, got nil")
+ }
+ if !errors.Is(err, ErrTooLarge) {
+ t.Errorf("err = %v, want ErrTooLarge", err)
+ }
+}
+
+// TestMediaMigrator_RejectsBadInput pins the input-validation
+// contract: empty URL, invalid URL, missing uploader.
+func TestMediaMigrator_RejectsBadInput(t *testing.T) {
+ m := NewMediaMigrator(MediaConfig{}, newFakePutter(), newFakeInserter())
+ if _, err := m.IngestURL(context.Background(), "", "u"); err == nil {
+ t.Error("empty URL: expected error")
+ }
+ if _, err := m.IngestURL(context.Background(), "not a url", "u"); err == nil {
+ t.Error("invalid URL: expected error")
+ }
+ if _, err := m.IngestURL(context.Background(), "http://x/y.jpg", ""); err == nil {
+ t.Error("empty uploaderID: expected error")
+ }
+}
+
+// TestParseMediaMode pins the canonical CLI form so flag parsing
+// stays stable across releases.
+func TestParseMediaMode(t *testing.T) {
+ for _, tt := range []struct {
+ in string
+ want MediaMode
+ ok bool
+ }{
+ {"", MediaModeCopy, true},
+ {"copy", MediaModeCopy, true},
+ {"COPY", MediaModeCopy, true},
+ {"proxy", MediaModeProxy, true},
+ {"PROXY", MediaModeProxy, true},
+ {"junk", MediaModeCopy, false},
+ } {
+ got, err := ParseMediaMode(tt.in)
+ if (err == nil) != tt.ok {
+ t.Errorf("ParseMediaMode(%q) ok = %v, want %v", tt.in, err == nil, tt.ok)
+ }
+ if tt.ok && got != tt.want {
+ t.Errorf("ParseMediaMode(%q) = %v, want %v", tt.in, got, tt.want)
+ }
+ }
+}
+
+// TestRewriteContent confirms the per-asset URL substitution does
+// what it says.
+func TestRewriteContent(t *testing.T) {
+ in := `See file .
`
+ out := RewriteContent(in, map[string]string{
+ "https://old.example/a.jpg": "https://cdn.gonext.example/2026/05/a.jpg",
+ "https://old.example/b.pdf": "https://proxy.gonext.example/m/abc",
+ })
+ if !strings.Contains(out, "https://cdn.gonext.example/2026/05/a.jpg") {
+ t.Errorf("missing copy-mode replacement: %s", out)
+ }
+ if !strings.Contains(out, "https://proxy.gonext.example/m/abc") {
+ t.Errorf("missing proxy-mode replacement: %s", out)
+ }
+ if strings.Contains(out, "old.example") {
+ t.Errorf("source URL still present after rewrite: %s", out)
+ }
+}
+
+// TestRewriteContent_EmptyInputs handles the no-op paths.
+func TestRewriteContent_EmptyInputs(t *testing.T) {
+ if got := RewriteContent("", nil); got != "" {
+ t.Errorf("empty content: got %q", got)
+ }
+ if got := RewriteContent("hello", nil); got != "hello" {
+ t.Errorf("no replacements: got %q", got)
+ }
+ if got := RewriteContent("hello", map[string]string{"": ""}); got != "hello" {
+ t.Errorf("empty pairs: got %q", got)
+ }
+}
+
+// keysOf returns sorted keys of a map for error messages.
+func keysOf(m map[string]storedObject) []string {
+ out := make([]string, 0, len(m))
+ for k := range m {
+ out = append(out, k)
+ }
+ return out
+}
diff --git a/packages/go/migrate/importer/types.go b/packages/go/migrate/importer/types.go
index 929d964c..6e2c5143 100644
--- a/packages/go/migrate/importer/types.go
+++ b/packages/go/migrate/importer/types.go
@@ -108,6 +108,24 @@ type Options struct {
// user resets — but callers can override it for tests or to
// pin a specific hash format.
PlaceholderPasswordHash string
+
+ // MediaMigrator, when non-nil, is the per-asset ingestion
+ // orchestrator the importer hands every WXR attachment URL.
+ // Wired up by the CLI or the migration wizard UI from the
+ // operator's selection of "copy" vs "proxy" mode (#187, #234).
+ //
+ // Nil disables media migration entirely — the importer still
+ // records attachment posts but does not download or proxy the
+ // underlying bytes; the post bodies retain their original
+ // source URLs and the imported site falls back to hot-linking.
+ MediaMigrator *MediaMigrator
+
+ // MediaUploaderID is the GoNext users.id the MediaMigrator
+ // attributes every migrated media row to. Required when
+ // MediaMigrator is non-nil; empty triggers an error at Run-
+ // validation time. Typically the operator who triggered the
+ // migration.
+ MediaUploaderID string
}
// resolved returns a copy of Options with defaults applied. Internal.
@@ -162,6 +180,28 @@ type Report struct {
// secondary breakdown for the CLI report.
Attachments int
+ // MediaCopied is the number of attachment URLs the MediaMigrator
+ // successfully downloaded and stored locally. Always zero when
+ // the migration ran in proxy mode (or when no MediaMigrator was
+ // wired). Issue #187.
+ MediaCopied int
+
+ // MediaProxied is the number of attachment URLs the MediaMigrator
+ // registered as proxied rows. Always zero in copy mode.
+ MediaProxied int
+
+ // MediaSkipped is the number of attachment URLs the
+ // MediaMigrator did not ingest because a row already existed
+ // (idempotency hit on FindBySourceURL). Both modes can produce
+ // skips on a re-run.
+ MediaSkipped int
+
+ // MediaBytesFetched is the total number of source bytes the
+ // MediaMigrator downloaded in copy mode. Always zero in proxy
+ // mode. Useful for the CLI report to surface "we transferred
+ // X MB during the migration".
+ MediaBytesFetched int64
+
// Errors collects per-record failures. Never nil-checked by
// callers — an empty slice means "no errors" and an unset slice
// means the same thing. Re-allocated to nil if the user trims