diff --git a/.dockerignore b/.dockerignore index f201e9225..956c90ab5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,4 @@ _build deps .elixir_ls priv +native/philomena/target diff --git a/config/runtime.exs b/config/runtime.exs index 6d00d8ad6..0e976d4ea 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -27,6 +27,7 @@ config :philomena, image_url_root: System.fetch_env!("IMAGE_URL_ROOT"), badge_url_root: System.fetch_env!("BADGE_URL_ROOT"), mailer_address: System.fetch_env!("MAILER_ADDRESS"), + mediaproc_addr: System.fetch_env!("MEDIAPROC_ADDR"), tag_file_root: System.fetch_env!("TAG_FILE_ROOT"), hide_version: System.get_env("HIDE_VERSION", "false"), site_domains: System.fetch_env!("SITE_DOMAINS"), diff --git a/docker-compose.yml b/docker-compose.yml index 9e2cedb35..42c994b83 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,6 +39,7 @@ services: - IMAGE_URL_ROOT=/img - BADGE_URL_ROOT=/badge-img - TAG_URL_ROOT=/tag-img + - MEDIAPROC_ADDR=mediaproc:1500 - OPENSEARCH_URL=http://opensearch:9200 - REDIS_HOST=valkey - DATABASE_URL=ecto://postgres:postgres@postgres/philomena_dev @@ -61,6 +62,7 @@ services: - app_deps_data:/srv/philomena/deps - app_native_data:/srv/philomena/priv/native depends_on: + - mediaproc - postgres - opensearch - valkey @@ -99,6 +101,18 @@ services: - s3_data:/data attach: false + mediaproc: + build: + context: . + dockerfile: ./docker/mediaproc/Dockerfile + attach: false + deploy: + resources: + limits: + cpus: '4' + memory: 8gb + pids: 8192 + web: build: context: . diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index 9e8a87d44..76a9cda57 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -1,47 +1,10 @@ FROM elixir:1.18.4-alpine -ADD https://api.github.com/repos/philomena-dev/FFmpeg/git/refs/heads/release/6.1 /tmp/ffmpeg_version.json -RUN (echo "https://github.com/philomena-dev/prebuilt-ffmpeg/raw/master"; cat /etc/apk/repositories) > /tmp/repositories \ - && cp /tmp/repositories /etc/apk/repositories \ - && apk update --allow-untrusted \ - && apk add --allow-untrusted \ - bash \ - inotify-tools \ - build-base \ - git \ - ffmpeg \ - ffmpeg-dev \ - npm \ - nodejs \ - file-dev \ - libjpeg-turbo-dev \ - libpng-dev \ - gifsicle \ - optipng \ - libjpeg-turbo-utils \ - librsvg \ - rsvg-convert \ - imagemagick \ - postgresql17-client \ - wget \ - rust \ - cargo \ +RUN apk add bash inotify-tools build-base git npm nodejs postgresql17-client wget rust cargo \ && mix local.hex --force \ && mix local.rebar --force -ADD https://api.github.com/repos/philomena-dev/cli_intensities/git/refs/heads/master /tmp/cli_intensities_version.json -RUN git clone --depth 1 https://github.com/philomena-dev/cli_intensities /tmp/cli_intensities \ - && cd /tmp/cli_intensities \ - && make -j$(nproc) install - -ADD https://api.github.com/repos/philomena-dev/mediatools/git/refs/heads/master /tmp/mediatools_version.json -RUN git clone --depth 1 https://github.com/philomena-dev/mediatools /tmp/mediatools \ - && ln -s /usr/lib/librsvg-2.so.2 /usr/lib/librsvg-2.so \ - && cd /tmp/mediatools \ - && make -j$(nproc) install - # docker-compose configures a bind-mount of the repo root dir to /srv/philomena ENV PATH=$PATH:/root/.cargo/bin:/srv/philomena/docker/app - EXPOSE 5173 CMD run-development diff --git a/docker/mediaproc/Dockerfile b/docker/mediaproc/Dockerfile new file mode 100644 index 000000000..8aa2c9e2d --- /dev/null +++ b/docker/mediaproc/Dockerfile @@ -0,0 +1,75 @@ +FROM rust:1.90-slim-trixie + +RUN apt update \ + && apt install -y build-essential git libmagic-dev libturbojpeg0-dev libpng-dev \ + gifsicle optipng libjpeg-turbo-progs librsvg2-bin librsvg2-dev file imagemagick \ + libx264-dev libx265-dev libvpx-dev libdav1d-dev libaom-dev libopus-dev \ + libmp3lame-dev libvorbis-dev libwebp-dev libjxl-dev yasm wget + +ADD https://api.github.com/repos/philomena-dev/FFmpeg/git/refs/heads/release/7.1 /tmp/ffmpeg_version.json +ADD https://api.github.com/repos/philomena-dev/cli_intensities/git/refs/heads/master /tmp/cli_intensities_version.json +ADD https://api.github.com/repos/philomena-dev/mediatools/git/refs/heads/master /tmp/mediatools_version.json + +RUN wget -qO /tmp/FFmpeg.tar.gz https://github.com/philomena-dev/FFmpeg/archive/refs/heads/release/7.1.tar.gz \ + && wget -qO /tmp/cli_intensities.tar.gz https://github.com/philomena-dev/cli_intensities/archive/refs/heads/master.tar.gz \ + && wget -qO /tmp/mediatools.tar.gz https://github.com/philomena-dev/mediatools/archive/refs/heads/master.tar.gz + +RUN cd /tmp \ + && tar -xf FFmpeg.tar.gz \ + && tar -xf cli_intensities.tar.gz \ + && tar -xf mediatools.tar.gz \ + && cd /tmp/FFmpeg-release-7.1 \ + && ./configure \ + --prefix=/usr \ + --disable-everything \ + --disable-stripping \ + --disable-static \ + --disable-ffplay \ + --disable-doc \ + --disable-htmlpages \ + --disable-manpages \ + --disable-podpages \ + --disable-txtpages \ + --disable-protocols \ + --enable-shared \ + --enable-pic \ + --enable-pthreads \ + --enable-gpl \ + --enable-avfilter \ + --enable-bsf=extract_extradata \ + --enable-decoder=aac,apng,av1,gif,h264,hevc,jpeg2000,jpegxl,libaom-av1,libdav1d,libvorbis,libvpx_vp8,libvpx_vp9,mp3,mjpeg,opus,png,vorbis,vp8,vp9,webvtt \ + --enable-demuxer=apng,gif,image2,image_gif_pipe,image_jpeg_pipe,image_png_pipe,image_webp_pipe,matroska,mjpeg,mjpeg_2000,mov,webm \ + --enable-encoder=aac,apng,gif,jpegxl,libmp3lame,libaom-av1,libvorbis,libopus,libvpx_vp8,libvpx_vp9,libx265,libx264,opus,mjpeg,png,vorbis,webvtt \ + --enable-filter=concat,palettegen,paletteuse,scale,setpts,setsar,settb,split,trim \ + --enable-libaom \ + --enable-libjxl \ + --enable-libdav1d \ + --enable-libopus \ + --enable-libmp3lame \ + --enable-libvpx \ + --enable-libvorbis \ + --enable-libx264 \ + --enable-libx265 \ + --enable-libwebp \ + --enable-muxer=apng,image2,gif,matroska,mp4,webp,webm \ + --enable-parser=aac,gif,h264,hevc,jpeg2000,jpegxl,mjpeg,opus,png,vorbis,vp8,vp9,webp \ + --enable-protocol=concat,data,file,subfile \ + && make -j$(nproc) install \ + && cd /tmp/cli_intensities-master \ + && make -j$(nproc) install \ + && cd /tmp/mediatools-master \ + && make -j$(nproc) install + +COPY native/philomena /tmp/philomena +COPY docker/mediaproc/safe-rsvg-convert /usr/bin/safe-rsvg-convert + +RUN cd /tmp/philomena \ + && cargo build --release -p mediaproc_server \ + && cp target/release/mediaproc_server /usr/bin/mediaproc_server + +# Set up unprivileged user account +RUN useradd -ms /bin/bash mediaproc +USER mediaproc +WORKDIR /home/mediaproc +ENV RUST_LOG=trace +CMD ["/usr/bin/mediaproc_server", "0.0.0.0:1500"] diff --git a/docker/app/safe-rsvg-convert b/docker/mediaproc/safe-rsvg-convert similarity index 100% rename from docker/app/safe-rsvg-convert rename to docker/mediaproc/safe-rsvg-convert diff --git a/lib/philomena/native.ex b/lib/philomena/native.ex index 14eeaa17f..b4f2f2444 100644 --- a/lib/philomena/native.ex +++ b/lib/philomena/native.ex @@ -12,6 +12,10 @@ defmodule Philomena.Native do @spec camo_image_url(String.t()) :: String.t() def camo_image_url(_uri), do: :erlang.nif_error(:nif_not_loaded) + @spec async_process_command(String.t(), String.t(), [String.t()]) :: :ok + def async_process_command(_server_addr, _program, _arguments), + do: :erlang.nif_error(:nif_not_loaded) + @spec zip_open_writer(Path.t()) :: {:ok, reference()} | {:error, atom()} def zip_open_writer(_path), do: :erlang.nif_error(:nif_not_loaded) diff --git a/lib/philomena_media/analyzers/gif.ex b/lib/philomena_media/analyzers/gif.ex index 982d7a319..2c1365d0e 100644 --- a/lib/philomena_media/analyzers/gif.ex +++ b/lib/philomena_media/analyzers/gif.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Gif do alias PhilomenaMedia.Analyzers.Analyzer alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Remote @behaviour Analyzer @@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Gif do end defp stats(file) do - case System.cmd("mediastat", [file]) do + case Remote.cmd("mediastat", [file]) do {output, 0} -> [_size, frames, width, height, num, den] = output diff --git a/lib/philomena_media/analyzers/jpeg.ex b/lib/philomena_media/analyzers/jpeg.ex index 60b29e04f..cca1d8810 100644 --- a/lib/philomena_media/analyzers/jpeg.ex +++ b/lib/philomena_media/analyzers/jpeg.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Jpeg do alias PhilomenaMedia.Analyzers.Analyzer alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Remote @behaviour Analyzer @@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Jpeg do end defp stats(file) do - case System.cmd("mediastat", [file]) do + case Remote.cmd("mediastat", [file]) do {output, 0} -> [_size, _frames, width, height, num, den] = output diff --git a/lib/philomena_media/analyzers/png.ex b/lib/philomena_media/analyzers/png.ex index 83cb506f4..a01d68abf 100644 --- a/lib/philomena_media/analyzers/png.ex +++ b/lib/philomena_media/analyzers/png.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Png do alias PhilomenaMedia.Analyzers.Analyzer alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Remote @behaviour Analyzer @@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Png do end defp stats(file) do - case System.cmd("mediastat", [file]) do + case Remote.cmd("mediastat", [file]) do {output, 0} -> [_size, frames, width, height, num, den] = output diff --git a/lib/philomena_media/analyzers/svg.ex b/lib/philomena_media/analyzers/svg.ex index f83a55f00..0b55a5681 100644 --- a/lib/philomena_media/analyzers/svg.ex +++ b/lib/philomena_media/analyzers/svg.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Svg do alias PhilomenaMedia.Analyzers.Analyzer alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Remote @behaviour Analyzer @@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Svg do end defp stats(file) do - case System.cmd("svgstat", [file]) do + case Remote.cmd("svgstat", [file]) do {output, 0} -> [_size, _frames, width, height, _num, _den] = output diff --git a/lib/philomena_media/analyzers/webm.ex b/lib/philomena_media/analyzers/webm.ex index b215e01e1..fb785923e 100644 --- a/lib/philomena_media/analyzers/webm.ex +++ b/lib/philomena_media/analyzers/webm.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Webm do alias PhilomenaMedia.Analyzers.Analyzer alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Remote @behaviour Analyzer @@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Webm do end defp stats(file) do - case System.cmd("mediastat", [file]) do + case Remote.cmd("mediastat", [file]) do {output, 0} -> [_size, frames, width, height, num, den] = output diff --git a/lib/philomena_media/gif_preview.ex b/lib/philomena_media/gif_preview.ex index ac40044b4..beef1e781 100644 --- a/lib/philomena_media/gif_preview.ex +++ b/lib/philomena_media/gif_preview.ex @@ -3,6 +3,8 @@ defmodule PhilomenaMedia.GifPreview do GIF preview generation for video files. """ + alias PhilomenaMedia.Remote + @type duration :: float() @type dimensions :: {pos_integer(), pos_integer()} @@ -49,7 +51,7 @@ defmodule PhilomenaMedia.GifPreview do end) {_output, 0} = - System.cmd( + Remote.cmd( "ffmpeg", commands(decoder, video, gif, clamp(duration), dimensions, num_images, target_framerate) ) diff --git a/lib/philomena_media/intensities.ex b/lib/philomena_media/intensities.ex index ea0952952..f2df677b6 100644 --- a/lib/philomena_media/intensities.ex +++ b/lib/philomena_media/intensities.ex @@ -17,6 +17,8 @@ defmodule PhilomenaMedia.Intensities do of image dimensions, with poor precision and a poor-to-fair accuracy. """ + alias PhilomenaMedia.Remote + @type t :: %__MODULE__{ nw: float(), ne: float(), @@ -50,7 +52,7 @@ defmodule PhilomenaMedia.Intensities do """ @spec file(Path.t()) :: {:ok, t()} | :error def file(input) do - System.cmd("image-intensities", [input]) + Remote.cmd("image-intensities", [input]) |> case do {output, 0} -> [nw, ne, sw, se] = diff --git a/lib/philomena_media/mime.ex b/lib/philomena_media/mime.ex index 1b29aa759..3bd483050 100644 --- a/lib/philomena_media/mime.ex +++ b/lib/philomena_media/mime.ex @@ -27,7 +27,7 @@ defmodule PhilomenaMedia.Mime do """ @spec file(Path.t()) :: {:ok, t()} | {:unsupported_mime, t()} | :error def file(path) do - System.cmd("file", ["-b", "--mime-type", path]) + PhilomenaMedia.Remote.cmd("file", ["-b", "--mime-type", path]) |> case do {output, 0} -> true_mime(String.trim(output)) diff --git a/lib/philomena_media/processors/gif.ex b/lib/philomena_media/processors/gif.ex index 6e185f9fb..497567198 100644 --- a/lib/philomena_media/processors/gif.ex +++ b/lib/philomena_media/processors/gif.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Gif do alias PhilomenaMedia.Intensities alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Remote alias PhilomenaMedia.Processors.Processor alias PhilomenaMedia.Processors @@ -46,7 +47,7 @@ defmodule PhilomenaMedia.Processors.Gif do defp optimize(file) do optimized = Briefly.create!(extname: ".gif") - {_output, 0} = System.cmd("gifsicle", ["--careful", "-O2", file, "-o", optimized]) + {_output, 0} = Remote.cmd("gifsicle", ["--careful", "-O2", file, "-o", optimized]) optimized end @@ -54,7 +55,7 @@ defmodule PhilomenaMedia.Processors.Gif do defp preview(duration, file) do preview = Briefly.create!(extname: ".png") - {_output, 0} = System.cmd("mediathumb", [file, to_string(duration / 2), preview]) + {_output, 0} = Remote.cmd("mediathumb", [file, to_string(duration / 2), preview]) preview end @@ -63,7 +64,7 @@ defmodule PhilomenaMedia.Processors.Gif do palette = Briefly.create!(extname: ".png") {_output, 0} = - System.cmd("ffmpeg", [ + Remote.cmd("ffmpeg", [ "-loglevel", "0", "-y", @@ -88,7 +89,7 @@ defmodule PhilomenaMedia.Processors.Gif do filter_graph = "[0:v]#{scale_filter}[x];[x][1:v]#{palette_filter}" {_output, 0} = - System.cmd("ffmpeg", [ + Remote.cmd("ffmpeg", [ "-loglevel", "0", "-y", @@ -109,7 +110,7 @@ defmodule PhilomenaMedia.Processors.Gif do mp4 = Briefly.create!(extname: ".mp4") {_output, 0} = - System.cmd("ffmpeg", [ + Remote.cmd("ffmpeg", [ "-loglevel", "0", "-y", @@ -127,7 +128,7 @@ defmodule PhilomenaMedia.Processors.Gif do ]) {_output, 0} = - System.cmd("ffmpeg", [ + Remote.cmd("ffmpeg", [ "-loglevel", "0", "-y", diff --git a/lib/philomena_media/processors/jpeg.ex b/lib/philomena_media/processors/jpeg.ex index 45aed8462..30b2d5aff 100644 --- a/lib/philomena_media/processors/jpeg.ex +++ b/lib/philomena_media/processors/jpeg.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do alias PhilomenaMedia.Intensities alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Remote alias PhilomenaMedia.Processors.Processor alias PhilomenaMedia.Processors @@ -42,7 +43,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do defp requires_lossy_transformation?(file) do with {output, 0} <- - System.cmd("magick", ["identify", "-format", "%[orientation]\t%[profile:icc]", file]), + Remote.cmd("magick", ["identify", "-format", "%[orientation]\t%[profile:icc]", file]), [orientation, profile] <- String.split(output, "\t") do orientation not in ["Undefined", "TopLeft"] or profile != "" else @@ -59,7 +60,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do if requires_lossy_transformation?(file) do # Transcode: strip EXIF, embedded profile and reorient image {_output, 0} = - System.cmd("magick", [ + Remote.cmd("magick", [ file, "-profile", srgb_profile(), @@ -69,7 +70,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do ]) else # Transmux only: Strip EXIF without touching orientation - validate_return(System.cmd("jpegtran", ["-copy", "none", "-outfile", stripped, file])) + validate_return(Remote.cmd("jpegtran", ["-copy", "none", "-outfile", stripped, file])) end stripped @@ -78,7 +79,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do defp optimize(file) do optimized = Briefly.create!(extname: ".jpg") - validate_return(System.cmd("jpegtran", ["-optimize", "-outfile", optimized, file])) + validate_return(Remote.cmd("jpegtran", ["-optimize", "-outfile", optimized, file])) optimized end @@ -88,7 +89,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do scale_filter = "scale=w=#{width}:h=#{height}:force_original_aspect_ratio=decrease" {_output, 0} = - System.cmd("ffmpeg", [ + Remote.cmd("ffmpeg", [ "-loglevel", "0", "-y", @@ -101,7 +102,7 @@ defmodule PhilomenaMedia.Processors.Jpeg do scaled ]) - {_output, 0} = System.cmd("jpegtran", ["-optimize", "-outfile", scaled, scaled]) + {_output, 0} = Remote.cmd("jpegtran", ["-optimize", "-outfile", scaled, scaled]) [{:copy, scaled, "#{thumb_name}.jpg"}] end diff --git a/lib/philomena_media/processors/png.ex b/lib/philomena_media/processors/png.ex index 7864f7dcd..24222ce8e 100644 --- a/lib/philomena_media/processors/png.ex +++ b/lib/philomena_media/processors/png.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Png do alias PhilomenaMedia.Intensities alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Remote alias PhilomenaMedia.Processors.Processor alias PhilomenaMedia.Processors @@ -49,7 +50,7 @@ defmodule PhilomenaMedia.Processors.Png do optimized = Briefly.create!(extname: ".png") {_output, 0} = - System.cmd("optipng", ["-fix", "-i0", "-o2", "-quiet", "-clobber", file, "-out", optimized]) + Remote.cmd("optipng", ["-fix", "-i0", "-o2", "-quiet", "-clobber", file, "-out", optimized]) # Remove useless .bak file File.rm(optimized <> ".bak") @@ -65,7 +66,7 @@ defmodule PhilomenaMedia.Processors.Png do {_output, 0} = if animated? do - System.cmd("ffmpeg", [ + Remote.cmd("ffmpeg", [ "-loglevel", "0", "-y", @@ -80,10 +81,10 @@ defmodule PhilomenaMedia.Processors.Png do scaled ]) else - System.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", file, "-vf", scale_filter, scaled]) + Remote.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", file, "-vf", scale_filter, scaled]) end - System.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled]) + Remote.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled]) [{:copy, scaled, "#{thumb_name}.png"}] end diff --git a/lib/philomena_media/processors/svg.ex b/lib/philomena_media/processors/svg.ex index aaa3dd5ca..0f9b6e6cb 100644 --- a/lib/philomena_media/processors/svg.ex +++ b/lib/philomena_media/processors/svg.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Svg do alias PhilomenaMedia.Intensities alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Remote alias PhilomenaMedia.Processors.Processor alias PhilomenaMedia.Processors @@ -42,7 +43,7 @@ defmodule PhilomenaMedia.Processors.Svg do defp preview(file) do preview = Briefly.create!(extname: ".png") - {_output, 0} = System.cmd("safe-rsvg-convert", [file, preview]) + {_output, 0} = Remote.cmd("safe-rsvg-convert", [file, preview]) preview end @@ -52,9 +53,9 @@ defmodule PhilomenaMedia.Processors.Svg do scale_filter = "scale=w=#{width}:h=#{height}:force_original_aspect_ratio=decrease" {_output, 0} = - System.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", preview, "-vf", scale_filter, scaled]) + Remote.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", preview, "-vf", scale_filter, scaled]) - {_output, 0} = System.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled]) + {_output, 0} = Remote.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled]) [{:copy, scaled, "#{thumb_name}.png"}] end diff --git a/lib/philomena_media/processors/webm.ex b/lib/philomena_media/processors/webm.ex index 00bbfd85a..26d488ab0 100644 --- a/lib/philomena_media/processors/webm.ex +++ b/lib/philomena_media/processors/webm.ex @@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Webm do alias PhilomenaMedia.Intensities alias PhilomenaMedia.Analyzers.Result + alias PhilomenaMedia.Remote alias PhilomenaMedia.GifPreview alias PhilomenaMedia.Processors.Processor alias PhilomenaMedia.Processors @@ -56,7 +57,7 @@ defmodule PhilomenaMedia.Processors.Webm do defp preview(duration, file) do preview = Briefly.create!(extname: ".png") - {_output, 0} = System.cmd("mediathumb", [file, to_string(duration / 2), preview]) + {_output, 0} = Remote.cmd("mediathumb", [file, to_string(duration / 2), preview]) preview end @@ -65,7 +66,7 @@ defmodule PhilomenaMedia.Processors.Webm do stripped = Briefly.create!(extname: ".webm") {_output, 0} = - System.cmd("ffmpeg", [ + Remote.cmd("ffmpeg", [ "-loglevel", "0", "-y", @@ -108,7 +109,7 @@ defmodule PhilomenaMedia.Processors.Webm do mp4 = Briefly.create!(extname: ".mp4") {_output, 0} = - System.cmd("ffmpeg", [ + Remote.cmd("ffmpeg", [ "-loglevel", "0", "-y", @@ -168,7 +169,7 @@ defmodule PhilomenaMedia.Processors.Webm do mp4 = Briefly.create!(extname: ".mp4") {_output, 0} = - System.cmd("ffmpeg", [ + Remote.cmd("ffmpeg", [ "-loglevel", "0", "-y", @@ -211,7 +212,7 @@ defmodule PhilomenaMedia.Processors.Webm do defp select_decoder(file) do {output, 0} = - System.cmd("ffprobe", [ + Remote.cmd("ffprobe", [ "-loglevel", "0", "-select_streams", diff --git a/lib/philomena_media/remote.ex b/lib/philomena_media/remote.ex new file mode 100644 index 000000000..518a8613b --- /dev/null +++ b/lib/philomena_media/remote.ex @@ -0,0 +1,18 @@ +defmodule PhilomenaMedia.Remote do + @doc """ + Out-of-process replacement for `System.cmd/2` that calls the requested + command elsewhere, translating file accesses, and returns the result. + """ + def cmd(command, args) do + :ok = Philomena.Native.async_process_command(mediaproc_addr(), command, args) + + receive do + {:command_reply, command_reply} -> + {command_reply.stdout, command_reply.status} + end + end + + defp mediaproc_addr do + Application.get_env(:philomena, :mediaproc_addr) + end +end diff --git a/native/philomena/Cargo.lock b/native/philomena/Cargo.lock index ac6eec147..9959ce29b 100644 --- a/native/philomena/Cargo.lock +++ b/native/philomena/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.1" @@ -17,6 +26,62 @@ dependencies = [ "memchr", ] +[[package]] +name = "anstream" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.60.2", +] + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + [[package]] name = "arbitrary" version = "1.4.2" @@ -26,12 +91,59 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "backtrace" +version = "0.3.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets 0.52.6", +] + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "2.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" + [[package]] name = "bumpalo" version = "3.19.0" @@ -69,6 +181,52 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" +[[package]] +name = "clap" +version = "4.5.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2134bb3ea021b78629caa971416385309e0131b351b25e01dc16fb54e1b5fae" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2ba64afa3c0a6df7fa517765e31314e983f51dda798ffba27b988194fb65dc9" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + [[package]] name = "comrak" version = "0.41.1" @@ -119,18 +277,89 @@ dependencies = [ "syn", ] +[[package]] +name = "educe" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4bd92664bf78c4d3dba9b7cdafce6fa15b13ed3ed16175218196942e99168a8" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "entities" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5320ae4c3782150d900b79807611a59a99fc9a1d61d686faafc24b93fc8d7ca" +[[package]] +name = "enum-ordinalize" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea0dcfa4e54eeb516fe454635a95753ddd39acda650ce703031c6973e315dd5" +dependencies = [ + "enum-ordinalize-derive", +] + +[[package]] +name = "enum-ordinalize-derive" +version = "4.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d28318a75d4aead5c4db25382e8ef717932d0346600cacae6357eb5941bc5ff" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + [[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.0", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "find-msvc-tools" version = "0.1.2" @@ -163,6 +392,95 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "getrandom" version = "0.2.16" @@ -171,9 +489,27 @@ checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.7+wasi-0.2.4", ] +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + [[package]] name = "hashbrown" version = "0.16.0" @@ -197,6 +533,12 @@ dependencies = [ "itoa", ] +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + [[package]] name = "icu_collections" version = "2.0.0" @@ -323,6 +665,23 @@ dependencies = [ "rustversion", ] +[[package]] +name = "io-uring" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itoa" version = "1.0.15" @@ -349,20 +708,60 @@ dependencies = [ "libc", ] +[[package]] +name = "jiff" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde", +] + +[[package]] +name = "jiff-static" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "js-sys" +version = "0.3.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" -version = "0.2.175" +version = "0.2.176" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" +checksum = "58f929b4d672ea937a23a1ab494143d968337a5f47e56d0815df1e0890ddf174" [[package]] name = "libloading" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ "cfg-if", - "windows-targets 0.53.3", + "windows-link 0.2.0", ] [[package]] @@ -374,18 +773,67 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + [[package]] name = "litemap" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +[[package]] +name = "lock_api" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "mediaproc" +version = "0.1.0" +dependencies = [ + "once_cell", + "serde", + "tarpc", + "tokio", +] + +[[package]] +name = "mediaproc_client" +version = "0.1.0" +dependencies = [ + "clap", + "mediaproc", + "tokio", +] + +[[package]] +name = "mediaproc_server" +version = "0.1.0" +dependencies = [ + "clap", + "env_logger", + "futures", + "mediaproc", + "tarpc", + "tempfile", + "tokio", + "tracing", +] + [[package]] name = "memchr" version = "2.7.5" @@ -401,12 +849,98 @@ dependencies = [ "adler2", ] +[[package]] +name = "mio" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + +[[package]] +name = "opentelemetry" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "570074cc999d1a58184080966e5bd3bf3a9a4af650c3b05047c2621e7405cd17" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cefe0543875379e47eb5f1e68ff83f45cc41366a92dfd0d073d513bf68e9a05" + +[[package]] +name = "opentelemetry_sdk" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c627d9f4c9cdc1f21a29ee4bfbd6028fcb8bcf2a857b43f3abdf72c9c862f3" +dependencies = [ + "async-trait", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry", + "percent-encoding", + "rand", + "thiserror", +] + +[[package]] +name = "parking_lot" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.6", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -421,14 +955,63 @@ dependencies = [ "comrak", "http", "jemallocator", + "mediaproc", "once_cell", "regex", "ring", "rustler", + "tokio", "url", "zip", ] +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.3" @@ -438,6 +1021,15 @@ dependencies = [ "zerovec", ] +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.101" @@ -456,6 +1048,51 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", +] + +[[package]] +name = "redox_syscall" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.11.2" @@ -499,10 +1136,29 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.16", "libc", "untrusted", - "windows-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" + +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.0", ] [[package]] @@ -536,11 +1192,23 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" -version = "1.0.225" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6c24dee235d0da097043389623fb913daddf92c76e9f5a1db88607a0bcbd1d" +checksum = "0dca6411025b24b60bfa7ec1fe1f8e710ac09782dca409ee8237ba74b51295fd" dependencies = [ "serde_core", "serde_derive", @@ -548,36 +1216,73 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.225" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "659356f9a0cb1e529b24c01e43ad2bdf520ec4ceaf83047b83ddcc2251f96383" +checksum = "ba2ba63999edb9dac981fb34b3e5c0d111a69b0924e253ed29d83f7c99e966a4" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.225" +version = "1.0.226" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea936adf78b1f766949a4977b91d2f5595825bd6ec079aa9543ad2685fc4516" +checksum = "8db53ae22f34573731bafa1db20f04027b2d25e02d8205921b569171699cdb33" dependencies = [ "proc-macro2", "quote", "syn", ] +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook-registry" +version = "1.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +dependencies = [ + "libc", +] + [[package]] name = "simd-adler32" version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + [[package]] name = "slug" version = "0.1.6" @@ -594,12 +1299,34 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "syn" version = "2.0.106" @@ -622,6 +1349,84 @@ dependencies = [ "syn", ] +[[package]] +name = "tarpc" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14d1be17be018ebeec4c489449adb5ef227746775974c311ce79e09886ef83c7" +dependencies = [ + "anyhow", + "fnv", + "futures", + "humantime", + "opentelemetry", + "opentelemetry-semantic-conventions", + "pin-project", + "rand", + "serde", + "static_assertions", + "tarpc-plugins", + "thiserror", + "tokio", + "tokio-serde", + "tokio-util", + "tracing", + "tracing-opentelemetry", +] + +[[package]] +name = "tarpc-plugins" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26ef4401b013b1f5218ba33ea8f1eddbfcc00ec8db073ef995c192e71f08f027" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +dependencies = [ + "fastrand", + "getrandom 0.3.3", + "once_cell", + "rustix", + "windows-sys 0.61.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + [[package]] name = "tinystr" version = "0.8.1" @@ -647,6 +1452,127 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokio" +version = "1.47.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" +dependencies = [ + "backtrace", + "bytes", + "io-uring", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "slab", + "socket2", + "tokio-macros", + "windows-sys 0.59.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-serde" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf600e7036b17782571dd44fa0a5cea3c82f60db5137f774a325a76a0d6852b" +dependencies = [ + "bincode", + "bytes", + "educe", + "futures-core", + "futures-sink", + "pin-project", + "serde", + "serde_json", +] + +[[package]] +name = "tokio-util" +version = "0.7.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "slab", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc58af5d3f6c5811462cabb3289aec0093f7338e367e5a33d28c0433b3c7360b" +dependencies = [ + "js-sys", + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "tracing", + "tracing-core", + "tracing-subscriber", + "web-time", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" +dependencies = [ + "sharded-slab", + "thread_local", + "tracing-core", +] + [[package]] name = "typed-arena" version = "2.0.2" @@ -698,17 +1624,47 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" -version = "0.2.103" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab10a69fbd0a177f5f649ad4d8d3305499c42bab9aef2f7ff592d0ec8f833819" +checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" dependencies = [ "cfg-if", "once_cell", @@ -719,9 +1675,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.103" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb702423545a6007bbc368fde243ba47ca275e549c8a28617f56f6ba53b1d1c" +checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" dependencies = [ "bumpalo", "log", @@ -733,9 +1689,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.103" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc65f4f411d91494355917b605e1480033152658d71f722a90647f56a70c88a0" +checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -743,9 +1699,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.103" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc003a991398a8ee604a401e194b6b3a39677b3173d6e74495eb51b82e99a32" +checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" dependencies = [ "proc-macro2", "quote", @@ -756,19 +1712,35 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.103" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "293c37f4efa430ca14db3721dfbe48d8c33308096bd44d80ebaa775ab71ba1cf" +checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" dependencies = [ "unicode-ident", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "windows-link" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" +[[package]] +name = "windows-link" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" + [[package]] name = "windows-sys" version = "0.52.0" @@ -778,6 +1750,33 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.3", +] + +[[package]] +name = "windows-sys" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" +dependencies = [ + "windows-link 0.2.0", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -800,7 +1799,7 @@ version = "0.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" dependencies = [ - "windows-link", + "windows-link 0.1.3", "windows_aarch64_gnullvm 0.53.0", "windows_aarch64_msvc 0.53.0", "windows_i686_gnu 0.53.0", @@ -907,6 +1906,12 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + [[package]] name = "writeable" version = "0.6.1" @@ -937,6 +1942,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zerofrom" version = "0.1.6" diff --git a/native/philomena/Cargo.toml b/native/philomena/Cargo.toml index e15c151cd..3c98032d4 100644 --- a/native/philomena/Cargo.toml +++ b/native/philomena/Cargo.toml @@ -9,15 +9,25 @@ name = "philomena" path = "src/lib.rs" crate-type = ["dylib"] +[workspace] +members = [ + "mediaproc", + "mediaproc_client", + "mediaproc_server", +] +default-members = ["mediaproc"] + [dependencies] base64 = "0.22" comrak = { git = "https://github.com/philomena-dev/comrak", branch = "philomena-0.41.1", default-features = false } http = "1.3" jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] } +mediaproc = { path = "./mediaproc" } once_cell = "1.21" regex = "1" ring = "0.17" rustler = "0.37" +tokio = { version = "1.0", features = ["full"] } url = "2.5" zip = { version = "5.1.1", features = ["deflate"], default-features = false } diff --git a/native/philomena/mediaproc/Cargo.toml b/native/philomena/mediaproc/Cargo.toml new file mode 100644 index 000000000..246b707b3 --- /dev/null +++ b/native/philomena/mediaproc/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "mediaproc" +version = "0.1.0" +edition = "2024" + +[dependencies] +once_cell = "1.20" +serde = { version = "1.0", features = ["derive"] } +tarpc = { version = "0.35", features = ["full"] } +tokio = { version = "1.0", features = ["full"] } diff --git a/native/philomena/mediaproc/src/client.rs b/native/philomena/mediaproc/src/client.rs new file mode 100644 index 000000000..f76ef6d5e --- /dev/null +++ b/native/philomena/mediaproc/src/client.rs @@ -0,0 +1,146 @@ +use std::collections::{HashMap, HashSet}; +use std::ffi::OsString; +use std::path::Path; +use std::time::{Duration, Instant}; + +use crate::{CommandReply, ExecuteCommandError, FileMap, MediaProcessorClient}; +use once_cell::sync::Lazy; +use tarpc::context::Context; + +#[derive(Default)] +struct CallParameters { + /// Mapping from replaced name to original name. + replacements: HashMap, + /// List of post-processed arguments. + arguments: Vec, + /// Mapping of replaced name to file contents. + file_map: FileMap, +} + +/// List of file extensions which can be forwarded. +static FORWARDED_EXTS: Lazy> = Lazy::new(|| { + vec![ + "gif", "jpg", "jpeg", "png", "svg", "webm", "webp", "mp4", "icc", + ] + .into_iter() + .map(Into::into) + .collect() +}); + +fn forwarded_ext(path: &Path) -> Option<&str> { + match path.extension() { + Some(ext) if FORWARDED_EXTS.contains(ext) => ext.to_str(), + _ => None, + } +} + +fn create_replacements(arguments: impl Iterator) -> CallParameters { + use std::fs::read; + + // Maps original name to replaced name. + let mut processed = HashMap::::new(); + let mut counter: usize = 0; + + let mut output = CallParameters::default(); + + output.arguments = arguments + .map(|arg| { + let path = Path::new(&arg); + + // Avoid adding additional replacements if the same file is passed multiple times. + if let Some(replaced_name) = processed.get(&arg) { + return replaced_name.clone(); + } + + // Only try things that look like paths. + if !path.is_absolute() { + return arg; + } + + // Don't forward paths that don't exist or can't be read. + let Ok(contents) = read(path) else { + return arg; + }; + + // Only forward extension if extension is in allow list. + let replaced_name = match forwarded_ext(path) { + Some(ext) => format!("{}.{}", counter, ext), + None => format!("{}", counter), + }; + + counter = counter.saturating_add(1); + + processed.insert(arg.clone(), replaced_name.clone()); // original -> replaced + output.replacements.insert(replaced_name.clone(), arg); // replaced -> original + output.file_map.insert(replaced_name.clone(), contents); // replaced -> [contents] + + replaced_name + }) + .collect(); + + output +} + +fn update_replacements( + replacements: HashMap, + file_map: FileMap, +) -> Result<(), ExecuteCommandError> { + use std::fs::write; + + for (replaced_name, contents) in file_map { + let original_name = replacements + .get(&replaced_name) + .ok_or(ExecuteCommandError::InvalidFileMapName)?; + + write(original_name, contents).map_err(|_| ExecuteCommandError::LocalFilesystemError)?; + } + + Ok(()) +} + +fn context_with_1_hour_deadline() -> Context { + let mut context = Context::current(); + context.deadline = Instant::now() + Duration::from_secs(60 * 60); + context +} + +pub async fn execute_command( + client: &MediaProcessorClient, + program: String, + arguments: Vec, +) -> Result { + let call_params = create_replacements(arguments.into_iter()); + let (reply, file_map) = client + .execute_command( + context_with_1_hour_deadline(), + program, + call_params.arguments, + call_params.file_map, + ) + .await + .map_err(|_| ExecuteCommandError::UnknownError)??; + + update_replacements(call_params.replacements, file_map)?; + + Ok(reply) +} + +pub async fn connect_to_socket_server(server_addr: &str) -> Option { + let codec = tarpc::tokio_serde::formats::Bincode::default; + + for addr in tokio::net::lookup_host(server_addr).await.ok()? { + let mut transport = tarpc::serde_transport::tcp::connect(addr, codec); + transport.config_mut().max_frame_length(usize::MAX); + + let transport = match transport.await { + Ok(transport) => transport, + _ => continue, + }; + + return Some( + MediaProcessorClient::new(tarpc::client::Config::default(), transport).spawn(), + ); + } + + None +} diff --git a/native/philomena/mediaproc/src/lib.rs b/native/philomena/mediaproc/src/lib.rs new file mode 100644 index 000000000..a53520ba4 --- /dev/null +++ b/native/philomena/mediaproc/src/lib.rs @@ -0,0 +1,63 @@ +use once_cell::sync::Lazy; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; + +pub mod client; + +#[tarpc::service] +pub trait MediaProcessor { + /// Executes a command on the media processor server. + async fn execute_command( + program: String, + arguments: Vec, + file_map: FileMap, + ) -> Result<(CommandReply, FileMap), ExecuteCommandError>; +} + +/// Errors which can occur during command execution. +#[derive(Debug, Deserialize, Serialize)] +pub enum ExecuteCommandError { + /// Requested program was not allowed to be executed. + UnpermittedProgram(String), + /// Failed to launch program. + ExecutionError, + /// File map name character was not allowed ('..', '/', '\\'). + InvalidFileMapName, + /// Generic filesystem error. + RemoteFilesystemError, + /// Generic filesystem error. + LocalFilesystemError, + /// Unknown error. + UnknownError, +} + +/// Enumeration of permitted program names. +pub static PERMITTED_PROGRAMS: Lazy> = Lazy::new(|| { + vec![ + "ffprobe", + "ffmpeg", + "file", + "gifsicle", + "image-intensities", + "jpegtran", + "magick", + "mediastat", + "mediathumb", + "optipng", + "safe-rsvg-convert", + "svgstat", + ] + .into_iter() + .collect() +}); + +/// Mapping between file name and file contents. +pub type FileMap = HashMap>; + +/// Output reply after command execution has finished. +#[derive(Debug, Deserialize, Serialize)] +pub struct CommandReply { + pub status: u8, + pub stdout: Vec, + pub stderr: Vec, +} diff --git a/native/philomena/mediaproc_client/Cargo.toml b/native/philomena/mediaproc_client/Cargo.toml new file mode 100644 index 000000000..572718160 --- /dev/null +++ b/native/philomena/mediaproc_client/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "mediaproc_client" +version = "0.1.0" +edition = "2024" + +[dependencies] +clap = { version = "4.5", features = ["derive"] } +mediaproc = { path = "../mediaproc" } +tokio = { version = "1.0", features = ["full"] } diff --git a/native/philomena/mediaproc_client/src/main.rs b/native/philomena/mediaproc_client/src/main.rs new file mode 100644 index 000000000..de9974e08 --- /dev/null +++ b/native/philomena/mediaproc_client/src/main.rs @@ -0,0 +1,62 @@ +use std::io::Write; +use std::process::ExitCode; + +use clap::{Parser, Subcommand}; +use mediaproc::MediaProcessorClient; +use mediaproc::client::{connect_to_socket_server, execute_command}; + +#[derive(Parser, Debug)] +#[command(version, about = "RPC Media Processor Client", long_about = None)] +struct Arguments { + /// Server address to connect to, like localhost:1500 + server_addr: String, + + /// Subcommand to execute. + #[command(subcommand)] + invocation_type: InvocationType, +} + +#[derive(Subcommand, Debug)] +enum InvocationType { + /// Execute a command with the given arguments on the remote server. + ExecuteCommand { + /// Program name to execute. + /// + /// One of magick, ffprobe, ffmpeg, file, gifsicle, image-intensities, + /// jpegtran, mediastat, optipng, safe-rsvg-convert. + program: String, + /// Arguments to pass to program. + args: Vec, + }, +} + +#[tokio::main(flavor = "current_thread")] +async fn main() -> ExitCode { + let args = Arguments::parse(); + let client = connect_to_socket_server(&args.server_addr) + .await + .expect("failed to connect to server"); + + match args.invocation_type { + InvocationType::ExecuteCommand { program, args } => { + run_command_client(&client, program, args).await + } + } +} + +async fn run_command_client( + client: &MediaProcessorClient, + program: String, + args: Vec, +) -> ExitCode { + let reply = execute_command(client, program, args).await.unwrap(); + + write_then_drop(std::io::stderr(), reply.stderr); + write_then_drop(std::io::stdout(), reply.stdout); + + reply.status.into() +} + +fn write_then_drop(mut stream: impl Write, data: Vec) { + stream.write_all(&data).unwrap() +} diff --git a/native/philomena/mediaproc_server/Cargo.toml b/native/philomena/mediaproc_server/Cargo.toml new file mode 100644 index 000000000..de05d4ea0 --- /dev/null +++ b/native/philomena/mediaproc_server/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "mediaproc_server" +version = "0.1.0" +edition = "2024" + +[dependencies] +env_logger = "0.11" +clap = { version = "4.5", features = ["derive"] } +futures = "0.3" +mediaproc = { path = "../mediaproc" } +tarpc = { version = "0.35", features = ["full"] } +tempfile = "3" +tokio = { version = "1.0", features = ["full"] } +tracing = "0.1" diff --git a/native/philomena/mediaproc_server/src/command_server.rs b/native/philomena/mediaproc_server/src/command_server.rs new file mode 100644 index 000000000..0108ef2cb --- /dev/null +++ b/native/philomena/mediaproc_server/src/command_server.rs @@ -0,0 +1,63 @@ +use std::collections::HashSet; +use std::os::unix::process::ExitStatusExt; + +use mediaproc::{CommandReply, ExecuteCommandError, FileMap, PERMITTED_PROGRAMS}; +use tokio::process::Command; + +fn validate_name(name: &str) -> Result<(), ExecuteCommandError> { + if name == "." || name.contains("..") || name.contains('/') || name.contains('\\') { + return Err(ExecuteCommandError::InvalidFileMapName); + } + + Ok(()) +} + +pub async fn execute_command( + program: String, + arguments: Vec, + file_map: FileMap, +) -> Result<(CommandReply, FileMap), ExecuteCommandError> { + use std::fs::{read, write}; + + // Check program name. + if !PERMITTED_PROGRAMS.contains(&program.as_ref()) { + return Err(ExecuteCommandError::UnpermittedProgram(program)); + } + + // Create a new temporary directory which we will work in. + let dir = tempfile::tempdir().map_err(|_| ExecuteCommandError::RemoteFilesystemError)?; + + // Verify and write out all files. + let mut files = HashSet::::new(); + for (name, contents) in file_map { + validate_name(&name)?; + files.insert(name.clone()); + + let path = dir.path().join(name); + write(path, contents).map_err(|_| ExecuteCommandError::RemoteFilesystemError)?; + } + + // Run the command. + let output = Command::new(program) + .args(arguments) + .current_dir(dir.path()) + .output() + .await + .map_err(|_| ExecuteCommandError::ExecutionError)?; + + // Read back all files. + let mut file_map = FileMap::new(); + for name in files { + let path = dir.path().join(name.clone()); + let contents = read(path).map_err(|_| ExecuteCommandError::RemoteFilesystemError)?; + file_map.insert(name, contents); + } + + let reply = CommandReply { + status: output.status.into_raw() as u8, + stdout: output.stdout, + stderr: output.stderr, + }; + + Ok((reply, file_map)) +} diff --git a/native/philomena/mediaproc_server/src/main.rs b/native/philomena/mediaproc_server/src/main.rs new file mode 100644 index 000000000..0e4541570 --- /dev/null +++ b/native/philomena/mediaproc_server/src/main.rs @@ -0,0 +1,69 @@ +use std::net::SocketAddr; + +use clap::Parser; +use futures::{Future, StreamExt, future}; +use mediaproc::{CommandReply, ExecuteCommandError, FileMap, MediaProcessor}; +use tarpc::context; +use tarpc::server::Channel; + +mod command_server; +mod signal; + +#[derive(Parser, Debug)] +#[command(version, about = "RPC Media Processor Server", long_about = None)] +struct Arguments { + /// Socket address to bind to, like 127.0.0.1:1500 + server_addr: SocketAddr, +} + +#[derive(Clone)] +struct MediaProcessorServer; + +impl MediaProcessor for MediaProcessorServer { + async fn execute_command( + self, + _: context::Context, + program: String, + arguments: Vec, + file_map: FileMap, + ) -> Result<(CommandReply, FileMap), ExecuteCommandError> { + command_server::execute_command(program, arguments, file_map).await + } +} + +fn main() { + env_logger::init(); + + let args = Arguments::parse(); + + serve(&args); +} + +async fn spawn(fut: impl Future + Send + 'static) { + tokio::spawn(fut); +} + +#[tokio::main] +async fn serve(args: &Arguments) { + signal::install_handlers(); + + let codec = tarpc::tokio_serde::formats::Bincode::default; + let mut listener = tarpc::serde_transport::tcp::listen(args.server_addr, codec) + .await + .unwrap(); + + listener.config_mut().max_frame_length(usize::MAX); + listener + // Ignore accept errors. + .filter_map(|r| future::ready(r.ok())) + .map(tarpc::server::BaseChannel::with_defaults) + .map(|channel| { + tokio::spawn( + channel + .execute(MediaProcessorServer.serve()) + .for_each(spawn), + ); + }) + .collect() + .await +} diff --git a/native/philomena/mediaproc_server/src/signal.rs b/native/philomena/mediaproc_server/src/signal.rs new file mode 100644 index 000000000..0d879b57d --- /dev/null +++ b/native/philomena/mediaproc_server/src/signal.rs @@ -0,0 +1,15 @@ +use tokio::signal::unix::{SignalKind, signal}; + +pub fn install_handlers() { + let mut sigterm = signal(SignalKind::terminate()).unwrap(); + let mut sigint = signal(SignalKind::interrupt()).unwrap(); + + tokio::spawn(async move { + tokio::select! { + _ = sigterm.recv() => tracing::debug!("Received SIGTERM"), + _ = sigint.recv() => tracing::debug!("Received SIGINT"), + }; + + std::process::exit(1); + }); +} diff --git a/native/philomena/src/asyncnif.rs b/native/philomena/src/asyncnif.rs new file mode 100644 index 000000000..faf845844 --- /dev/null +++ b/native/philomena/src/asyncnif.rs @@ -0,0 +1,26 @@ +use once_cell::sync::Lazy; +use rustler::{Atom, Env, OwnedEnv, Term}; +use std::future::Future; +use std::marker::Send; +use tokio::runtime::Runtime; + +static RUNTIME: Lazy = Lazy::new(|| Runtime::new().unwrap()); + +pub fn call_async(caller_env: Env, fut: F, w: W) -> Atom +where + F: Future + Send + 'static, + W: for<'a> FnOnce(Env<'a>, T) -> Term<'a>, + W: Send + 'static, +{ + let pid = caller_env.pid(); + + RUNTIME.spawn(async move { + let output = fut.await; + let owned_env = OwnedEnv::new(); + owned_env.run(move |env| { + let _ = env.send(&pid, w(env, output)); + }); + }); + + rustler::types::atom::ok() +} diff --git a/native/philomena/src/lib.rs b/native/philomena/src/lib.rs index 6f7f72b2b..f2f5893ea 100644 --- a/native/philomena/src/lib.rs +++ b/native/philomena/src/lib.rs @@ -1,10 +1,12 @@ use jemallocator::Jemalloc; -use rustler::{Atom, Binary}; +use rustler::{Atom, Binary, Env}; use std::collections::HashMap; +mod asyncnif; mod camo; mod domains; mod markdown; +mod remote; #[cfg(test)] mod tests; mod zip; @@ -35,6 +37,19 @@ fn camo_image_url(input: &str) -> String { camo::image_url_careful(input) } +// Remote NIF wrappers. + +#[rustler::nif] +fn async_process_command( + env: Env, + server_addr: String, + program: String, + arguments: Vec, +) -> Atom { + let fut = remote::process_command(server_addr, program, arguments); + asyncnif::call_async(env, fut, remote::with_env) +} + // Zip NIF wrappers. #[rustler::nif] diff --git a/native/philomena/src/remote.rs b/native/philomena/src/remote.rs new file mode 100644 index 000000000..dd30c75f6 --- /dev/null +++ b/native/philomena/src/remote.rs @@ -0,0 +1,66 @@ +use mediaproc::CommandReply; +use mediaproc::client::{connect_to_socket_server, execute_command}; +use rustler::{Encoder, Env, NifStruct, OwnedBinary, Term, atoms}; + +atoms! { + nil, + command_reply, +} + +#[derive(NifStruct)] +#[module = "Elixir.Philomena.Native.CommandReply"] +struct CommandReply_<'a> { + stdout: Term<'a>, + stderr: Term<'a>, + status: u8, +} + +fn binary_or_nil<'a>(env: Env<'a>, data: Vec) -> Term<'a> { + match OwnedBinary::new(data.len()) { + Some(mut binary) => { + binary.copy_from_slice(&data); + binary.release(env).to_term(env) + } + None => nil().to_term(env), + } +} + +pub async fn process_command( + server_addr: String, + program: String, + arguments: Vec, +) -> CommandReply { + let client = match connect_to_socket_server(&server_addr).await { + Some(client) => client, + None => { + return CommandReply { + stdout: vec![], + stderr: "failed to connect to server".into(), + status: 255, + }; + } + }; + + match execute_command(&client, program, arguments).await { + Ok(reply) => reply, + Err(err) => CommandReply { + stdout: vec![], + stderr: format!("failed to execute command: {err:?}").into(), + status: 255, + }, + } +} + +/// Converts the response into a {:command_reply, %CommandReply{...}} message +/// which gets sent back to the caller. +pub fn with_env<'a>(env: Env<'a>, r: CommandReply) -> Term<'a> { + ( + command_reply(), + CommandReply_ { + stdout: binary_or_nil(env, r.stdout), + stderr: binary_or_nil(env, r.stderr), + status: r.status, + }, + ) + .encode(env) +} diff --git a/priv/repo/seeds_development.exs b/priv/repo/seeds_development.exs index 35f157c1c..1492e341d 100644 --- a/priv/repo/seeds_development.exs +++ b/priv/repo/seeds_development.exs @@ -46,7 +46,7 @@ request_attributes = [ IO.puts "---- Generating images" for image_def <- resources["remote_images"] do - file = Briefly.create!() + file = Briefly.create!(extname: ".png") now = DateTime.utc_now() |> DateTime.to_unix(:microsecond) IO.puts "Fetching #{image_def["url"]} ..."