Fix x265 SIGSEGV during h265 encoding

Dockerfile

@@ -13,6 +13,7 @@ COPY sample.mp4 app/sample.mp4
 COPY sample-av1.webm app/sample-av1.webm
 COPY aac.patch app/aac.patch
 COPY hevc_ps.patch app/hevc_ps.patch
+COPY x265-api.patch app/x265-api.patch
 
 RUN apk add curl 
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y

Dockerfile-aws

@@ -13,6 +13,7 @@ COPY sample.mp4 app/sample.mp4
 COPY sample-av1.webm app/sample-av1.webm
 COPY aac.patch app/aac.patch
 COPY hevc_ps.patch app/hevc_ps.patch
+COPY x265-api.patch app/x265-api.patch
 
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
 RUN source "$HOME/.cargo/env"

compile-ffmpeg.mjs

@@ -159,6 +159,9 @@ if (fs.existsSync("ffmpeg")) {
   execSync("git apply hevc_ps.patch --directory ffmpeg", {
     stdio: "inherit",
   });
+  execSync("git apply x265-api.patch --directory ffmpeg", {
+    stdio: "inherit",
+  });
 } else {
   execSync("git clone https://github.com/ffmpeg/ffmpeg.git", {
     stdio: "inherit",
@@ -173,6 +176,9 @@ if (fs.existsSync("ffmpeg")) {
   execSync("git apply hevc_ps.patch --directory ffmpeg", {
     stdio: "inherit",
   });
+  execSync("git apply x265-api.patch --directory ffmpeg", {
+    stdio: "inherit",
+  });
 }
 
 const extraCFlags = [

test-ffmpeg.mjs

@@ -101,6 +101,28 @@ const exit5 = spawnSync(
 );
 assert(exit5.status === 0);
 
+const exit_x265 = spawnSync(
+  ffmpegBinary,
+  [
+    "-i",
+    "sample.mp4",
+    "-t",
+    "1",
+    "-c:v",
+    "libx265",
+    "out-test-x265.mp4",
+    "-y",
+  ],
+  {
+    env,
+    stdio: "inherit",
+  }
+);
+if (exit_x265.status !== 0) {
+  console.log("x265 encoding test failed with status", exit_x265.status, "signal", exit_x265.signal);
+}
+assert(exit_x265.status === 0);
+
 const exit6 = spawnSync(
   ffmpegBinary,
   [

x265-api.patch

@@ -0,0 +1,31 @@
+diff --git a/libavcodec/libx265.c b/libavcodec/libx265.c
+index 38f0ab6b15..5d4fc43e92 100644
+--- a/libavcodec/libx265.c
++++ b/libavcodec/libx265.c
+@@ -661,7 +661,7 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
+ {
+     libx265Context *ctx = avctx->priv_data;
+     x265_picture x265pic;
+-#if X265_BUILD >= 210
++#if (X265_BUILD >= 210) && (X265_BUILD < 213)
+     x265_picture x265pic_layers_out[MAX_SCALABLE_LAYERS];
+     x265_picture* x265pic_lyrptr_out[MAX_SCALABLE_LAYERS];
+ #else
+@@ -805,7 +805,7 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
+ #endif
+     }
+
+-#if X265_BUILD >= 210
++#if (X265_BUILD >= 210) && (X265_BUILD < 213)
+     for (i = 0; i < MAX_SCALABLE_LAYERS; i++)
+         x265pic_lyrptr_out[i] = &x265pic_layers_out[i];
+
+@@ -844,7 +844,7 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
+             pkt->flags |= AV_PKT_FLAG_KEY;
+     }
+
+-#if X265_BUILD >= 210
++#if (X265_BUILD >= 210) && (X265_BUILD < 213)
+     x265pic_out = x265pic_lyrptr_out[0];
+ #else
+     x265pic_out = &x265pic_solo_out;