diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml index cd57262..910c844 100644 --- a/.github/workflows/reproducible-build.yml +++ b/.github/workflows/reproducible-build.yml @@ -29,7 +29,7 @@ jobs: - name: Build release APK (first) run: | - ./gradlew clean :app:assembleRelease \ + ./gradlew clean test :app:assembleRelease \ -PskipSigning \ --no-daemon \ --max-workers=1 \ diff --git a/app/build.gradle b/app/build.gradle index 0ef4d08..a8f1e37 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -147,5 +147,6 @@ dependencies { // BouncyCastle for certificate generation implementation 'org.bouncycastle:bcprov-jdk15on:1.70' implementation 'org.bouncycastle:bcpkix-jdk15on:1.70' -} + testImplementation 'junit:junit:4.13.2' +} diff --git a/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/StreamingService.kt b/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/StreamingService.kt index b39222c..db8ddc2 100644 --- a/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/StreamingService.kt +++ b/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/StreamingService.kt @@ -479,7 +479,7 @@ class StreamingService : LifecycleService() { val enc = H264HardwareEncoder(sz.width, sz.height, fpsCoerced, H264HardwareEncoder.bitrateFor(sz.width, sz.height), true) { d, k -> h264StreamingEncoder?.broadcastH264(d, k) } h264StreamingEncoder?.setEncoder(enc) streamingServerHelper?.resetH264Wait() - return CameraGlPipe(enc.inputSurface!!, sz.width, sz.height).also { it.start(); glPipe = it } + return CameraGlPipe(enc.inputSurface!!, sz.width, sz.height, fpsCoerced).also { it.start(); glPipe = it } } private fun applyStored(b: CaptureBackend) { diff --git a/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/CameraGlPipe.kt b/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/CameraGlPipe.kt index 242791b..669f5dd 100644 --- a/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/CameraGlPipe.kt +++ b/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/CameraGlPipe.kt @@ -21,11 +21,17 @@ import java.util.concurrent.atomic.AtomicBoolean * (which the LEGACY HAL drives fine), a GL thread draws it into the encoder's input Surface — no CPU * copy, so the HW encoder sustains 1080p30. Hand [surfaceTexture] to Camera1.setPreviewTexture. */ -class CameraGlPipe(private val encoderSurface: Surface, private val width: Int, private val height: Int) { +class CameraGlPipe( + private val encoderSurface: Surface, + private val width: Int, + private val height: Int, + targetFps: Int +) { private var thread: Thread? = null @Volatile private var running = true private val ready = CountDownLatch(1) private val frameAvailable = AtomicBoolean(false) + private val frameRateLimiter = FrameRateLimiter(targetFps) private var eglDisplay: EGLDisplay = EGL14.EGL_NO_DISPLAY private var eglContext: EGLContext = EGL14.EGL_NO_CONTEXT @@ -60,10 +66,13 @@ class CameraGlPipe(private val encoderSurface: Surface, private val width: Int, while (running) { if (frameAvailable.compareAndSet(true, false)) { surfaceTexture.updateTexImage() - surfaceTexture.getTransformMatrix(stMatrix) - drawFrame() - EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTexture.timestamp) - EGL14.eglSwapBuffers(eglDisplay, eglSurface) + val timestamp = surfaceTexture.timestamp.takeIf { it > 0L } ?: System.nanoTime() + if (frameRateLimiter.shouldEmit(timestamp)) { + surfaceTexture.getTransformMatrix(stMatrix) + drawFrame() + EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, timestamp) + EGL14.eglSwapBuffers(eglDisplay, eglSurface) + } } else { Thread.sleep(2) } @@ -157,6 +166,7 @@ class CameraGlPipe(private val encoderSurface: Surface, private val width: Int, fun stop() { running = false try { thread?.join(500) } catch (_: Exception) {} + frameRateLimiter.reset() } companion object { private const val TAG = "CameraGlPipe" } diff --git a/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/FrameRateLimiter.kt b/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/FrameRateLimiter.kt new file mode 100644 index 0000000..8be99d1 --- /dev/null +++ b/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/FrameRateLimiter.kt @@ -0,0 +1,29 @@ +package com.github.digitallyrefined.androidipcamera.helpers + +/** Timestamp-based frame pacing for camera surfaces that produce faster than the encoder target. */ +internal class FrameRateLimiter(targetFps: Int) { + private val intervalNs = 1_000_000_000L / targetFps.coerceIn(1, 60) + private var nextFrameNs = Long.MIN_VALUE + private var lastTimestampNs = Long.MIN_VALUE + + fun shouldEmit(timestampNs: Long): Boolean { + if (lastTimestampNs != Long.MIN_VALUE && timestampNs < lastTimestampNs) reset() + lastTimestampNs = timestampNs + + if (nextFrameNs == Long.MIN_VALUE) { + nextFrameNs = timestampNs + intervalNs + return true + } + if (timestampNs < nextFrameNs) return false + + do { + nextFrameNs += intervalNs + } while (nextFrameNs <= timestampNs) + return true + } + + fun reset() { + nextFrameNs = Long.MIN_VALUE + lastTimestampNs = Long.MIN_VALUE + } +} diff --git a/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/H264Bitstream.kt b/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/H264Bitstream.kt new file mode 100644 index 0000000..58135ad --- /dev/null +++ b/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/H264Bitstream.kt @@ -0,0 +1,237 @@ +package com.github.digitallyrefined.androidipcamera.helpers + +import java.io.ByteArrayOutputStream + +/** Pure helpers for turning MediaCodec AVC output into self-contained Annex-B access units. */ +internal object H264Bitstream { + private val startCode = byteArrayOf(0, 0, 0, 1) + + data class Normalized(val bytes: ByteArray, val nalLengthSize: Int? = null) + + data class NalRange( + val start: Int, + val end: Int, + val type: Int, + val startCodeStart: Int, + val startCodeLength: Int + ) + + /** + * Accept Annex-B, AVCDecoderConfigurationRecord (avcC), a length-prefixed access unit, or one + * raw NAL unit. The result always uses four-byte Annex-B start codes. + */ + fun normalize(input: ByteArray, nalLengthSizeHint: Int? = null): Normalized? { + if (input.isEmpty()) return null + + annexBRanges(input)?.let { ranges -> + val alreadyCanonical = ranges.first().startCodeStart == 0 && + ranges.all { it.startCodeLength == 4 } + return Normalized(if (alreadyCanonical) input else joinRanges(input, ranges)) + } + + parseAvcConfiguration(input)?.let { return it } + + // AVC normally uses four-byte lengths. Other sizes are only safe to infer from avcC, + // otherwise a raw NAL header can be mistaken for a one-byte length. + val lengthSizes = if (nalLengthSizeHint != null && nalLengthSizeHint in 1..4) { + listOf(nalLengthSizeHint) + } else { + listOf(4) + } + for (lengthSize in lengthSizes) { + parseLengthPrefixed(input, lengthSize)?.let { nals -> + return Normalized(joinNals(nals), lengthSize) + } + } + + if (!isValidNal(input)) return null + return Normalized(joinNals(listOf(input))) + } + + fun nalRanges(annexB: ByteArray): List = annexBRanges(annexB) ?: emptyList() + + fun containsNalType(annexB: ByteArray, type: Int): Boolean = + nalRanges(annexB).any { it.type == type } + + fun containsVcl(annexB: ByteArray): Boolean = + nalRanges(annexB).any { it.type in 1..5 } + + fun joinNals(nals: List): ByteArray { + val size = nals.sumOf { startCode.size + it.size } + val out = ByteArrayOutputStream(size) + for (nal in nals) { + out.write(startCode) + out.write(nal) + } + return out.toByteArray() + } + + private fun joinRanges(input: ByteArray, ranges: List): ByteArray = + joinNals(ranges.map { input.copyOfRange(it.start, it.end) }) + + private fun annexBRanges(input: ByteArray): List? { + val first = findStartCode(input, 0) ?: return null + if (first.first > 3 || (0 until first.first).any { input[it].toInt() != 0 }) return null + + val ranges = mutableListOf() + var marker: Pair? = first + while (marker != null) { + val nalStart = marker.first + marker.second + val next = findStartCode(input, nalStart) + val nalEnd = next?.first ?: input.size + if (nalEnd <= nalStart) return null + if (!isValidNalHeader(input[nalStart])) return null + ranges += NalRange( + nalStart, + nalEnd, + input[nalStart].toInt() and 0x1f, + marker.first, + marker.second + ) + marker = next + } + return ranges.takeIf { it.isNotEmpty() } + } + + private fun findStartCode(input: ByteArray, from: Int): Pair? { + var i = from.coerceAtLeast(0) + while (i + 2 < input.size) { + if (input[i].toInt() == 0 && input[i + 1].toInt() == 0) { + if (input[i + 2].toInt() == 1) return i to 3 + if (i + 3 < input.size && input[i + 2].toInt() == 0 && input[i + 3].toInt() == 1) { + return i to 4 + } + } + i++ + } + return null + } + + private fun parseLengthPrefixed(input: ByteArray, lengthSize: Int): List? { + if (lengthSize !in 1..4 || input.size <= lengthSize) return null + val nals = mutableListOf() + var offset = 0 + while (offset < input.size) { + if (input.size - offset < lengthSize) return null + var length = 0L + repeat(lengthSize) { + length = (length shl 8) or (input[offset++].toLong() and 0xff) + } + if (length <= 0 || length > Int.MAX_VALUE || length > input.size - offset) return null + val end = offset + length.toInt() + val nal = input.copyOfRange(offset, end) + if (!isValidNal(nal)) return null + nals += nal + offset = end + } + return nals.takeIf { it.isNotEmpty() } + } + + /** Parse the SPS/PPS part of ISO/IEC 14496-15 AVCDecoderConfigurationRecord. */ + private fun parseAvcConfiguration(input: ByteArray): Normalized? { + if (input.size < 7 || input[0].toInt() != 1) return null + val lengthSize = (input[4].toInt() and 0x03) + 1 + val nals = mutableListOf() + var offset = 6 + val spsCount = input[5].toInt() and 0x1f + if (spsCount == 0) return null + + repeat(spsCount) { + val parsed = readTwoByteLengthNal(input, offset) ?: return null + if ((parsed.first[0].toInt() and 0x1f) != 7) return null + nals += parsed.first + offset = parsed.second + } + if (offset >= input.size) return null + val ppsCount = input[offset++].toInt() and 0xff + if (ppsCount == 0) return null + repeat(ppsCount) { + val parsed = readTwoByteLengthNal(input, offset) ?: return null + if ((parsed.first[0].toInt() and 0x1f) != 8) return null + nals += parsed.first + offset = parsed.second + } + return Normalized(joinNals(nals), lengthSize) + } + + private fun readTwoByteLengthNal(input: ByteArray, offset: Int): Pair? { + if (offset + 2 > input.size) return null + val length = ((input[offset].toInt() and 0xff) shl 8) or (input[offset + 1].toInt() and 0xff) + val start = offset + 2 + val end = start + length + if (length <= 0 || end > input.size) return null + val nal = input.copyOfRange(start, end) + if (!isValidNal(nal)) return null + return nal to end + } + + private fun isValidNal(nal: ByteArray): Boolean { + return nal.isNotEmpty() && isValidNalHeader(nal[0]) + } + + private fun isValidNalHeader(header: Byte): Boolean = + (header.toInt() and 0x80) == 0 && (header.toInt() and 0x1f) in 1..31 +} + +/** Caches codec parameter sets and makes each emitted IDR independently decodable. */ +internal class H264ParameterSetCache { + data class AccessUnit(val bytes: ByteArray, val isKeyframe: Boolean) + + private var sps: List = emptyList() + private var pps: List = emptyList() + private var nalLengthSize: Int? = null + + fun prepare(input: ByteArray, keyframeFlag: Boolean): AccessUnit? { + val normalized = H264Bitstream.normalize(input, nalLengthSize) ?: return null + if (normalized.nalLengthSize != null) nalLengthSize = normalized.nalLengthSize + + val ranges = H264Bitstream.nalRanges(normalized.bytes) + val newSps = ranges.filter { it.type == 7 } + .map { normalized.bytes.copyOfRange(it.start, it.end) } + val newPps = ranges.filter { it.type == 8 } + .map { normalized.bytes.copyOfRange(it.start, it.end) } + if (newSps.isNotEmpty()) sps = newSps + if (newPps.isNotEmpty()) pps = newPps + + // Codec-config buffers contain no picture. Cache them, but do not send them as a frame. + if (ranges.none { it.type in 1..5 }) return null + + val isIdr = ranges.any { it.type == 5 } + if (!keyframeFlag && !isIdr) return AccessUnit(normalized.bytes, false) + + if (!hasCompleteParameters()) { + // Keep newly connected clients waiting rather than accepting an undecodable IDR. + return AccessUnit(normalized.bytes, false) + } + + val nals = ArrayList(ranges.size + sps.size + pps.size) + // Access-unit delimiter belongs first; parameter sets belong before SEI and slices. + for (range in ranges.filter { it.type == 9 }) { + nals += normalized.bytes.copyOfRange(range.start, range.end) + } + nals += sps + nals += pps + for (range in ranges) { + if (range.type != 7 && range.type != 8 && range.type != 9) { + nals += normalized.bytes.copyOfRange(range.start, range.end) + } + } + return AccessUnit(H264Bitstream.joinNals(nals), true) + } + + fun hasCompleteParameters(): Boolean = sps.isNotEmpty() && pps.isNotEmpty() +} + +/** Accumulates MediaCodec buffers marked BUFFER_FLAG_PARTIAL_FRAME. */ +internal class H264AccessUnitAssembler { + private val pending = ByteArrayOutputStream() + + fun append(bytes: ByteArray, isPartial: Boolean): ByteArray? { + if (pending.size() == 0 && !isPartial) return bytes + pending.write(bytes) + if (isPartial) return null + return pending.toByteArray().also { pending.reset() } + } + + fun reset() = pending.reset() +} diff --git a/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/H264HardwareEncoder.kt b/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/H264HardwareEncoder.kt index 660f588..91b4282 100644 --- a/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/H264HardwareEncoder.kt +++ b/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/H264HardwareEncoder.kt @@ -5,7 +5,6 @@ import android.media.MediaCodec import android.media.MediaCodecInfo import android.media.MediaCodecList import android.media.MediaFormat -import android.os.Build import android.os.Bundle import android.util.Log import android.view.Surface @@ -34,6 +33,9 @@ class H264HardwareEncoder( @Volatile private var running = true private var thread: Thread? = null private var semiPlanar = true // byte-buffer layout: NV12 (interleaved UV) vs I420 (planar U then V) + private val parameterSets = H264ParameterSetCache() + private val accessUnitAssembler = H264AccessUnitAssembler() + private var partialFlags = 0 init { val colorFormat = if (useSurface) MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface @@ -44,9 +46,6 @@ class H264HardwareEncoder( setInteger(MediaFormat.KEY_FRAME_RATE, frameRate) setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1) try { setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR) } catch (_: Exception) {} - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - setInteger(MediaFormat.KEY_PREPEND_HEADER_TO_SYNC_FRAMES, 1) - } } codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) if (useSurface) inputSurface = codec.createInputSurface() @@ -208,14 +207,36 @@ class H264HardwareEncoder( try { while (running) { val idx = synchronized(codecLock) { codec.dequeueOutputBuffer(info, 10000) } - if (idx >= 0) { - val buf = synchronized(codecLock) { codec.getOutputBuffer(idx) } - if (buf != null && info.size > 0) { - buf.position(info.offset); buf.limit(info.offset + info.size) - val data = ByteArray(info.size); buf.get(data) - onNal(data, (info.flags and MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0) + if (idx == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + val format = synchronized(codecLock) { codec.outputFormat } + cacheCodecSpecificData(format) + } else if (idx >= 0) { + var prepared: H264ParameterSetCache.AccessUnit? = null + try { + val buf = synchronized(codecLock) { codec.getOutputBuffer(idx) } + if (buf != null && info.size > 0) { + buf.position(info.offset) + buf.limit(info.offset + info.size) + val data = ByteArray(info.size) + buf.get(data) + + partialFlags = partialFlags or info.flags + val isPartial = (info.flags and MediaCodec.BUFFER_FLAG_PARTIAL_FRAME) != 0 + val accessUnit = accessUnitAssembler.append(data, isPartial) + if (accessUnit != null) { + val flags = partialFlags + partialFlags = 0 + prepared = parameterSets.prepare( + accessUnit, + (flags and MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0 + ) + } + } + } finally { + synchronized(codecLock) { codec.releaseOutputBuffer(idx, false) } } - synchronized(codecLock) { codec.releaseOutputBuffer(idx, false) } + // Never retain a MediaCodec output buffer while network work is scheduled. + prepared?.let { onNal(it.bytes, it.isKeyframe) } } } } catch (e: Exception) { @@ -223,13 +244,31 @@ class H264HardwareEncoder( } } + private fun cacheCodecSpecificData(format: MediaFormat) { + for (key in arrayOf("csd-0", "csd-1")) { + try { + val source = format.getByteBuffer(key)?.duplicate() ?: continue + val data = ByteArray(source.remaining()) + source.get(data) + parameterSets.prepare(data, false) + } catch (e: Exception) { + Log.w(TAG, "Unable to read $key: ${e.message}") + } + } + if (!parameterSets.hasCompleteParameters()) { + Log.w(TAG, "Output format did not contain complete AVC SPS/PPS") + } + } + fun stop() { + running = false + try { thread?.join(500) } catch (_: Exception) {} synchronized(codecLock) { - running = false - try { thread?.join(500) } catch (_: Exception) {} try { codec.stop() } catch (_: Exception) {} try { codec.release() } catch (_: Exception) {} try { inputSurface?.release() } catch (_: Exception) {} + accessUnitAssembler.reset() + partialFlags = 0 } } diff --git a/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/H264StreamingEncoder.kt b/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/H264StreamingEncoder.kt index 9445f01..ae9ea44 100644 --- a/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/H264StreamingEncoder.kt +++ b/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/H264StreamingEncoder.kt @@ -5,9 +5,11 @@ import android.util.Log import android.util.Size import androidx.camera.core.ImageProxy import androidx.preference.PreferenceManager -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch +import java.util.concurrent.ArrayBlockingQueue +import java.util.concurrent.RejectedExecutionException +import java.util.concurrent.ThreadPoolExecutor +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong /** * H.264 streaming encoder implementation. @@ -20,6 +22,7 @@ class H264StreamingEncoder( ) : StreamingEncoder { companion object { private const val TAG = "H264StreamingEncoder" + private const val WRITER_QUEUE_CAPACITY = 30 } var h264HardwareEncoder: H264HardwareEncoder? = null @@ -27,11 +30,21 @@ class H264StreamingEncoder( private var glPipe: CameraGlPipe? = null private var backend: CaptureBackend? = null private var captureRunning = false + private val writerGeneration = AtomicLong() + private val networkWriter = ThreadPoolExecutor( + 1, + 1, + 30L, + TimeUnit.SECONDS, + ArrayBlockingQueue(WRITER_QUEUE_CAPACITY), + { runnable -> Thread(runnable, "H264NetworkWriter").apply { isDaemon = true } } + ).apply { allowCoreThreadTimeOut(true) } override fun processFrame(image: ImageProxy) { try { var enc = h264HardwareEncoder if (enc == null || enc.width != image.width || enc.height != image.height) { + invalidatePendingWrites() enc?.stop() val prefs = PreferenceManager.getDefaultSharedPreferences(context) val fps = prefs.getString("stream_fps", "30")?.toIntOrNull() ?: 30 @@ -71,6 +84,7 @@ class H264StreamingEncoder( glPipe = null h264HardwareEncoder?.stop() h264HardwareEncoder = null + invalidatePendingWrites() captureRunning = false } @@ -83,32 +97,63 @@ class H264StreamingEncoder( */ fun broadcastH264(data: ByteArray, isKey: Boolean) { val helper = streamingServerHelper ?: return - val list = helper.getH264Clients() - if (list.isEmpty()) return - - // Run network I/O on IO dispatcher to avoid main-thread network operations - CoroutineScope(Dispatchers.IO).launch { - val toRemove = mutableListOf() - for (c in list) { - try { - if (c.waitingKey) { - if (!isKey) continue - c.waitingKey = false - } - c.outputStream.write(data) - c.outputStream.flush() - } catch (e: Exception) { - toRemove.add(c) - } + val clients = helper.getH264Clients() + if (clients.isEmpty()) return + val generation = writerGeneration.get() + + // One bounded worker preserves access-unit ordering without ever blocking MediaCodec. + if (networkWriter.queue.remainingCapacity() == 0) { + disconnectStalledClients(helper) + return + } + try { + // Capture recipients now so queued frames from a disconnected viewer can never be + // delivered as a stale burst to a viewer that connects later. + networkWriter.execute { writeH264(data, isKey, clients, generation) } + } catch (_: RejectedExecutionException) { + disconnectStalledClients(helper) + } + } + + private fun disconnectStalledClients(helper: StreamingServerHelper) { + invalidatePendingWrites() + val stalled = helper.getH264Clients() + Log.w(TAG, "H.264 writer stalled; disconnecting ${stalled.size} client(s)") + onLog("H.264 client disconnected because its network writer stalled") + stalled.forEach { helper.removeH264Client(it) } + } + + private fun writeH264( + data: ByteArray, + isKey: Boolean, + clients: List, + generation: Long + ) { + if (generation != writerGeneration.get()) return + val helper = streamingServerHelper ?: return + val toRemove = mutableListOf() + for (c in clients) { + if (generation != writerGeneration.get()) return + try { + val wasWaitingForKey = c.waitingKey + if (wasWaitingForKey && !isKey) continue + c.outputStream.write(data) + c.outputStream.flush() + // A restart may happen while a socket write is blocked. Never let an access unit + // from the old encoder generation mark the new stream as synchronized. + if (wasWaitingForKey && generation == writerGeneration.get()) c.waitingKey = false + } catch (e: Exception) { + toRemove.add(c) } - toRemove.forEach { helper.removeH264Client(it) } } + toRemove.forEach { helper.removeH264Client(it) } } /** * Reset H.264 wait state for all clients (called when encoder restarts). */ fun resetWait() { + invalidatePendingWrites() streamingServerHelper?.resetH264Wait() } @@ -128,6 +173,12 @@ class H264StreamingEncoder( * Set the H.264 hardware encoder (used by Camera1 backend). */ fun setEncoder(encoder: H264HardwareEncoder) { + invalidatePendingWrites() h264HardwareEncoder = encoder } + + private fun invalidatePendingWrites() { + writerGeneration.incrementAndGet() + networkWriter.queue.clear() + } } diff --git a/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/StreamingServerHelper.kt b/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/StreamingServerHelper.kt index df2a8e1..dd4516e 100644 --- a/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/StreamingServerHelper.kt +++ b/app/src/main/kotlin/com/github/digitallyrefined/androidipcamera/helpers/StreamingServerHelper.kt @@ -38,6 +38,7 @@ import java.nio.charset.StandardCharsets import java.nio.ByteBuffer import java.nio.ByteOrder import java.security.KeyStore +import java.security.SecureRandom import java.util.Locale import kotlin.math.ceil import kotlin.math.floor @@ -81,9 +82,12 @@ class StreamingServerHelper( private var isStarting = false private val clients = CopyOnWriteArrayList() private val h264Clients = CopyOnWriteArrayList() // raw H.264 (/h264) viewers + private val audioClients = CopyOnWriteArrayList() private val failedAttempts = ConcurrentHashMap() @Volatile private var appInForeground: Boolean = true + @Volatile + private var serverGeneration: Long = 0 // SECURITY: Rate limiting constants (only for unauthenticated connections) private val MAX_FAILED_ATTEMPTS = 5 // 5 failed attempts allowed @@ -100,7 +104,7 @@ class StreamingServerHelper( fun getH264Clients(): List = h264Clients.toList() fun resetH264Wait() { h264Clients.forEach { it.waitingKey = true } } // resync viewers at next keyframe fun removeH264Client(client: Client) { - h264Clients.remove(client) + if (!h264Clients.remove(client)) return try { client.socket.close() } catch (_: Exception) {} onClientDisconnected() } @@ -128,16 +132,15 @@ class StreamingServerHelper( } fun stopServer() { - serverJob?.cancel() - serverSocket?.close() - clients.forEach { client -> - try { - client.socket.close() - } catch (e: Exception) { - // Ignore close errors - } + synchronized(this) { + serverGeneration++ + isStarting = false + serverJob?.cancel() + serverSocket?.close() + serverJob = null + serverSocket = null } - clients.clear() + closeClientConnection() } private fun isRateLimited(clientIp: String): Boolean { @@ -174,6 +177,7 @@ class StreamingServerHelper( } fun startStreamingServer() { + val generation: Long // Prevent concurrent starts synchronized(this) { if (isStarting) { @@ -186,6 +190,8 @@ class StreamingServerHelper( } isStarting = true + serverGeneration++ + generation = serverGeneration } // Show toast when starting server @@ -218,6 +224,7 @@ class StreamingServerHelper( } catch (e: Exception) { // Ignore cancellation exceptions } + closeClientConnection() // Small delay to ensure port is released try { Thread.sleep(500) @@ -229,12 +236,20 @@ class StreamingServerHelper( synchronized(this) { serverJob = CoroutineScope(Dispatchers.IO).launch { + var localServerSocket: ServerSocket? = null try { val prefs = PreferenceManager.getDefaultSharedPreferences(context) val secureStorage = SecureStorage(context) val certificatePath = prefs.getString("certificate_path", null) - val rawPassword = secureStorage.getSecureString(SecureStorage.KEY_CERT_PASSWORD, null) + var rawPassword = secureStorage.getSecureString(SecureStorage.KEY_CERT_PASSWORD, null) + if (certificatePath == null && rawPassword.isNullOrEmpty()) { + rawPassword = generateCertificatePassword() + if (!secureStorage.putSecureString(SecureStorage.KEY_CERT_PASSWORD, rawPassword)) { + onLog("Unable to store the generated certificate password") + return@launch + } + } val certificatePassword = rawPassword?.let { if (it.isEmpty()) null else it.toCharArray() } @@ -248,7 +263,8 @@ class StreamingServerHelper( try { val personalCertFile = File(context.filesDir, "personal_certificate.p12") if (!personalCertFile.exists()) { - // Try to copy from assets first + // Preserve compatibility with packaged certificates, then generate a + // device-local one when the upstream source tree has no certificate asset. try { context.assets.open("personal_certificate.p12").use { input -> personalCertFile.outputStream().use { output -> @@ -256,14 +272,17 @@ class StreamingServerHelper( } } } catch (assetException: Exception) { - // Certificate not in assets - provide helpful error - Handler(Looper.getMainLooper()).post { - onLog("Certificate not found.") - Toast.makeText(context, - "Certificate missing, reset app to generate a new certificate", - Toast.LENGTH_LONG).show() + val password = rawPassword + if (password.isNullOrEmpty() || + CertificateHelper.generateCertificate(context, password) == null) { + Handler(Looper.getMainLooper()).post { + onLog("Certificate generation failed") + Toast.makeText(context, + "Unable to generate TLS certificate", + Toast.LENGTH_LONG).show() + } + return@launch } - return@launch } } finalCertificatePath = personalCertFile.absolutePath @@ -288,7 +307,7 @@ class StreamingServerHelper( val tlsVersionPref = prefs.getString("tls_version", "1.3") ?: "1.3" val useTLS = tlsVersionPref != "disabled" - serverSocket = if (useTLS) { + val createdServerSocket = if (useTLS) { try { // Determine which certificate file to use val certFile = if (certificatePath != null) { @@ -307,16 +326,17 @@ class StreamingServerHelper( File(finalCertificatePath!!) } - // Try TLS versions with fallback - val tlsVersionsToTry = when (tlsVersionPref) { - "1.3" -> listOf("TLSv1.3", "TLSv1.2") - "1.2" -> listOf("TLSv1.2") - else -> listOf("TLSv1.3", "TLSv1.2") + // Android 10 introduced the platform TLS 1.3 implementation. Older + // Android versions use their platform TLS 1.2 provider instead. + val tlsVersionsToTry = when { + tlsVersionPref == "1.2" -> listOf("TLSv1.2") + android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.Q -> + listOf("TLSv1.3") + else -> listOf("TLSv1.2") } var sslServerSocket: SSLServerSocket? = null var lastError: Exception? = null - var actualTlsVersionUsed: String? = null for (tlsVersion in tlsVersionsToTry) { try { @@ -329,7 +349,7 @@ class StreamingServerHelper( val sslContext = try { SSLContext.getInstance(tlsVersion) } catch (e: Exception) { - // TLS version not supported, try next + // The selected TLS version is unavailable. lastError = e continue } @@ -342,7 +362,6 @@ class StreamingServerHelper( // Don't restrict cipher suites - let the system negotiate soTimeout = 30000 } - actualTlsVersionUsed = tlsVersion onLog("Server started with TLS $tlsVersion") break } @@ -352,24 +371,10 @@ class StreamingServerHelper( } } - if (sslServerSocket == null) { - // Both TLS versions failed, disable TLS - prefs.edit().putString("tls_version", "disabled").apply() - throw lastError ?: IOException("Failed to create SSL server socket with any TLS version") - } - - // Update preference if we fell back to a different version - if (actualTlsVersionUsed != null && tlsVersionPref != actualTlsVersionUsed) { - val newPrefValue = when (actualTlsVersionUsed) { - "TLSv1.3" -> "1.3" - "TLSv1.2" -> "1.2" - else -> "disabled" - } - prefs.edit().putString("tls_version", newPrefValue).apply() - onLog("Updated TLS version preference to $actualTlsVersionUsed (fallback)") - } + val readyServerSocket = sslServerSocket + ?: throw lastError ?: IOException("Failed to create SSL server socket with any TLS version") - sslServerSocket + readyServerSocket } catch (keystoreException: Exception) { Handler(Looper.getMainLooper()).post { onLog("Certificate loading failed: ${keystoreException.message}") @@ -400,26 +405,36 @@ class StreamingServerHelper( return@launch } } + localServerSocket = createdServerSocket + val published = synchronized(this@StreamingServerHelper) { + if (generation != serverGeneration || !isActive) false + else { + serverSocket = createdServerSocket + true + } + } + if (!published) return@launch + onLog("Server started on port $streamPort (${if (useTLS) "HTTPS" else "HTTP"})") // Clear the starting flag now that server is running synchronized(this@StreamingServerHelper) { - isStarting = false + if (generation == serverGeneration) isStarting = false } Handler(Looper.getMainLooper()).post { Toast.makeText(context, "Server started", Toast.LENGTH_SHORT).show() } while (isActive && !Thread.currentThread().isInterrupted) { try { - val socket = serverSocket?.accept() ?: continue + val socket = createdServerSocket.accept() val clientIp = socket.inetAddress.hostAddress // Handle each connection in a separate coroutine to avoid blocking the accept loop CoroutineScope(Dispatchers.IO).launch { - handleClientConnection(socket, clientIp) + handleClientConnection(socket, clientIp, generation) } } catch (e: IOException) { // Check if server socket was closed - if (serverSocket == null || serverSocket!!.isClosed) { + if (createdServerSocket.isClosed) { onLog("Server socket closed, stopping server") break } @@ -429,7 +444,7 @@ class StreamingServerHelper( break } catch (e: Exception) { // Check if server socket was closed - if (serverSocket == null || serverSocket!!.isClosed) { + if (createdServerSocket.isClosed) { onLog("Server socket closed, stopping server") break } @@ -439,16 +454,17 @@ class StreamingServerHelper( } catch (e: IOException) { onLog("Could not start server: ${e.message}") } finally { - // Clear the starting flag if server failed to start + try { localServerSocket?.close() } catch (_: IOException) {} synchronized(this@StreamingServerHelper) { - isStarting = false + if (serverSocket === localServerSocket) serverSocket = null + if (generation == serverGeneration) isStarting = false } } } } } - private suspend fun handleClientConnection(socket: Socket, clientIp: String) { + private suspend fun handleClientConnection(socket: Socket, clientIp: String, generation: Long) { try { val outputStream = socket.getOutputStream() val writer = PrintWriter(outputStream, true) @@ -682,6 +698,14 @@ class StreamingServerHelper( try { socket.close() } catch (_: Exception) {} return } + val accepted = synchronized(this) { + if (generation != serverGeneration) false + else { audioClients.add(socket); true } + } + if (!accepted) { + try { socket.close() } catch (_: Exception) {} + return + } try { writer.print("HTTP/1.1 200 OK\r\n") writer.print("Connection: keep-alive\r\n") @@ -770,6 +794,9 @@ class StreamingServerHelper( } finally { try { socket.close() } catch (_: Exception) {} } + } finally { + audioClients.remove(socket) + try { socket.close() } catch (_: Exception) {} } return } @@ -791,7 +818,14 @@ class StreamingServerHelper( writer.print("Content-Type: video/h264\r\n\r\n") writer.flush() val client = Client(socket, outputStream, writer, System.currentTimeMillis(), isAuthenticated = enableAuth) - h264Clients.add(client) + val accepted = synchronized(this) { + if (generation != serverGeneration) false + else { h264Clients.add(client); true } + } + if (!accepted) { + try { socket.close() } catch (_: Exception) {} + return + } onClientConnected() // starts camera + encoder try { while (socket.isConnected && !socket.isClosed) { @@ -800,9 +834,7 @@ class StreamingServerHelper( } } catch (_: Exception) { } finally { - h264Clients.remove(client) - try { socket.close() } catch (_: Exception) {} - onClientDisconnected() + removeH264Client(client) } return } @@ -822,7 +854,15 @@ class StreamingServerHelper( writer.flush() // Add client to list - frames will be sent from MainActivity.processImage() - clients.add(Client(socket, outputStream, writer, System.currentTimeMillis(), isAuthenticated = enableAuth)) + val client = Client(socket, outputStream, writer, System.currentTimeMillis(), isAuthenticated = enableAuth) + val accepted = synchronized(this) { + if (generation != serverGeneration) false + else { clients.add(client); true } + } + if (!accepted) { + try { socket.close() } catch (_: Exception) {} + return + } onClientConnected() // Keep connection alive - frames will be sent from MainActivity.processImage() @@ -840,14 +880,7 @@ class StreamingServerHelper( } catch (e: IOException) { // Client disconnected } finally { - // Remove client when connection closes - clients.removeIf { it.socket == socket } - try { - socket.close() - } catch (e: Exception) { - // Ignore - } - onClientDisconnected() + removeClient(client) } } else { writer.print("HTTP/1.1 404 Not Found\r\n") @@ -1001,6 +1034,8 @@ class StreamingServerHelper( synchronized(this) { // Get references to close outside synchronized block + serverGeneration++ + isStarting = false jobToCancel = serverJob socketToClose = serverSocket serverSocket = null @@ -1044,19 +1079,31 @@ class StreamingServerHelper( } fun closeClientConnection() { - clients.forEach { client -> + val clientsToClose = clients.toList() + val h264ClientsToClose = h264Clients.toList() + val audioClientsToClose = audioClients.toList() + clients.clear() + h264Clients.clear() + audioClients.clear() + (clientsToClose + h264ClientsToClose).forEach { client -> try { client.socket.close() } catch (e: IOException) { onLog("Error closing client connection: ${e.message}") } } - clients.clear() - onClientDisconnected() + audioClientsToClose.forEach { socket -> + try { + socket.close() + } catch (e: IOException) { + onLog("Error closing audio connection: ${e.message}") + } + } + if (clientsToClose.isNotEmpty() || h264ClientsToClose.isNotEmpty()) onClientDisconnected() } fun removeClient(client: Client) { - clients.remove(client) + if (!clients.remove(client)) return try { client.socket.close() } catch (e: IOException) { @@ -1457,6 +1504,14 @@ class StreamingServerHelper( -1 } + private fun generateCertificatePassword(): String { + val alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + val random = SecureRandom() + return buildString(32) { + repeat(32) { append(alphabet[random.nextInt(alphabet.length)]) } + } + } + private fun getWifiStrength(): Int = try { val hasWifiPermission = if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.TIRAMISU) { ContextCompat.checkSelfPermission(context, Manifest.permission.NEARBY_WIFI_DEVICES) == PackageManager.PERMISSION_GRANTED || diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index eba99d2..22e3d90 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1,7 +1,7 @@ - TLS 1.3 (Recommended) + Automatic (TLS 1.3 on Android 10+, otherwise TLS 1.2) TLS 1.2 TLS Disabled (HTTP only) diff --git a/app/src/main/res/xml/preferences.xml b/app/src/main/res/xml/preferences.xml index 0c76024..498c3e9 100644 --- a/app/src/main/res/xml/preferences.xml +++ b/app/src/main/res/xml/preferences.xml @@ -25,7 +25,7 @@ diff --git a/app/src/test/kotlin/com/github/digitallyrefined/androidipcamera/helpers/FrameRateLimiterTest.kt b/app/src/test/kotlin/com/github/digitallyrefined/androidipcamera/helpers/FrameRateLimiterTest.kt new file mode 100644 index 0000000..e0782f8 --- /dev/null +++ b/app/src/test/kotlin/com/github/digitallyrefined/androidipcamera/helpers/FrameRateLimiterTest.kt @@ -0,0 +1,44 @@ +package com.github.digitallyrefined.androidipcamera.helpers + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class FrameRateLimiterTest { + @Test + fun `paces a 30 fps source to 10 fps`() { + val limiter = FrameRateLimiter(10) + val emitted = (0 until 90).count { frame -> + limiter.shouldEmit(frame * 1_000_000_000L / 30) + } + + assertTrue(emitted in 30..31) + } + + @Test + fun `keeps accurate average with a 25 fps source`() { + val limiter = FrameRateLimiter(10) + val emitted = (0 until 250).count { frame -> + limiter.shouldEmit(frame * 1_000_000_000L / 25) + } + + assertTrue(emitted in 100..101) + } + + @Test + fun `resets when camera timestamps move backwards`() { + val limiter = FrameRateLimiter(10) + assertTrue(limiter.shouldEmit(1_000_000_000L)) + limiter.shouldEmit(1_050_000_000L) + + assertTrue(limiter.shouldEmit(10L)) + } + + @Test + fun `clamps invalid target to one frame per second`() { + val limiter = FrameRateLimiter(0) + val emitted = (0..20).count { limiter.shouldEmit(it * 100_000_000L) } + + assertEquals(3, emitted) + } +} diff --git a/app/src/test/kotlin/com/github/digitallyrefined/androidipcamera/helpers/H264BitstreamTest.kt b/app/src/test/kotlin/com/github/digitallyrefined/androidipcamera/helpers/H264BitstreamTest.kt new file mode 100644 index 0000000..7267615 --- /dev/null +++ b/app/src/test/kotlin/com/github/digitallyrefined/androidipcamera/helpers/H264BitstreamTest.kt @@ -0,0 +1,140 @@ +package com.github.digitallyrefined.androidipcamera.helpers + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class H264BitstreamTest { + private val sps = bytes(0x67, 0x64, 0x00, 0x1f) + private val pps = bytes(0x68, 0xee, 0x3c, 0x80) + private val idr = bytes(0x65, 0x11, 0x22, 0x33) + private val pFrame = bytes(0x41, 0x44, 0x55) + + @Test + fun `normalizes mixed Annex-B start codes`() { + val input = concat(bytes(0, 0, 1), sps, bytes(0, 0, 0, 1), pps) + + val normalized = H264Bitstream.normalize(input)!! + + assertEquals(listOf(7, 8), nalTypes(normalized.bytes)) + assertArrayEquals(annexB(sps, pps), normalized.bytes) + } + + @Test + fun `normalizes every supported length prefix size`() { + for (lengthSize in 1..4) { + val input = concat(lengthPrefix(sps.size, lengthSize), sps, lengthPrefix(pps.size, lengthSize), pps) + + val normalized = H264Bitstream.normalize(input, lengthSize)!! + + assertEquals(lengthSize, normalized.nalLengthSize) + assertArrayEquals(annexB(sps, pps), normalized.bytes) + } + } + + @Test + fun `parses avcC and preserves its NAL length size`() { + val avcC = concat( + bytes(1, 0x64, 0, 0x1f, 0xff, 0xe1), + bytes(0, sps.size), sps, + bytes(1, 0, pps.size), pps + ) + + val normalized = H264Bitstream.normalize(avcC)!! + + assertEquals(4, normalized.nalLengthSize) + assertArrayEquals(annexB(sps, pps), normalized.bytes) + } + + @Test + fun `separate codec config is cached and omitted until a picture arrives`() { + val cache = H264ParameterSetCache() + + assertNull(cache.prepare(annexB(sps), false)) + assertNull(cache.prepare(annexB(pps), false)) + + val accessUnit = cache.prepare(annexB(idr), false)!! + assertTrue(accessUnit.isKeyframe) + assertArrayEquals(annexB(sps, pps, idr), accessUnit.bytes) + } + + @Test + fun `config and IDR in one buffer produces one self-contained keyframe`() { + val cache = H264ParameterSetCache() + + val accessUnit = cache.prepare(annexB(sps, pps, idr), true)!! + + assertTrue(accessUnit.isKeyframe) + assertEquals(listOf(7, 8, 5), nalTypes(accessUnit.bytes)) + } + + @Test + fun `IDR detection does not depend on MediaCodec keyframe flag`() { + val cache = H264ParameterSetCache() + cache.prepare(annexB(sps, pps), false) + + val accessUnit = cache.prepare(annexB(idr), false)!! + + assertTrue(accessUnit.isKeyframe) + } + + @Test + fun `incomplete codec config never releases a waiting client`() { + val cache = H264ParameterSetCache() + cache.prepare(annexB(sps), false) + + val accessUnit = cache.prepare(annexB(idr), true)!! + + assertFalse(accessUnit.isKeyframe) + assertEquals(listOf(5), nalTypes(accessUnit.bytes)) + } + + @Test + fun `ordinary inter frame is preserved and not marked key`() { + val cache = H264ParameterSetCache() + + val accessUnit = cache.prepare(annexB(pFrame), false)!! + + assertFalse(accessUnit.isKeyframe) + assertArrayEquals(annexB(pFrame), accessUnit.bytes) + } + + @Test + fun `malformed and truncated inputs are rejected`() { + assertNull(H264Bitstream.normalize(bytes(0, 0, 0, 5, 0x65))) + assertNull(H264Bitstream.normalize(bytes(0x80, 1, 2, 3))) + assertNull(H264Bitstream.normalize(byteArrayOf())) + } + + @Test + fun `partial MediaCodec buffers are emitted only after the final part`() { + val assembler = H264AccessUnitAssembler() + val first = bytes(1, 2) + val second = bytes(3, 4) + + assertNull(assembler.append(first, true)) + assertArrayEquals(concat(first, second), assembler.append(second, false)) + } + + private fun nalTypes(data: ByteArray): List = H264Bitstream.nalRanges(data).map { it.type } + + private fun annexB(vararg nals: ByteArray): ByteArray = H264Bitstream.joinNals(nals.toList()) + + private fun lengthPrefix(length: Int, size: Int): ByteArray = + ByteArray(size) { index -> ((length ushr (8 * (size - index - 1))) and 0xff).toByte() } + + private fun bytes(vararg values: Int): ByteArray = ByteArray(values.size) { values[it].toByte() } + + private fun concat(vararg arrays: ByteArray): ByteArray { + val result = ByteArray(arrays.sumOf { it.size }) + var offset = 0 + for (array in arrays) { + array.copyInto(result, offset) + offset += array.size + } + return result + } +}