diff --git a/examples/mobile/android/.gitignore b/examples/mobile/android/.gitignore index aa724b770..e7d9180e8 100644 --- a/examples/mobile/android/.gitignore +++ b/examples/mobile/android/.gitignore @@ -7,9 +7,20 @@ /.idea/workspace.xml /.idea/navEditor.xml /.idea/assetWizardSettings.xml +/.idea/AndroidProjectSystem.xml +/.idea/migrations.xml +/.idea/misc.xml +/.idea/vcs.xml +/.idea/codeStyles +/.idea/runConfigurations.xml +/.idea/gradle.xml +/.idea/deploymentTargetSelector.xml .DS_Store /build /captures .externalNativeBuild .cxx local.properties + +# Generated native libraries (built via cargo-ndk) +/app/src/main/jniLibs/ diff --git a/examples/mobile/android/.idea/AndroidProjectSystem.xml b/examples/mobile/android/.idea/AndroidProjectSystem.xml new file mode 100644 index 000000000..4a53bee8c --- /dev/null +++ b/examples/mobile/android/.idea/AndroidProjectSystem.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/examples/mobile/android/.idea/gradle.xml b/examples/mobile/android/.idea/gradle.xml deleted file mode 100644 index a9f4e522a..000000000 --- a/examples/mobile/android/.idea/gradle.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/examples/mobile/android/.idea/misc.xml b/examples/mobile/android/.idea/misc.xml deleted file mode 100644 index 9f71c83d4..000000000 --- a/examples/mobile/android/.idea/misc.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/examples/mobile/android/.idea/vcs.xml b/examples/mobile/android/.idea/vcs.xml deleted file mode 100644 index c2365ab11..000000000 --- a/examples/mobile/android/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/examples/mobile/android/app/build.gradle b/examples/mobile/android/app/build.gradle index 6c9278ef5..12c410620 100644 --- a/examples/mobile/android/app/build.gradle +++ b/examples/mobile/android/app/build.gradle @@ -5,12 +5,12 @@ plugins { android { namespace 'io.livekit.rustexample' - compileSdk 33 + compileSdk 34 defaultConfig { applicationId "io.livekit.rustexample" minSdk 24 - targetSdk 33 + targetSdk 34 versionCode 1 versionName "1.0" @@ -27,17 +27,17 @@ android { } } compileOptions { - sourceCompatibility JavaVersion.VERSION_1_8 - targetCompatibility JavaVersion.VERSION_1_8 + sourceCompatibility JavaVersion.VERSION_17 + targetCompatibility JavaVersion.VERSION_17 } kotlinOptions { - jvmTarget = '1.8' + jvmTarget = '17' } buildFeatures { compose true } composeOptions { - kotlinCompilerExtensionVersion '1.3.2' + kotlinCompilerExtensionVersion '1.5.8' } packagingOptions { resources { @@ -48,11 +48,11 @@ android { dependencies { implementation files('libs/libwebrtc.jar') - implementation 'androidx.core:core-ktx:1.8.0' - implementation platform('org.jetbrains.kotlin:kotlin-bom:1.8.0') - implementation 'androidx.lifecycle:lifecycle-runtime-ktx:2.3.1' - implementation 'androidx.activity:activity-compose:1.5.1' - implementation platform('androidx.compose:compose-bom:2022.10.00') + implementation 'androidx.core:core-ktx:1.12.0' + implementation platform('org.jetbrains.kotlin:kotlin-bom:1.9.22') + implementation 'androidx.lifecycle:lifecycle-runtime-ktx:2.7.0' + implementation 'androidx.activity:activity-compose:1.8.2' + implementation platform('androidx.compose:compose-bom:2024.02.01') implementation 'androidx.compose.ui:ui' implementation 'androidx.compose.ui:ui-graphics' implementation 'androidx.compose.ui:ui-tooling-preview' @@ -60,7 +60,7 @@ dependencies { testImplementation 'junit:junit:4.13.2' androidTestImplementation 'androidx.test.ext:junit:1.1.5' androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1' - androidTestImplementation platform('androidx.compose:compose-bom:2022.10.00') + androidTestImplementation platform('androidx.compose:compose-bom:2024.02.01') androidTestImplementation 'androidx.compose.ui:ui-test-junit4' debugImplementation 'androidx.compose.ui:ui-tooling' debugImplementation 'androidx.compose.ui:ui-test-manifest' diff --git a/examples/mobile/android/app/libs/libwebrtc.jar b/examples/mobile/android/app/libs/libwebrtc.jar index f0d85d8f6..e32ccc0aa 100644 Binary files a/examples/mobile/android/app/libs/libwebrtc.jar and b/examples/mobile/android/app/libs/libwebrtc.jar differ diff --git a/examples/mobile/android/app/src/main/AndroidManifest.xml b/examples/mobile/android/app/src/main/AndroidManifest.xml index cfe001452..e61672e90 100644 --- a/examples/mobile/android/app/src/main/AndroidManifest.xml +++ b/examples/mobile/android/app/src/main/AndroidManifest.xml @@ -3,6 +3,8 @@ xmlns:tools="http://schemas.android.com/tools"> + + + if (isGranted) { + Log.i(TAG, "RECORD_AUDIO permission granted") + startAudioAfterPermission() + } else { + Log.w(TAG, "RECORD_AUDIO permission denied") + statusMessage.value = "Microphone permission denied" + } + } + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - val app = App() - app.connect( - "ws://localhost:7880", - "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE5MDY2MTMyODgsImlzcyI6IkFQSVRzRWZpZFpqclFvWSIsIm5hbWUiOiJuYXRpdmUiLCJuYmYiOjE2NzI2MTMyODgsInN1YiI6Im5hdGl2ZSIsInZpZGVvIjp7InJvb20iOiJ0ZXN0Iiwicm9vbUFkbWluIjp0cnVlLCJyb29tQ3JlYXRlIjp0cnVlLCJyb29tSm9pbiI6dHJ1ZSwicm9vbUxpc3QiOnRydWV9fQ.uSNIangMRu8jZD5mnRYoCHjcsQWCrJXgHCs0aNIgBFY" - ) + // Initialize the LiveKit app + app = App() + + // Check if native library is available + if (app?.isNativeAvailable() != true) { + statusMessage.value = "Native library not available" + } setContent { RustexampleTheme { - // A surface container using the 'background' color from the theme Surface( - modifier = Modifier.fillMaxSize(), - color = MaterialTheme.colorScheme.background - ) { Greeting("Android") } + modifier = Modifier.fillMaxSize(), + color = MaterialTheme.colorScheme.background + ) { + MainScreen( + isConnected = isConnected.value, + isAudioActive = isAudioActive.value, + statusMessage = statusMessage.value, + defaultUrl = DEFAULT_URL, + defaultToken = DEFAULT_TOKEN, + onConnectClick = { url, token -> connectToRoom(url, token) }, + onDisconnectClick = { disconnectFromRoom() }, + onStartAudioClick = { startAudio() }, + onStopAudioClick = { stopAudio() } + ) + } + } + } + } + + private fun connectToRoom(url: String, token: String) { + val currentApp = app ?: run { + statusMessage.value = "App not initialized" + return + } + + if (!currentApp.isNativeAvailable()) { + statusMessage.value = "Native library not available" + Log.e(TAG, "Native library not available") + return + } + + if (url.isBlank() || token.isBlank()) { + statusMessage.value = "Server URL and token are required" + return + } + + // Initialize MediaManager with the App reference + if (mediaManager == null) { + mediaManager = MediaManager(this, currentApp) + } + + statusMessage.value = "Connecting..." + currentApp.connect(url, token) + } + // Update state after a short delay to allow connection to establish + // In a real app, you'd want callbacks from the native side + android.os.Handler(mainLooper).postDelayed({ + val connected = currentApp.isConnected() + isConnected.value = connected + statusMessage.value = if (connected) "Connected to room" else "Connection failed" + Log.i(TAG, "Connection result: $connected") + }, 1000) + } + + private fun disconnectFromRoom() { + stopAudio() + app?.disconnect() + isConnected.value = false + statusMessage.value = "Disconnected" + Log.i(TAG, "Disconnected from room") + } + + private fun startAudio() { + if (ContextCompat.checkSelfPermission( + this, + Manifest.permission.RECORD_AUDIO + ) == PackageManager.PERMISSION_GRANTED + ) { + startAudioAfterPermission() + } else { + requestPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) + } + } + + private fun startAudioAfterPermission() { + val success = mediaManager?.startAll() ?: false + if (success) { + isAudioActive.value = true + statusMessage.value = "Audio active - mic and speaker running" + Log.i(TAG, "Audio started") + } else { + statusMessage.value = "Failed to start audio" + Log.e(TAG, "Failed to start audio") + } + } + + private fun stopAudio() { + mediaManager?.stopAll() + isAudioActive.value = false + if (isConnected.value) { + statusMessage.value = "Connected (audio stopped)" + } + Log.i(TAG, "Audio stopped") + } + + override fun onDestroy() { + super.onDestroy() + mediaManager?.release() + mediaManager = null + app?.disconnect() + } +} + +@Composable +fun MainScreen( + isConnected: Boolean, + isAudioActive: Boolean, + statusMessage: String, + defaultUrl: String, + defaultToken: String, + onConnectClick: (String, String) -> Unit, + onDisconnectClick: () -> Unit, + onStartAudioClick: () -> Unit, + onStopAudioClick: () -> Unit +) { + val url = remember { mutableStateOf(defaultUrl) } + val token = remember { mutableStateOf(defaultToken) } + + Column( + modifier = Modifier + .fillMaxSize() + .padding(16.dp), + horizontalAlignment = Alignment.CenterHorizontally, + verticalArrangement = Arrangement.Top + ) { + Text( + text = "LiveKit Rust SDK Demo", + style = MaterialTheme.typography.headlineMedium, + modifier = Modifier.padding(bottom = 8.dp) + ) + + Text( + text = statusMessage, + style = MaterialTheme.typography.bodyMedium, + color = if (isConnected) MaterialTheme.colorScheme.primary + else MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.padding(bottom = 24.dp) + ) + + if (!isConnected) { + OutlinedTextField( + value = url.value, + onValueChange = { url.value = it }, + label = { Text("Server URL") }, + modifier = Modifier.fillMaxWidth(), + singleLine = true + ) + + Spacer(modifier = Modifier.height(8.dp)) + + OutlinedTextField( + value = token.value, + onValueChange = { token.value = it }, + label = { Text("Access Token") }, + modifier = Modifier.fillMaxWidth(), + singleLine = true, + visualTransformation = PasswordVisualTransformation() + ) + + Spacer(modifier = Modifier.height(16.dp)) + + Button( + onClick = { onConnectClick(url.value, token.value) }, + modifier = Modifier.fillMaxWidth() + ) { + Text("Connect") + } + } else { + // Connected state + Column( + modifier = Modifier.fillMaxWidth(), + horizontalAlignment = Alignment.CenterHorizontally + ) { + if (!isAudioActive) { + Button( + onClick = onStartAudioClick, + modifier = Modifier.fillMaxWidth() + ) { + Text("Start Audio") + } + + Text( + text = "Start audio to enable microphone capture and speaker playback", + style = MaterialTheme.typography.bodySmall, + modifier = Modifier.padding(top = 8.dp) + ) + } else { + Button( + onClick = onStopAudioClick, + modifier = Modifier.fillMaxWidth() + ) { + Text("Stop Audio") + } + + Text( + text = "Microphone: Capturing and sending to LiveKit\nSpeaker: Playing remote participants", + style = MaterialTheme.typography.bodySmall, + modifier = Modifier.padding(top = 8.dp) + ) + } + + Spacer(modifier = Modifier.height(24.dp)) + + Button( + onClick = onDisconnectClick, + modifier = Modifier.fillMaxWidth() + ) { + Text("Disconnect") + } } } + + Spacer(modifier = Modifier.weight(1f)) + + Text( + text = "Audio: 48kHz, Mono, 16-bit PCM\n10ms frames (480 samples)", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant + ) } } +@Preview(showBackground = true) @Composable -fun Greeting(name: String, modifier: Modifier = Modifier) { - Text(text = "Hello $name!", modifier = modifier) +fun MainScreenDisconnectedPreview() { + RustexampleTheme { + MainScreen( + isConnected = false, + isAudioActive = false, + statusMessage = "Ready to connect", + defaultUrl = "ws://localhost:7880", + defaultToken = "your-token-here", + onConnectClick = { _, _ -> }, + onDisconnectClick = {}, + onStartAudioClick = {}, + onStopAudioClick = {} + ) + } } @Preview(showBackground = true) @Composable -fun GreetingPreview() { - RustexampleTheme { Greeting("Android") } +fun MainScreenConnectedPreview() { + RustexampleTheme { + MainScreen( + isConnected = true, + isAudioActive = true, + statusMessage = "Audio active - mic and speaker running", + defaultUrl = "ws://localhost:7880", + defaultToken = "your-token-here", + onConnectClick = { _, _ -> }, + onDisconnectClick = {}, + onStartAudioClick = {}, + onStopAudioClick = {} + ) + } } diff --git a/examples/mobile/android/app/src/main/java/io/livekit/rustexample/MediaManager.kt b/examples/mobile/android/app/src/main/java/io/livekit/rustexample/MediaManager.kt new file mode 100644 index 000000000..e796561a5 --- /dev/null +++ b/examples/mobile/android/app/src/main/java/io/livekit/rustexample/MediaManager.kt @@ -0,0 +1,370 @@ +package io.livekit.rustexample + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import android.media.AudioAttributes +import android.media.AudioFormat +import android.media.AudioManager +import android.media.AudioRecord +import android.media.AudioTrack +import android.media.MediaRecorder +import android.util.Log +import androidx.core.content.ContextCompat +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.concurrent.atomic.AtomicBoolean + +class MediaManager( + private val context: Context, + private val app: App // Reference to App for pushing/pulling audio +) { + + companion object { + private const val TAG = "MediaManager" + + // Audio configuration - must match Rust side + const val SAMPLE_RATE = 48000 + const val CHANNEL_CONFIG_IN = AudioFormat.CHANNEL_IN_MONO + const val CHANNEL_CONFIG_OUT = AudioFormat.CHANNEL_OUT_MONO + const val AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT + const val FRAME_DURATION_MS = 10 + const val SAMPLES_PER_FRAME = SAMPLE_RATE * FRAME_DURATION_MS / 1000 // 480 samples + const val BYTES_PER_SAMPLE = 2 // 16-bit PCM + const val BUFFER_SIZE_FRAMES = 10 + } + + private var audioRecord: AudioRecord? = null + private var audioTrack: AudioTrack? = null + + private var captureThread: Thread? = null + private var playbackThread: Thread? = null + + private val isCapturing = AtomicBoolean(false) + private val isPlaying = AtomicBoolean(false) + + fun hasRecordPermission(): Boolean { + return ContextCompat.checkSelfPermission( + context, + Manifest.permission.RECORD_AUDIO + ) == PackageManager.PERMISSION_GRANTED + } + + fun startMicrophone(): Boolean { + if (!hasRecordPermission()) { + Log.e(TAG, "RECORD_AUDIO permission not granted") + return false + } + + if (isCapturing.get()) { + Log.w(TAG, "Microphone already started") + return true + } + + val minBufferSize = AudioRecord.getMinBufferSize( + SAMPLE_RATE, + CHANNEL_CONFIG_IN, + AUDIO_FORMAT + ) + + if (minBufferSize == AudioRecord.ERROR || minBufferSize == AudioRecord.ERROR_BAD_VALUE) { + Log.e(TAG, "Failed to get minimum buffer size for AudioRecord") + return false + } + + val bufferSize = maxOf(minBufferSize, SAMPLES_PER_FRAME * BYTES_PER_SAMPLE * BUFFER_SIZE_FRAMES) + + try { + audioRecord = AudioRecord( + MediaRecorder.AudioSource.VOICE_COMMUNICATION, + SAMPLE_RATE, + CHANNEL_CONFIG_IN, + AUDIO_FORMAT, + bufferSize + ) + + if (audioRecord?.state != AudioRecord.STATE_INITIALIZED) { + Log.e(TAG, "AudioRecord failed to initialize") + audioRecord?.release() + audioRecord = null + return false + } + + audioRecord?.startRecording() + isCapturing.set(true) + + captureThread = Thread({ + android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_URGENT_AUDIO) + captureLoop() + }, "AudioCaptureThread") + captureThread?.start() + + Log.i(TAG, "Microphone started successfully") + return true + + } catch (e: SecurityException) { + Log.e(TAG, "SecurityException starting microphone", e) + return false + } catch (e: Exception) { + Log.e(TAG, "Exception starting microphone", e) + audioRecord?.release() + audioRecord = null + return false + } + } + + fun stopMicrophone() { + if (!isCapturing.get()) { + return + } + + isCapturing.set(false) + + captureThread?.let { thread -> + try { + thread.join(1000) + } catch (e: InterruptedException) { + Log.w(TAG, "Interrupted while waiting for capture thread") + } + } + captureThread = null + + audioRecord?.let { record -> + try { + record.stop() + } catch (e: Exception) { + Log.w(TAG, "Exception stopping AudioRecord", e) + } + record.release() + } + audioRecord = null + + Log.i(TAG, "Microphone stopped") + } + + fun startSpeaker(): Boolean { + if (isPlaying.get()) { + Log.w(TAG, "Speaker already started") + return true + } + + val minBufferSize = AudioTrack.getMinBufferSize( + SAMPLE_RATE, + CHANNEL_CONFIG_OUT, + AUDIO_FORMAT + ) + + if (minBufferSize == AudioTrack.ERROR || minBufferSize == AudioTrack.ERROR_BAD_VALUE) { + Log.e(TAG, "Failed to get minimum buffer size for AudioTrack") + return false + } + + val bufferSize = maxOf(minBufferSize, SAMPLES_PER_FRAME * BYTES_PER_SAMPLE * BUFFER_SIZE_FRAMES) + + try { + val audioAttributes = AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build() + + val audioFormat = AudioFormat.Builder() + .setSampleRate(SAMPLE_RATE) + .setChannelMask(CHANNEL_CONFIG_OUT) + .setEncoding(AUDIO_FORMAT) + .build() + + audioTrack = AudioTrack( + audioAttributes, + audioFormat, + bufferSize, + AudioTrack.MODE_STREAM, + android.media.AudioManager.AUDIO_SESSION_ID_GENERATE + ) + + if (audioTrack?.state != AudioTrack.STATE_INITIALIZED) { + Log.e(TAG, "AudioTrack failed to initialize") + audioTrack?.release() + audioTrack = null + return false + } + + audioTrack?.play() + isPlaying.set(true) + + playbackThread = Thread({ + android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_URGENT_AUDIO) + playbackLoop() + }, "AudioPlaybackThread") + playbackThread?.start() + + Log.i(TAG, "Speaker started successfully") + return true + + } catch (e: Exception) { + Log.e(TAG, "Exception starting speaker", e) + audioTrack?.release() + audioTrack = null + return false + } + } + + fun stopSpeaker() { + if (!isPlaying.get()) { + return + } + + isPlaying.set(false) + + playbackThread?.let { thread -> + try { + thread.join(1000) + } catch (e: InterruptedException) { + Log.w(TAG, "Interrupted while waiting for playback thread") + } + } + playbackThread = null + + audioTrack?.let { track -> + try { + track.stop() + } catch (e: Exception) { + Log.w(TAG, "Exception stopping AudioTrack", e) + } + track.release() + } + audioTrack = null + + Log.i(TAG, "Speaker stopped") + } + + fun startAll(): Boolean { + val micStarted = startMicrophone() + val speakerStarted = startSpeaker() + return micStarted && speakerStarted + } + + fun stopAll() { + stopMicrophone() + stopSpeaker() + } + + fun isCapturing(): Boolean = isCapturing.get() + fun isPlaying(): Boolean = isPlaying.get() + + private fun captureLoop() { + val frameSize = SAMPLES_PER_FRAME * BYTES_PER_SAMPLE + val byteBuffer = ByteArray(frameSize) + val shortBuffer = ShortArray(SAMPLES_PER_FRAME) + + Log.d(TAG, "Capture loop started, frame size: $frameSize bytes, $SAMPLES_PER_FRAME samples") + + var frameCount = 0L + + while (isCapturing.get()) { + val record = audioRecord ?: break + + // Read bytes from AudioRecord + val bytesRead = record.read(byteBuffer, 0, frameSize) + + when { + bytesRead > 0 -> { + // Convert bytes to shorts + val samplesRead = bytesRead / BYTES_PER_SAMPLE + ByteBuffer.wrap(byteBuffer, 0, bytesRead) + .order(ByteOrder.LITTLE_ENDIAN) + .asShortBuffer() + .get(shortBuffer, 0, samplesRead) + + // Push to LiveKit via native code + val consumed = app.pushAudio(shortBuffer.copyOf(samplesRead)) + + frameCount++ + if (frameCount % 100 == 0L) { // Log every ~1 second + Log.d(TAG, "Captured and pushed $samplesRead samples (frame $frameCount)") + } + } + bytesRead == AudioRecord.ERROR_INVALID_OPERATION -> { + Log.e(TAG, "AudioRecord ERROR_INVALID_OPERATION") + break + } + bytesRead == AudioRecord.ERROR_BAD_VALUE -> { + Log.e(TAG, "AudioRecord ERROR_BAD_VALUE") + break + } + bytesRead == AudioRecord.ERROR_DEAD_OBJECT -> { + Log.e(TAG, "AudioRecord ERROR_DEAD_OBJECT") + break + } + bytesRead == AudioRecord.ERROR -> { + Log.e(TAG, "AudioRecord ERROR") + break + } + } + } + + Log.d(TAG, "Capture loop ended after $frameCount frames") + } + + private fun playbackLoop() { + val shortBuffer = ShortArray(SAMPLES_PER_FRAME) + val byteBuffer = ByteArray(SAMPLES_PER_FRAME * BYTES_PER_SAMPLE) + + Log.d(TAG, "Playback loop started, frame size: $SAMPLES_PER_FRAME samples") + + var frameCount = 0L + var silentFrames = 0L + + while (isPlaying.get()) { + val track = audioTrack ?: break + + // Pull audio from LiveKit via native code + val samplesReceived = app.pullAudio(shortBuffer) + + // Convert shorts to bytes for AudioTrack + ByteBuffer.wrap(byteBuffer) + .order(ByteOrder.LITTLE_ENDIAN) + .asShortBuffer() + .put(shortBuffer) + + val bytesWritten = track.write(byteBuffer, 0, SAMPLES_PER_FRAME * BYTES_PER_SAMPLE) + + frameCount++ + if (samplesReceived == 0) { + silentFrames++ + } + + if (frameCount % 100 == 0L) { // Log every ~1 second + val bufferSize = app.getPlaybackBufferSize() + Log.d(TAG, "Playback frame $frameCount: received $samplesReceived samples, " + + "buffer size: $bufferSize, silent frames: $silentFrames") + silentFrames = 0 + } + + when { + bytesWritten < 0 -> { + when (bytesWritten) { + AudioTrack.ERROR_INVALID_OPERATION -> { + Log.e(TAG, "AudioTrack ERROR_INVALID_OPERATION") + } + AudioTrack.ERROR_BAD_VALUE -> { + Log.e(TAG, "AudioTrack ERROR_BAD_VALUE") + } + AudioTrack.ERROR_DEAD_OBJECT -> { + Log.e(TAG, "AudioTrack ERROR_DEAD_OBJECT") + break + } + else -> { + Log.e(TAG, "AudioTrack error: $bytesWritten") + } + } + } + } + } + + Log.d(TAG, "Playback loop ended after $frameCount frames") + } + + fun release() { + stopAll() + } +} diff --git a/examples/mobile/android/build.gradle b/examples/mobile/android/build.gradle index 431431374..479de0248 100644 --- a/examples/mobile/android/build.gradle +++ b/examples/mobile/android/build.gradle @@ -1,6 +1,6 @@ // Top-level build file where you can add configuration options common to all sub-projects/modules. plugins { - id 'com.android.application' version '8.0.2' apply false - id 'com.android.library' version '8.0.2' apply false - id 'org.jetbrains.kotlin.android' version '1.7.20' apply false + id 'com.android.application' version '8.2.2' apply false + id 'com.android.library' version '8.2.2' apply false + id 'org.jetbrains.kotlin.android' version '1.9.22' apply false } \ No newline at end of file diff --git a/examples/mobile/android/gradle/wrapper/gradle-wrapper.jar b/examples/mobile/android/gradle/wrapper/gradle-wrapper.jar index e708b1c02..980502d16 100644 Binary files a/examples/mobile/android/gradle/wrapper/gradle-wrapper.jar and b/examples/mobile/android/gradle/wrapper/gradle-wrapper.jar differ diff --git a/examples/mobile/android/gradle/wrapper/gradle-wrapper.properties b/examples/mobile/android/gradle/wrapper/gradle-wrapper.properties index 373fdcabe..df97d72b8 100644 --- a/examples/mobile/android/gradle/wrapper/gradle-wrapper.properties +++ b/examples/mobile/android/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,7 @@ -#Mon Jul 10 13:30:11 CEST 2023 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.0-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/examples/mobile/android/gradlew b/examples/mobile/android/gradlew index 4f906e0c8..faf93008b 100755 --- a/examples/mobile/android/gradlew +++ b/examples/mobile/android/gradlew @@ -1,7 +1,7 @@ -#!/usr/bin/env sh +#!/bin/sh # -# Copyright 2015 the original author or authors. +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,69 +15,103 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -87,9 +121,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -98,88 +132,120 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=`expr $i + 1` + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/examples/mobile/android/gradlew.bat b/examples/mobile/android/gradlew.bat index ac1b06f93..9b42019c7 100644 --- a/examples/mobile/android/gradlew.bat +++ b/examples/mobile/android/gradlew.bat @@ -13,8 +13,10 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -25,7 +27,8 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,13 +43,13 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute +if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -56,11 +59,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -75,13 +78,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal diff --git a/examples/mobile/src/lib.rs b/examples/mobile/src/lib.rs index 7eb190ac0..d2c28431b 100644 --- a/examples/mobile/src/lib.rs +++ b/examples/mobile/src/lib.rs @@ -1,8 +1,55 @@ +use futures::StreamExt; use lazy_static::lazy_static; -use livekit::{Room, RoomOptions}; +use livekit::{ + options::TrackPublishOptions, + prelude::*, + track::{LocalAudioTrack, LocalTrack, RemoteTrack, TrackSource}, + webrtc::{ + audio_frame::AudioFrame, + audio_source::native::NativeAudioSource, + audio_stream::native::NativeAudioStream, + prelude::{AudioSourceOptions, RtcAudioSource}, + }, + Room, RoomOptions, +}; +use parking_lot::Mutex; +use std::collections::VecDeque; +use std::sync::Arc; + +const SAMPLE_RATE: u32 = 48000; +const NUM_CHANNELS: u32 = 1; +const SAMPLES_PER_10MS: u32 = SAMPLE_RATE / 100; // 480 samples per 10ms frame + +struct AudioState { + // For sending microphone audio to LiveKit + audio_source: Option, + capture_buffer: VecDeque, + + // For receiving remote audio from LiveKit + playback_buffer: VecDeque, + + // Room reference + room: Option>, + + // Track if we're connected + is_connected: bool, +} + +impl Default for AudioState { + fn default() -> Self { + Self { + audio_source: None, + capture_buffer: VecDeque::with_capacity(SAMPLE_RATE as usize), // 1 second buffer + playback_buffer: VecDeque::with_capacity(SAMPLE_RATE as usize), + room: None, + is_connected: false, + } + } +} struct App { async_runtime: tokio::runtime::Runtime, + audio_state: Arc>, } impl Default for App { @@ -12,6 +59,7 @@ impl Default for App { .enable_all() .build() .unwrap(), + audio_state: Arc::new(Mutex::new(AudioState::default())), } } } @@ -20,11 +68,36 @@ lazy_static! { static ref APP: App = App::default(); } +/// Connect to a LiveKit room and set up audio handling pub fn livekit_connect(url: String, token: String) { log::info!("Connecting to {} with token {}", url, token); + let audio_state = APP.audio_state.clone(); + APP.async_runtime.spawn(async move { - let res = Room::connect(&url, &token, RoomOptions::default()).await; + // Create audio source for microphone capture + let audio_source = NativeAudioSource::new( + AudioSourceOptions { + echo_cancellation: false, + noise_suppression: false, + auto_gain_control: false, + }, + SAMPLE_RATE, + NUM_CHANNELS, + 100, // 100ms buffer + ); + + // Store audio source + { + let mut state = audio_state.lock(); + state.audio_source = Some(audio_source.clone()); + } + + // Connect to room + let mut room_options = RoomOptions::default(); + room_options.auto_subscribe = true; + + let res = Room::connect(&url, &token, room_options).await; if let Err(err) = res { log::error!("Failed to connect: {}", err); @@ -32,14 +105,294 @@ pub fn livekit_connect(url: String, token: String) { } let (room, mut events) = res.unwrap(); + let room = Arc::new(room); + log::info!("Connected to room {}", String::from(room.sid().await)); + // Store room reference + { + let mut state = audio_state.lock(); + state.room = Some(room.clone()); + state.is_connected = true; + } + + // Create and publish local audio track + let track = LocalAudioTrack::create_audio_track( + "microphone", + RtcAudioSource::Native(audio_source), + ); + + if let Err(e) = room + .local_participant() + .publish_track( + LocalTrack::Audio(track), + TrackPublishOptions { + source: TrackSource::Microphone, + ..Default::default() + }, + ) + .await + { + log::error!("Failed to publish audio track: {}", e); + } else { + log::info!("Published local audio track"); + } + + // Handle room events while let Some(event) = events.recv().await { - log::info!("Received event {:?}", event); + match event { + RoomEvent::TrackSubscribed { + track, + publication, + participant, + } => { + log::info!( + "Track subscribed from {}: {} ({:?})", + participant.identity(), + publication.name(), + track.kind() + ); + + if let RemoteTrack::Audio(audio_track) = track { + let audio_state_clone = audio_state.clone(); + let participant_id = participant.identity().to_string(); + + // Spawn task to handle this audio stream + tokio::spawn(async move { + handle_remote_audio_stream( + audio_track, + audio_state_clone, + participant_id, + ) + .await; + }); + } + } + RoomEvent::TrackUnsubscribed { + track, + participant, + .. + } => { + log::info!( + "Track unsubscribed from {}: {:?}", + participant.identity(), + track.kind() + ); + } + RoomEvent::ParticipantConnected(participant) => { + log::info!( + "Participant connected: {} ({})", + participant.identity(), + participant.name() + ); + } + RoomEvent::ParticipantDisconnected(participant) => { + log::info!("Participant disconnected: {}", participant.identity()); + } + RoomEvent::Disconnected { reason } => { + log::info!("Disconnected from room: {:?}", reason); + let mut state = audio_state.lock(); + state.is_connected = false; + state.room = None; + break; + } + _ => { + log::debug!("Room event: {:?}", event); + } + } } }); } +/// Handle incoming audio from a remote participant +async fn handle_remote_audio_stream( + audio_track: RemoteAudioTrack, + audio_state: Arc>, + participant_id: String, +) { + log::info!( + "Starting audio stream for participant: {}, track sid: {:?}", + participant_id, + audio_track.sid() + ); + + let mut audio_stream = NativeAudioStream::new( + audio_track.rtc_track(), + SAMPLE_RATE as i32, + NUM_CHANNELS as i32, + ); + + let mut frame_count: u64 = 0; + let mut total_samples: u64 = 0; + + while let Some(frame) = audio_stream.next().await { + let samples: &[i16] = frame.data.as_ref(); + frame_count += 1; + total_samples += samples.len() as u64; + + // Log every 100 frames (~1 second) + if frame_count % 100 == 0 { + log::info!( + "Audio stream [{}]: received frame #{}, {} samples this frame, {} total samples, sample_rate={}, channels={}", + participant_id, + frame_count, + samples.len(), + total_samples, + frame.sample_rate, + frame.num_channels + ); + } + + // Add samples to playback buffer + let mut state = audio_state.lock(); + + // Limit buffer size to prevent memory growth (keep ~500ms max) + let max_buffer_size = (SAMPLE_RATE / 2) as usize; + while state.playback_buffer.len() + samples.len() > max_buffer_size { + state.playback_buffer.pop_front(); + } + + let buffer_size_before = state.playback_buffer.len(); + for &sample in samples { + state.playback_buffer.push_back(sample); + } + + // Log buffer state periodically + if frame_count % 100 == 0 { + log::info!( + "Playback buffer: {} -> {} samples", + buffer_size_before, + state.playback_buffer.len() + ); + } + } + + log::info!( + "Audio stream ended for participant: {}, total frames: {}, total samples: {}", + participant_id, + frame_count, + total_samples + ); +} + +/// Push captured microphone audio to LiveKit (called from Kotlin) +/// Returns the number of samples consumed +pub fn push_audio_capture(samples: &[i16]) -> usize { + let mut state = APP.audio_state.lock(); + + // Add to capture buffer + for &sample in samples { + state.capture_buffer.push_back(sample); + } + + // Process complete 10ms frames + let mut frames_sent = 0; + while state.capture_buffer.len() >= SAMPLES_PER_10MS as usize { + let mut frame_data: Vec = Vec::with_capacity(SAMPLES_PER_10MS as usize); + for _ in 0..SAMPLES_PER_10MS { + if let Some(sample) = state.capture_buffer.pop_front() { + frame_data.push(sample); + } + } + + if let Some(ref audio_source) = state.audio_source { + let audio_frame = AudioFrame { + data: frame_data.into(), + sample_rate: SAMPLE_RATE, + num_channels: NUM_CHANNELS, + samples_per_channel: SAMPLES_PER_10MS, + }; + + // Use blocking capture since we're called from a sync context + let source = audio_source.clone(); + drop(state); // Release lock before async operation + + APP.async_runtime.spawn(async move { + if let Err(e) = source.capture_frame(&audio_frame).await { + log::error!("Failed to capture audio frame: {}", e); + } + }); + + state = APP.audio_state.lock(); + frames_sent += 1; + } + } + + if frames_sent > 0 { + log::trace!("Sent {} audio frames to LiveKit", frames_sent); + } + + samples.len() +} + +/// Pull playback audio from LiveKit (called from Kotlin) +/// Returns the number of samples written to the buffer +pub fn pull_audio_playback(buffer: &mut [i16]) -> usize { + static PULL_COUNT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + + let mut state = APP.audio_state.lock(); + + let available = state.playback_buffer.len(); + let to_copy = available.min(buffer.len()); + + for i in 0..to_copy { + if let Some(sample) = state.playback_buffer.pop_front() { + buffer[i] = sample; + } + } + + // Fill remaining with silence + for i in to_copy..buffer.len() { + buffer[i] = 0; + } + + // Log periodically + let count = PULL_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if count % 100 == 0 { + log::debug!( + "pull_audio_playback #{}: requested={}, available={}, copied={}", + count, + buffer.len(), + available, + to_copy + ); + } + + to_copy +} + +/// Check if we're connected to a room +pub fn is_connected() -> bool { + APP.audio_state.lock().is_connected +} + +/// Get the number of samples available for playback +pub fn get_playback_buffer_size() -> usize { + APP.audio_state.lock().playback_buffer.len() +} + +/// Disconnect from the room +pub fn disconnect() { + let room = { + let mut state = APP.audio_state.lock(); + state.is_connected = false; + state.room.take() + }; + + if let Some(room) = room { + APP.async_runtime.spawn(async move { + if let Err(e) = room.close().await { + log::error!("Error closing room: {}", e); + } + log::info!("Disconnected from room"); + }); + } +} + +// ============================================================================ +// iOS Implementation +// ============================================================================ + #[cfg(target_os = "ios")] pub mod ios { use std::ffi::{c_char, CStr}; @@ -54,14 +407,41 @@ pub mod ios { super::livekit_connect(url, token); } + + #[no_mangle] + pub extern "C" fn livekit_push_audio(samples: *const i16, count: usize) -> usize { + let slice = unsafe { std::slice::from_raw_parts(samples, count) }; + super::push_audio_capture(slice) + } + + #[no_mangle] + pub extern "C" fn livekit_pull_audio(buffer: *mut i16, count: usize) -> usize { + let slice = unsafe { std::slice::from_raw_parts_mut(buffer, count) }; + super::pull_audio_playback(slice) + } + + #[no_mangle] + pub extern "C" fn livekit_disconnect() { + super::disconnect(); + } + + #[no_mangle] + pub extern "C" fn livekit_is_connected() -> bool { + super::is_connected() + } } +// ============================================================================ +// Android Implementation +// ============================================================================ + #[cfg(target_os = "android")] pub mod android { use android_logger::Config; use jni::{ - sys::{jint, JNI_VERSION_1_6}, - JavaVM, + objects::{JClass, JShortArray, JString}, + sys::{jboolean, jint, JNI_VERSION_1_6}, + JNIEnv, JavaVM, }; use log::LevelFilter; use std::os::raw::c_void; @@ -70,7 +450,9 @@ pub mod android { #[no_mangle] pub extern "C" fn JNI_OnLoad(vm: JavaVM, _: *mut c_void) -> jint { android_logger::init_once( - Config::default().with_max_level(LevelFilter::Debug).with_tag("livekit-rustexample"), + Config::default() + .with_max_level(LevelFilter::Debug) + .with_tag("livekit-rustexample"), ); log::info!("JNI_OnLoad, initializing LiveKit"); @@ -78,17 +460,112 @@ pub mod android { JNI_VERSION_1_6 } + /// Connect to a LiveKit room #[allow(non_snake_case)] #[no_mangle] - pub extern "C" fn Java_io_livekit_rustexample_App_connect( - mut env: jni::JNIEnv, - _: jni::objects::JClass, - url: jni::objects::JString, - token: jni::objects::JString, + pub extern "C" fn Java_io_livekit_rustexample_App_connectNative( + mut env: JNIEnv, + _: JClass, + url: JString, + token: JString, ) { let url: String = env.get_string(&url).unwrap().into(); let token: String = env.get_string(&token).unwrap().into(); super::livekit_connect(url, token); } + + /// Disconnect from the room + #[allow(non_snake_case)] + #[no_mangle] + pub extern "C" fn Java_io_livekit_rustexample_App_disconnectNative( + _env: JNIEnv, + _: JClass, + ) { + super::disconnect(); + } + + /// Check if connected to a room + #[allow(non_snake_case)] + #[no_mangle] + pub extern "C" fn Java_io_livekit_rustexample_App_isConnectedNative( + _env: JNIEnv, + _: JClass, + ) -> jboolean { + if super::is_connected() { 1 } else { 0 } + } + + /// Push captured audio samples to LiveKit + /// Takes a short array (16-bit PCM samples) + /// Returns the number of samples consumed + #[allow(non_snake_case)] + #[no_mangle] + pub extern "C" fn Java_io_livekit_rustexample_App_pushAudioNative( + env: JNIEnv, + _: JClass, + samples: JShortArray, + ) -> jint { + let len = match env.get_array_length(&samples) { + Ok(l) => l as usize, + Err(e) => { + log::error!("Failed to get array length: {}", e); + return 0; + } + }; + + if len == 0 { + return 0; + } + + let mut buffer: Vec = vec![0i16; len]; + if let Err(e) = env.get_short_array_region(&samples, 0, &mut buffer) { + log::error!("Failed to get short array region: {}", e); + return 0; + } + + super::push_audio_capture(&buffer) as jint + } + + /// Pull playback audio from LiveKit + /// Fills the provided short array with PCM samples + /// Returns the number of actual samples written (rest is silence) + #[allow(non_snake_case)] + #[no_mangle] + pub extern "C" fn Java_io_livekit_rustexample_App_pullAudioNative( + env: JNIEnv, + _: JClass, + buffer: JShortArray, + ) -> jint { + let len = match env.get_array_length(&buffer) { + Ok(l) => l as usize, + Err(e) => { + log::error!("Failed to get array length: {}", e); + return 0; + } + }; + + if len == 0 { + return 0; + } + + let mut rust_buffer: Vec = vec![0i16; len]; + let samples_written = super::pull_audio_playback(&mut rust_buffer); + + if let Err(e) = env.set_short_array_region(&buffer, 0, &rust_buffer) { + log::error!("Failed to set short array region: {}", e); + return 0; + } + + samples_written as jint + } + + /// Get the number of samples available in the playback buffer + #[allow(non_snake_case)] + #[no_mangle] + pub extern "C" fn Java_io_livekit_rustexample_App_getPlaybackBufferSizeNative( + _env: JNIEnv, + _: JClass, + ) -> jint { + super::get_playback_buffer_size() as jint + } }