diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index c1080daf7..45891440b 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -61,7 +61,7 @@ tools:ignore="DiscouragedApi" /> diff --git a/app/src/main/java/com/pedro/streamer/rotation/BitmapSource.kt b/app/src/main/java/com/pedro/streamer/rotation/BitmapSource.kt index dac4c9bae..2fa414e85 100644 --- a/app/src/main/java/com/pedro/streamer/rotation/BitmapSource.kt +++ b/app/src/main/java/com/pedro/streamer/rotation/BitmapSource.kt @@ -19,7 +19,6 @@ package com.pedro.streamer.rotation import android.graphics.Bitmap import android.graphics.Paint import android.graphics.SurfaceTexture -import android.util.Log import android.view.Surface import androidx.core.graphics.scale import com.pedro.library.util.sources.video.VideoSource diff --git a/app/src/main/java/com/pedro/streamer/rotation/RotationActivity.kt b/app/src/main/java/com/pedro/streamer/rotation/RotationActivity.kt index 630a426e3..390472ee6 100644 --- a/app/src/main/java/com/pedro/streamer/rotation/RotationActivity.kt +++ b/app/src/main/java/com/pedro/streamer/rotation/RotationActivity.kt @@ -16,6 +16,7 @@ package com.pedro.streamer.rotation +import android.annotation.SuppressLint import android.graphics.BitmapFactory import android.os.Build import android.os.Bundle @@ -31,8 +32,8 @@ import com.pedro.library.util.sources.video.Camera1Source import com.pedro.library.util.sources.video.Camera2Source import com.pedro.streamer.R import com.pedro.streamer.utils.FilterMenu -import com.pedro.streamer.utils.setColor import com.pedro.streamer.utils.toast +import com.pedro.streamer.utils.updateMenuColor /** @@ -60,10 +61,10 @@ class RotationActivity : AppCompatActivity(), OnTouchListener { val defaultAudioSource = menu.findItem(R.id.audio_source_microphone) val defaultOrientation = menu.findItem(R.id.orientation_horizontal) val defaultFilter = menu.findItem(R.id.no_filter) - currentVideoSource = updateMenuColor(currentVideoSource, defaultVideoSource) - currentAudioSource = updateMenuColor(currentAudioSource, defaultAudioSource) - currentOrientation = updateMenuColor(currentOrientation, defaultOrientation) - currentFilter = updateMenuColor(currentFilter, defaultFilter) + currentVideoSource = defaultVideoSource.updateMenuColor(this, currentVideoSource) + currentAudioSource = defaultAudioSource.updateMenuColor(this, currentAudioSource) + currentOrientation = defaultOrientation.updateMenuColor(this, currentOrientation) + currentFilter = defaultFilter.updateMenuColor(this, currentFilter) return true } @@ -71,37 +72,37 @@ class RotationActivity : AppCompatActivity(), OnTouchListener { try { when (item.itemId) { R.id.video_source_camera1 -> { - currentVideoSource = updateMenuColor(currentVideoSource, item) + currentVideoSource = item.updateMenuColor(this, currentVideoSource) cameraFragment.genericStream.changeVideoSource(Camera1Source(applicationContext)) } R.id.video_source_camera2 -> { - currentVideoSource = updateMenuColor(currentVideoSource, item) + currentVideoSource = item.updateMenuColor(this, currentVideoSource) cameraFragment.genericStream.changeVideoSource(Camera2Source(applicationContext)) } R.id.video_source_camerax -> { - currentVideoSource = updateMenuColor(currentVideoSource, item) + currentVideoSource = item.updateMenuColor(this, currentVideoSource) cameraFragment.genericStream.changeVideoSource(CameraXSource(applicationContext)) } R.id.video_source_bitmap -> { - currentVideoSource = updateMenuColor(currentVideoSource, item) + currentVideoSource = item.updateMenuColor(this, currentVideoSource) val bitmap = BitmapFactory.decodeResource(resources, R.mipmap.ic_launcher) cameraFragment.genericStream.changeVideoSource(BitmapSource(bitmap)) } R.id.audio_source_microphone -> { - currentAudioSource = updateMenuColor(currentAudioSource, item) + currentAudioSource = item.updateMenuColor(this, currentAudioSource) cameraFragment.genericStream.changeAudioSource(MicrophoneSource()) } R.id.orientation_horizontal -> { - currentOrientation = updateMenuColor(currentOrientation, item) + currentOrientation = item.updateMenuColor(this, currentOrientation) cameraFragment.setOrientationMode(false) } R.id.orientation_vertical -> { - currentOrientation = updateMenuColor(currentOrientation, item) + currentOrientation = item.updateMenuColor(this, currentOrientation) cameraFragment.setOrientationMode(true) } else -> { val result = filterMenu.onOptionsItemSelected(item, cameraFragment.genericStream.getGlInterface()) - if (result) currentFilter = updateMenuColor(currentFilter, item) + if (result) currentFilter = item.updateMenuColor(this, currentFilter) return result } } @@ -111,6 +112,7 @@ class RotationActivity : AppCompatActivity(), OnTouchListener { return super.onOptionsItemSelected(item) } + @SuppressLint("ClickableViewAccessibility") override fun onTouch(view: View, motionEvent: MotionEvent): Boolean { if (filterMenu.spriteGestureController.spriteTouched(view, motionEvent)) { filterMenu.spriteGestureController.moveSprite(view, motionEvent) @@ -119,10 +121,4 @@ class RotationActivity : AppCompatActivity(), OnTouchListener { } return false } - - private fun updateMenuColor(currentItem: MenuItem?, item: MenuItem): MenuItem { - currentItem?.setColor(this, R.color.black) - item.setColor(this, R.color.appColor) - return item - } } \ No newline at end of file diff --git a/app/src/main/java/com/pedro/streamer/screen/ScreenActivity.kt b/app/src/main/java/com/pedro/streamer/screen/ScreenActivity.kt index 9f1ce0005..3f0b28bb3 100644 --- a/app/src/main/java/com/pedro/streamer/screen/ScreenActivity.kt +++ b/app/src/main/java/com/pedro/streamer/screen/ScreenActivity.kt @@ -18,6 +18,8 @@ package com.pedro.streamer.screen import android.content.Intent import android.os.Build import android.os.Bundle +import android.view.Menu +import android.view.MenuItem import android.view.WindowManager import android.widget.EditText import android.widget.ImageView @@ -25,10 +27,13 @@ import androidx.activity.result.contract.ActivityResultContracts import androidx.annotation.RequiresApi import androidx.appcompat.app.AppCompatActivity import com.pedro.common.ConnectChecker +import com.pedro.library.base.recording.RecordController +import com.pedro.library.util.sources.audio.MixAudioSource import com.pedro.library.util.sources.audio.InternalAudioSource import com.pedro.library.util.sources.audio.MicrophoneSource import com.pedro.streamer.R import com.pedro.streamer.utils.toast +import com.pedro.streamer.utils.updateMenuColor /** * Example code to stream the device screen. @@ -49,8 +54,8 @@ import com.pedro.streamer.utils.toast class ScreenActivity : AppCompatActivity(), ConnectChecker { private lateinit var button: ImageView - private lateinit var toggleAudio: ImageView private lateinit var etUrl: EditText + private var currentAudioSource: MenuItem? = null private val activityResultContract = registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result -> val data = result.data @@ -76,21 +81,7 @@ class ScreenActivity : AppCompatActivity(), ConnectChecker { setContentView(R.layout.activity_display) button = findViewById(R.id.b_start_stop) etUrl = findViewById(R.id.et_rtp_url) - toggleAudio = findViewById(R.id.b_toggleAudio) - toggleAudio.setOnClickListener { - val service = ScreenService.INSTANCE - when (service?.toggleAudioSource()) { - is InternalAudioSource -> { - toggleAudio.setImageResource(R.drawable.microphone_off_icon) - toast("Using internal audio source") - } - is MicrophoneSource -> { - toggleAudio.setImageResource(R.drawable.microphone_icon) - toast("Using microphone audio source") - } - else -> {} - } - } + val bRecord = findViewById(R.id.b_record) val screenService = ScreenService.INSTANCE //No streaming/recording start service if (screenService == null) { @@ -101,6 +92,11 @@ class ScreenActivity : AppCompatActivity(), ConnectChecker { } else { button.setImageResource(R.drawable.stream_icon) } + if (screenService != null && screenService.isRecording()) { + bRecord.setImageResource(R.drawable.stop_icon) + } else { + bRecord.setImageResource(R.drawable.record_icon) + } button.setOnClickListener { val service = ScreenService.INSTANCE if (service != null) { @@ -113,6 +109,51 @@ class ScreenActivity : AppCompatActivity(), ConnectChecker { } } } + bRecord.setOnClickListener { + ScreenService.INSTANCE?.toggleRecord { state -> + when (state) { + RecordController.Status.STARTED -> { + bRecord.setImageResource(R.drawable.pause_icon) + } + RecordController.Status.STOPPED -> { + bRecord.setImageResource(R.drawable.record_icon) + } + RecordController.Status.RECORDING -> { + bRecord.setImageResource(R.drawable.stop_icon) + } + else -> {} + } + } + } + } + + override fun onCreateOptionsMenu(menu: Menu): Boolean { + menuInflater.inflate(R.menu.screen_menu, menu) + val defaultAudioSource = when (ScreenService.INSTANCE?.getCurrentAudioSource()) { + is MicrophoneSource -> menu.findItem(R.id.audio_source_microphone) + is InternalAudioSource -> menu.findItem(R.id.audio_source_internal) + is MixAudioSource -> menu.findItem(R.id.audio_source_mix) + else -> menu.findItem(R.id.audio_source_microphone) + } + currentAudioSource = defaultAudioSource.updateMenuColor(this, currentAudioSource) + return true + } + + override fun onOptionsItemSelected(item: MenuItem): Boolean { + try { + when (item.itemId) { + R.id.audio_source_microphone, R.id.audio_source_internal, R.id.audio_source_mix -> { + val service = ScreenService.INSTANCE + if (service != null) { + service.toggleAudioSource(item.itemId) + currentAudioSource = item.updateMenuColor(this, currentAudioSource) + } + } + } + } catch (e: IllegalArgumentException) { + toast("Change source error: ${e.message}") + } + return super.onOptionsItemSelected(item) } override fun onDestroy() { diff --git a/app/src/main/java/com/pedro/streamer/screen/ScreenService.kt b/app/src/main/java/com/pedro/streamer/screen/ScreenService.kt index dfdb5168e..ade995e98 100644 --- a/app/src/main/java/com/pedro/streamer/screen/ScreenService.kt +++ b/app/src/main/java/com/pedro/streamer/screen/ScreenService.kt @@ -29,14 +29,20 @@ import android.util.Log import androidx.annotation.RequiresApi import androidx.core.app.NotificationCompat import com.pedro.common.ConnectChecker +import com.pedro.library.base.recording.RecordController import com.pedro.library.generic.GenericStream +import com.pedro.library.util.sources.audio.MixAudioSource import com.pedro.library.util.sources.audio.AudioSource import com.pedro.library.util.sources.audio.InternalAudioSource import com.pedro.library.util.sources.audio.MicrophoneSource import com.pedro.library.util.sources.video.NoVideoSource import com.pedro.library.util.sources.video.ScreenSource import com.pedro.streamer.R +import com.pedro.streamer.utils.PathUtils import com.pedro.streamer.utils.toast +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale /** @@ -47,8 +53,8 @@ class ScreenService: Service(), ConnectChecker { companion object { private const val TAG = "DisplayService" - private const val channelId = "rtpDisplayStreamChannel" - const val notifyId = 123456 + private const val CHANNEL_ID = "DisplayStreamChannel" + const val NOTIFY_ID = 123456 var INSTANCE: ScreenService? = null } @@ -67,13 +73,15 @@ class ScreenService: Service(), ConnectChecker { private val isStereo = true private val aBitrate = 128 * 1000 private var prepared = false + private var recordPath = "" + private var selectedAudioSource: Int = R.id.audio_source_microphone override fun onCreate() { super.onCreate() Log.i(TAG, "RTP Display service create") notificationManager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - val channel = NotificationChannel(channelId, channelId, NotificationManager.IMPORTANCE_HIGH) + val channel = NotificationChannel(CHANNEL_ID, CHANNEL_ID, NotificationManager.IMPORTANCE_HIGH) notificationManager?.createNotificationChannel(channel) } genericStream = GenericStream(baseContext, this, NoVideoSource(), MicrophoneSource()).apply { @@ -82,7 +90,10 @@ class ScreenService: Service(), ConnectChecker { } prepared = try { genericStream.prepareVideo(width, height, vBitrate, rotation = rotation) && - genericStream.prepareAudio(sampleRate, isStereo, aBitrate) + genericStream.prepareAudio(sampleRate, isStereo, aBitrate, + echoCanceler = true, + noiseSuppressor = true + ) } catch (e: IllegalArgumentException) { false } @@ -91,7 +102,7 @@ class ScreenService: Service(), ConnectChecker { } private fun keepAliveTrick() { - val notification = NotificationCompat.Builder(this, channelId) + val notification = NotificationCompat.Builder(this, CHANNEL_ID) .setSmallIcon(R.drawable.notification_icon) .setSilent(true) .setOngoing(false) @@ -123,7 +134,7 @@ class ScreenService: Service(), ConnectChecker { fun stopStream() { if (genericStream.isStreaming) { genericStream.stopStream() - notificationManager?.cancel(notifyId) + notificationManager?.cancel(NOTIFY_ID) } } @@ -155,23 +166,63 @@ class ScreenService: Service(), ConnectChecker { //You need to call it after prepareVideo to override the default value. genericStream.getGlInterface().setCameraOrientation(0) genericStream.changeVideoSource(screenSource) + toggleAudioSource(selectedAudioSource) true } catch (ignored: IllegalArgumentException) { false } } - fun toggleAudioSource(): AudioSource { - if (genericStream.audioSource is InternalAudioSource) { + fun getCurrentAudioSource(): AudioSource = genericStream.audioSource + + fun toggleAudioSource(itemId: Int) { + when (itemId) { + R.id.audio_source_microphone -> { + selectedAudioSource = R.id.audio_source_microphone + if (genericStream.audioSource is MicrophoneSource) return genericStream.changeAudioSource(MicrophoneSource()) - } else { - mediaProjection?.let { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - genericStream.changeAudioSource(InternalAudioSource(it)) - } + } + R.id.audio_source_internal -> { + selectedAudioSource = R.id.audio_source_internal + if (genericStream.audioSource is InternalAudioSource) return + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + mediaProjection?.let { genericStream.changeAudioSource(InternalAudioSource(it)) } + } else { + throw IllegalArgumentException("You need min API 29+") + } + } + R.id.audio_source_mix -> { + selectedAudioSource = R.id.audio_source_mix + if (genericStream.audioSource is MixAudioSource) return + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + mediaProjection?.let { genericStream.changeAudioSource(MixAudioSource(it).apply { + //Using audio mix the internal audio volume is higher than microphone. Increase microphone fix it. + microphoneVolume = 2f + }) } + } else { + throw IllegalArgumentException("You need min API 29+") + } + } + } + } + + fun toggleRecord(state: (RecordController.Status) -> Unit) { + if (!genericStream.isRecording) { + val folder = PathUtils.getRecordPath() + if (!folder.exists()) folder.mkdir() + val sdf = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.getDefault()) + recordPath = "${folder.absolutePath}/${sdf.format(Date())}.mp4" + genericStream.startRecord(recordPath) { status -> + if (status == RecordController.Status.RECORDING) { + state(RecordController.Status.RECORDING) } } - return genericStream.audioSource + state(RecordController.Status.STARTED) + } else { + genericStream.stopRecord() + state(RecordController.Status.STOPPED) + PathUtils.updateGallery(this, recordPath) + } } fun startStream(endpoint: String) { diff --git a/app/src/main/java/com/pedro/streamer/utils/Extensions.kt b/app/src/main/java/com/pedro/streamer/utils/Extensions.kt index 178df9a5e..285c3e419 100644 --- a/app/src/main/java/com/pedro/streamer/utils/Extensions.kt +++ b/app/src/main/java/com/pedro/streamer/utils/Extensions.kt @@ -21,7 +21,6 @@ import android.app.Service import android.content.Context import android.graphics.BlendMode import android.graphics.BlendModeColorFilter -import android.graphics.Color import android.graphics.PorterDuff import android.graphics.drawable.Drawable import android.os.Build @@ -33,6 +32,7 @@ import androidx.annotation.ColorInt import androidx.annotation.ColorRes import androidx.core.content.ContextCompat import androidx.fragment.app.Fragment +import com.pedro.streamer.R /** @@ -68,4 +68,10 @@ fun Drawable.setColorFilter(@ColorInt color: Int) { } else { setColorFilter(color, PorterDuff.Mode.SRC_IN) } +} + +fun MenuItem.updateMenuColor(context: Context, currentItem: MenuItem?): MenuItem { + currentItem?.setColor(context, R.color.black) + setColor(context, R.color.appColor) + return this } \ No newline at end of file diff --git a/app/src/main/res/layout/activity_display.xml b/app/src/main/res/layout/activity_display.xml index af0637439..9d2e3c522 100644 --- a/app/src/main/res/layout/activity_display.xml +++ b/app/src/main/res/layout/activity_display.xml @@ -30,6 +30,19 @@ app:layout_constraintBottom_toBottomOf="parent" android:layout_margin="16dp" > + + + - - diff --git a/app/src/main/res/menu/screen_menu.xml b/app/src/main/res/menu/screen_menu.xml new file mode 100644 index 000000000..03fad96e8 --- /dev/null +++ b/app/src/main/res/menu/screen_menu.xml @@ -0,0 +1,19 @@ + + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 11162e15f..7fe90d36f 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -61,6 +61,8 @@ Vertical Audio source Microphone + Internal Audio + Mix audio Video source Camera1 Camera2 diff --git a/encoder/src/main/java/com/pedro/encoder/audio/AudioEncoder.java b/encoder/src/main/java/com/pedro/encoder/audio/AudioEncoder.java index e5b3f392f..5ba3f8bbf 100644 --- a/encoder/src/main/java/com/pedro/encoder/audio/AudioEncoder.java +++ b/encoder/src/main/java/com/pedro/encoder/audio/AudioEncoder.java @@ -43,7 +43,7 @@ public class AudioEncoder extends BaseEncoder implements GetMicrophoneData { private final GetAudioData getAudioData; private int bitRate = 64 * 1024; //in kbps private int sampleRate = 32000; //in hz - private int maxInputSize = 0; + public static final int inputSize = 8192; private boolean isStereo = true; private GetFrame getFrame; private long bytesRead = 0; @@ -59,13 +59,11 @@ public AudioEncoder(GetAudioData getAudioData) { /** * Prepare encoder with custom parameters */ - public boolean prepareAudioEncoder(int bitRate, int sampleRate, boolean isStereo, - int maxInputSize) { + public boolean prepareAudioEncoder(int bitRate, int sampleRate, boolean isStereo) { if (prepared) stop(); this.bitRate = bitRate; this.sampleRate = sampleRate; - this.maxInputSize = maxInputSize; this.isStereo = isStereo; isBufferMode = true; @@ -90,7 +88,7 @@ public boolean prepareAudioEncoder(int bitRate, int sampleRate, boolean isStereo int channelCount = (isStereo) ? 2 : 1; MediaFormat audioFormat = MediaFormat.createAudioFormat(type, sampleRate, channelCount); audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitRate); - audioFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, maxInputSize); + audioFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, inputSize); audioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); setCallback(); @@ -114,7 +112,7 @@ public void setGetFrame(GetFrame getFrame) { * Prepare encoder with default parameters */ public boolean prepareAudioEncoder() { - return prepareAudioEncoder(bitRate, sampleRate, isStereo, maxInputSize); + return prepareAudioEncoder(bitRate, sampleRate, isStereo); } @Override @@ -132,7 +130,7 @@ protected void stopImp() { @Override public boolean reset() { stop(false); - boolean result = prepareAudioEncoder(bitRate, sampleRate, isStereo, maxInputSize); + boolean result = prepareAudioEncoder(bitRate, sampleRate, isStereo); if (!result) return false; restart(); return true; diff --git a/encoder/src/main/java/com/pedro/encoder/input/audio/MicrophoneManager.java b/encoder/src/main/java/com/pedro/encoder/input/audio/MicrophoneManager.java index fe3988c86..e00fedf2f 100644 --- a/encoder/src/main/java/com/pedro/encoder/input/audio/MicrophoneManager.java +++ b/encoder/src/main/java/com/pedro/encoder/input/audio/MicrophoneManager.java @@ -31,6 +31,7 @@ import androidx.annotation.RequiresApi; import com.pedro.encoder.Frame; +import com.pedro.encoder.audio.AudioEncoder; /** * Created by pedro on 19/01/17. @@ -40,13 +41,10 @@ public class MicrophoneManager { private final String TAG = "MicrophoneManager"; - private final int DEFAULT_BUFFER_SIZE = 2048; - private int BUFFER_SIZE = 0; - private int CUSTOM_BUFFER_SIZE = 0; protected AudioRecord audioRecord; private final GetMicrophoneData getMicrophoneData; - protected byte[] pcmBuffer = new byte[BUFFER_SIZE]; - protected byte[] pcmBufferMuted = new byte[BUFFER_SIZE]; + protected byte[] pcmBuffer = new byte[AudioEncoder.inputSize]; + protected byte[] pcmBufferMuted = new byte[AudioEncoder.inputSize]; protected boolean running = false; private boolean created = false; //default parameters for microphone @@ -95,7 +93,7 @@ public boolean createMicrophone(int audioSource, int sampleRate, boolean isStere this.sampleRate = sampleRate; channel = isStereo ? AudioFormat.CHANNEL_IN_STEREO : AudioFormat.CHANNEL_IN_MONO; getPcmBufferSize(sampleRate, channel); - audioRecord = new AudioRecord(audioSource, sampleRate, channel, audioFormat, getMaxInputSize() * 5); + audioRecord = new AudioRecord(audioSource, sampleRate, channel, audioFormat, AudioEncoder.inputSize * 5); audioPostProcessEffect = new AudioPostProcessEffect(audioRecord.getAudioSessionId()); if (echoCanceler) audioPostProcessEffect.enableEchoCanceler(); if (noiseSuppressor) audioPostProcessEffect.enableNoiseSuppressor(); @@ -135,7 +133,7 @@ public boolean createInternalMicrophone(AudioPlaybackCaptureConfiguration config .setSampleRate(sampleRate) .setChannelMask(channel) .build()) - .setBufferSizeInBytes(getMaxInputSize() * 5) + .setBufferSizeInBytes(AudioEncoder.inputSize * 5) .build(); audioPostProcessEffect = new AudioPostProcessEffect(audioRecord.getAudioSessionId()); if (echoCanceler) audioPostProcessEffect.enableEchoCanceler(); @@ -215,7 +213,7 @@ public boolean isMuted() { protected Frame read() { long timeStamp = System.nanoTime() / 1000; int size = audioRecord.read(pcmBuffer, 0, pcmBuffer.length); - if (size < 0){ + if (size < 0) { Log.e(TAG, "read error: " + size); return null; } @@ -251,23 +249,10 @@ public synchronized void stop() { * Get PCM buffer size */ private void getPcmBufferSize(int sampleRate, int channel) { - if (CUSTOM_BUFFER_SIZE > 0) { - pcmBuffer = new byte[CUSTOM_BUFFER_SIZE]; - pcmBufferMuted = new byte[CUSTOM_BUFFER_SIZE]; - } else { - int minSize = AudioRecord.getMinBufferSize(sampleRate, channel, audioFormat); - BUFFER_SIZE = Math.max(minSize, DEFAULT_BUFFER_SIZE); - pcmBuffer = new byte[BUFFER_SIZE]; - pcmBufferMuted = new byte[BUFFER_SIZE]; - } - } - - public int getMaxInputSize() { - return CUSTOM_BUFFER_SIZE > 0 ? CUSTOM_BUFFER_SIZE : BUFFER_SIZE; - } - - public void setMaxInputSize(int size) { - CUSTOM_BUFFER_SIZE = size; + int minSize = AudioRecord.getMinBufferSize(sampleRate, channel, audioFormat); + int bufferSize = Math.max(minSize, AudioEncoder.inputSize); + pcmBuffer = new byte[bufferSize]; + pcmBufferMuted = new byte[bufferSize]; } public int getSampleRate() { diff --git a/encoder/src/main/java/com/pedro/encoder/input/audio/VolumeEffect.kt b/encoder/src/main/java/com/pedro/encoder/input/audio/VolumeEffect.kt new file mode 100644 index 000000000..be3c6c434 --- /dev/null +++ b/encoder/src/main/java/com/pedro/encoder/input/audio/VolumeEffect.kt @@ -0,0 +1,40 @@ +/* + * + * * Copyright (C) 2024 pedroSG94. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.pedro.encoder.input.audio + +class VolumeEffect: CustomAudioEffect() { + + var volume = 1f + + override fun process(pcmBuffer: ByteArray): ByteArray { + if (volume == 1f) return pcmBuffer + + for (i in pcmBuffer.indices step 2) { + var buf1 = pcmBuffer[i + 1].toShort() + var buf2 = pcmBuffer[i].toShort() + buf1 = ((buf1.toInt() and 0xff) shl 8).toShort() + buf2 = (buf2.toInt() and 0xff).toShort() + var res = (buf1.toInt() or buf2.toInt()).toShort() + res = (res * volume).toInt().toShort() + pcmBuffer[i] = res.toByte() + pcmBuffer[i + 1] = (res.toInt() shr 8).toByte() + } + return pcmBuffer + } +} \ No newline at end of file diff --git a/library/src/main/java/com/pedro/library/base/Camera1Base.java b/library/src/main/java/com/pedro/library/base/Camera1Base.java index fa9e38114..c3c5dc9a5 100644 --- a/library/src/main/java/com/pedro/library/base/Camera1Base.java +++ b/library/src/main/java/com/pedro/library/base/Camera1Base.java @@ -333,8 +333,7 @@ public boolean prepareAudio(int audioSource, int bitrate, int sampleRate, boolea return false; } onAudioInfoImp(isStereo, sampleRate); - audioInitialized = audioEncoder.prepareAudioEncoder(bitrate, sampleRate, isStereo, - microphoneManager.getMaxInputSize()); + audioInitialized = audioEncoder.prepareAudioEncoder(bitrate, sampleRate, isStereo); return audioInitialized; } @@ -795,17 +794,6 @@ public List getSupportedFps() { return cameraManager.getSupportedFps(); } - /** - * Set a custom size of audio buffer input. - * If you set 0 or less you can disable it to use library default value. - * Must be called before of prepareAudio method. - * - * @param size in bytes. Recommended multiple of 1024 (2048, 4096, 8196, etc) - */ - public void setAudioMaxInputSize(int size) { - microphoneManager.setMaxInputSize(size); - } - /** * Mute microphone, can be called before, while and after stream. */ diff --git a/library/src/main/java/com/pedro/library/base/Camera2Base.java b/library/src/main/java/com/pedro/library/base/Camera2Base.java index 95bda7497..e143e8130 100644 --- a/library/src/main/java/com/pedro/library/base/Camera2Base.java +++ b/library/src/main/java/com/pedro/library/base/Camera2Base.java @@ -341,8 +341,7 @@ public boolean prepareAudio(int audioSource, int bitrate, int sampleRate, boolea return false; } onAudioInfoImp(isStereo, sampleRate); - audioInitialized = audioEncoder.prepareAudioEncoder(bitrate, sampleRate, isStereo, - microphoneManager.getMaxInputSize()); + audioInitialized = audioEncoder.prepareAudioEncoder(bitrate, sampleRate, isStereo); return audioInitialized; } @@ -712,17 +711,6 @@ public CameraCharacteristics getCameraCharacteristics() { return cameraManager.getCameraCharacteristics(); } - /** - * Set a custom size of audio buffer input. - * If you set 0 or less you can disable it to use library default value. - * Must be called before of prepareAudio method. - * - * @param size in bytes. Recommended multiple of 1024 (2048, 4096, 8196, etc) - */ - public void setAudioMaxInputSize(int size) { - microphoneManager.setMaxInputSize(size); - } - /** * Mute microphone, can be called before, while and after stream. */ diff --git a/library/src/main/java/com/pedro/library/base/DisplayBase.java b/library/src/main/java/com/pedro/library/base/DisplayBase.java index 5f1e024ba..f558a8ea9 100644 --- a/library/src/main/java/com/pedro/library/base/DisplayBase.java +++ b/library/src/main/java/com/pedro/library/base/DisplayBase.java @@ -219,8 +219,7 @@ public boolean prepareAudio(int audioSource, int bitrate, int sampleRate, boolea return false; } onAudioInfoImp(isStereo, sampleRate); - audioInitialized = audioEncoder.prepareAudioEncoder(bitrate, sampleRate, isStereo, - microphoneManager.getMaxInputSize()); + audioInitialized = audioEncoder.prepareAudioEncoder(bitrate, sampleRate, isStereo); return audioInitialized; } @@ -261,8 +260,7 @@ public boolean prepareInternalAudio(int bitrate, int sampleRate, boolean isStere return false; } onAudioInfoImp(isStereo, sampleRate); - audioInitialized = audioEncoder.prepareAudioEncoder(bitrate, sampleRate, isStereo, - microphoneManager.getMaxInputSize()); + audioInitialized = audioEncoder.prepareAudioEncoder(bitrate, sampleRate, isStereo); return audioInitialized; } @@ -478,17 +476,6 @@ public GlInterface getGlInterface() { } } - /** - * Set a custom size of audio buffer input. - * If you set 0 or less you can disable it to use library default value. - * Must be called before of prepareAudio method. - * - * @param size in bytes. Recommended multiple of 1024 (2048, 4096, 8196, etc) - */ - public void setAudioMaxInputSize(int size) { - microphoneManager.setMaxInputSize(size); - } - /** * Mute microphone, can be called before, while and after stream. */ diff --git a/library/src/main/java/com/pedro/library/base/FromFileBase.java b/library/src/main/java/com/pedro/library/base/FromFileBase.java index c0011a485..171088335 100644 --- a/library/src/main/java/com/pedro/library/base/FromFileBase.java +++ b/library/src/main/java/com/pedro/library/base/FromFileBase.java @@ -245,7 +245,7 @@ public boolean prepareAudio(Context context, Uri uri, int bitRate) throws IOExce private boolean finishPrepareAudio(int bitRate) { audioDecoder.prepareAudio(); boolean result = audioEncoder.prepareAudioEncoder(bitRate, audioDecoder.getSampleRate(), - audioDecoder.isStereo(), audioDecoder.getOutsize()); + audioDecoder.isStereo()); onAudioInfoImp(audioDecoder.isStereo(), audioDecoder.getSampleRate()); audioEnabled = result; return result; diff --git a/library/src/main/java/com/pedro/library/base/OnlyAudioBase.java b/library/src/main/java/com/pedro/library/base/OnlyAudioBase.java index 4f1780677..1783fcead 100644 --- a/library/src/main/java/com/pedro/library/base/OnlyAudioBase.java +++ b/library/src/main/java/com/pedro/library/base/OnlyAudioBase.java @@ -131,8 +131,7 @@ public boolean prepareAudio(int audioSource, int bitrate, int sampleRate, boolea return false; } onAudioInfoImp(isStereo, sampleRate); - return audioEncoder.prepareAudioEncoder(bitrate, sampleRate, isStereo, - microphoneManager.getMaxInputSize()); + return audioEncoder.prepareAudioEncoder(bitrate, sampleRate, isStereo); } public boolean prepareAudio(int bitrate, int sampleRate, boolean isStereo, boolean echoCanceler, @@ -269,17 +268,6 @@ public RecordController.Status getRecordStatus() { return recordController.getStatus(); } - /** - * Set a custom size of audio buffer input. - * If you set 0 or less you can disable it to use library default value. - * Must be called before of prepareAudio method. - * - * @param size in bytes. Recommended multiple of 1024 (2048, 4096, 8196, etc) - */ - public void setAudioMaxInputSize(int size) { - microphoneManager.setMaxInputSize(size); - } - /** * Mute microphone, can be called before, while and after stream. */ diff --git a/library/src/main/java/com/pedro/library/base/StreamBase.kt b/library/src/main/java/com/pedro/library/base/StreamBase.kt index 3b51def49..1ed1a601a 100644 --- a/library/src/main/java/com/pedro/library/base/StreamBase.kt +++ b/library/src/main/java/com/pedro/library/base/StreamBase.kt @@ -135,7 +135,7 @@ abstract class StreamBase( val audioResult = audioSource.init(sampleRate, isStereo, echoCanceler, noiseSuppressor) if (audioResult) { onAudioInfoImp(sampleRate, isStereo) - return audioEncoder.prepareAudioEncoder(bitrate, sampleRate, isStereo, audioSource.getMaxInputSize()) + return audioEncoder.prepareAudioEncoder(bitrate, sampleRate, isStereo) } return false } diff --git a/library/src/main/java/com/pedro/library/util/sources/audio/AudioFileSource.kt b/library/src/main/java/com/pedro/library/util/sources/audio/AudioFileSource.kt index a38118277..dc20ddcf4 100644 --- a/library/src/main/java/com/pedro/library/util/sources/audio/AudioFileSource.kt +++ b/library/src/main/java/com/pedro/library/util/sources/audio/AudioFileSource.kt @@ -97,10 +97,6 @@ class AudioFileSource( if (running) stop() } - override fun getMaxInputSize(): Int = audioDecoder.size - - override fun setMaxInputSize(size: Int) { } - fun mute() { audioDecoder.mute() } diff --git a/library/src/main/java/com/pedro/library/util/sources/audio/AudioSource.kt b/library/src/main/java/com/pedro/library/util/sources/audio/AudioSource.kt index 23413dbb6..49ee4ed92 100644 --- a/library/src/main/java/com/pedro/library/util/sources/audio/AudioSource.kt +++ b/library/src/main/java/com/pedro/library/util/sources/audio/AudioSource.kt @@ -44,6 +44,4 @@ abstract class AudioSource { abstract fun stop() abstract fun isRunning(): Boolean abstract fun release() - abstract fun getMaxInputSize(): Int - abstract fun setMaxInputSize(size: Int) } \ No newline at end of file diff --git a/library/src/main/java/com/pedro/library/util/sources/audio/InternalAudioSource.kt b/library/src/main/java/com/pedro/library/util/sources/audio/InternalAudioSource.kt index ad5cecc95..db5fa85a8 100644 --- a/library/src/main/java/com/pedro/library/util/sources/audio/InternalAudioSource.kt +++ b/library/src/main/java/com/pedro/library/util/sources/audio/InternalAudioSource.kt @@ -24,8 +24,10 @@ import android.os.Handler import android.os.HandlerThread import androidx.annotation.RequiresApi import com.pedro.encoder.Frame +import com.pedro.encoder.input.audio.CustomAudioEffect import com.pedro.encoder.input.audio.GetMicrophoneData import com.pedro.encoder.input.audio.MicrophoneManager +import com.pedro.encoder.input.audio.VolumeEffect import com.pedro.library.util.sources.MediaProjectionHandler /** @@ -40,7 +42,8 @@ class InternalAudioSource( ): AudioSource(), GetMicrophoneData { private val TAG = "InternalAudioSource" - private val microphone = MicrophoneManager(this) + private val internalVolumeEffect = VolumeEffect() + private val microphone = MicrophoneManager(this).apply { setCustomAudioEffect(internalVolumeEffect) } private var handlerThread = HandlerThread(TAG) private val mediaProjectionCallback = mediaProjectionCallback ?: object : MediaProjection.Callback() {} @@ -96,12 +99,6 @@ class InternalAudioSource( MediaProjectionHandler.mediaProjection?.unregisterCallback(mediaProjectionCallback) } - override fun getMaxInputSize(): Int = microphone.maxInputSize - - override fun setMaxInputSize(size: Int) { - microphone.maxInputSize = size - } - override fun inputPCMData(frame: Frame) { getMicrophoneData?.inputPCMData(frame) } @@ -115,4 +112,12 @@ class InternalAudioSource( } fun isMuted(): Boolean = microphone.isMuted + + fun setAudioEffect(effect: CustomAudioEffect) { + microphone.setCustomAudioEffect(effect) + } + + var internalVolume: Float + set(value) { internalVolumeEffect.volume = value } + get() = internalVolumeEffect.volume } \ No newline at end of file diff --git a/library/src/main/java/com/pedro/library/util/sources/audio/MicrophoneSource.kt b/library/src/main/java/com/pedro/library/util/sources/audio/MicrophoneSource.kt index e7f10df7b..73e2eb76b 100644 --- a/library/src/main/java/com/pedro/library/util/sources/audio/MicrophoneSource.kt +++ b/library/src/main/java/com/pedro/library/util/sources/audio/MicrophoneSource.kt @@ -21,8 +21,10 @@ import android.media.MediaRecorder import android.os.Build import androidx.annotation.RequiresApi import com.pedro.encoder.Frame +import com.pedro.encoder.input.audio.CustomAudioEffect import com.pedro.encoder.input.audio.GetMicrophoneData import com.pedro.encoder.input.audio.MicrophoneManager +import com.pedro.encoder.input.audio.VolumeEffect /** * Created by pedro on 12/1/24. @@ -31,7 +33,8 @@ class MicrophoneSource( var audioSource: Int = MediaRecorder.AudioSource.DEFAULT, ): AudioSource(), GetMicrophoneData { - private val microphone = MicrophoneManager(this) + private val microphoneVolumeEffect = VolumeEffect() + private val microphone = MicrophoneManager(this).apply { setCustomAudioEffect(microphoneVolumeEffect) } private var preferredDevice: AudioDeviceInfo? = null override fun create(sampleRate: Int, isStereo: Boolean, echoCanceler: Boolean, noiseSuppressor: Boolean): Boolean { @@ -74,12 +77,6 @@ class MicrophoneSource( override fun release() {} - override fun getMaxInputSize(): Int = microphone.maxInputSize - - override fun setMaxInputSize(size: Int) { - microphone.maxInputSize = size - } - override fun inputPCMData(frame: Frame) { getMicrophoneData?.inputPCMData(frame) } @@ -93,4 +90,12 @@ class MicrophoneSource( } fun isMuted(): Boolean = microphone.isMuted + + fun setAudioEffect(effect: CustomAudioEffect) { + microphone.setCustomAudioEffect(effect) + } + + var microphoneVolume: Float + set(value) { microphoneVolumeEffect.volume = value } + get() = microphoneVolumeEffect.volume } \ No newline at end of file diff --git a/library/src/main/java/com/pedro/library/util/sources/audio/MixAudioSource.kt b/library/src/main/java/com/pedro/library/util/sources/audio/MixAudioSource.kt new file mode 100644 index 000000000..998d96b8c --- /dev/null +++ b/library/src/main/java/com/pedro/library/util/sources/audio/MixAudioSource.kt @@ -0,0 +1,158 @@ +/* + * + * * Copyright (C) 2024 pedroSG94. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.pedro.library.util.sources.audio + +import android.media.projection.MediaProjection +import android.os.Build +import androidx.annotation.RequiresApi +import com.pedro.common.trySend +import com.pedro.encoder.Frame +import com.pedro.encoder.audio.AudioEncoder +import com.pedro.encoder.input.audio.GetMicrophoneData +import com.pedro.encoder.input.audio.VolumeEffect +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.launch +import kotlinx.coroutines.runInterruptible +import java.util.concurrent.BlockingQueue +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.TimeUnit + +/** + * Mix microphone and internal audio sources in one source to allow send both at the same time. + * NOTES: + * Recommended configure prepareAudio with: + * echoCanceler = true, + * noiseSuppressor = true + * This is to avoid echo in microphone track. + * + * Recommended increase microphone volume to 2f, + * because the internal audio normally is higher and you can't hear audio track properly + */ +@RequiresApi(Build.VERSION_CODES.Q) +class MixAudioSource( + mediaProjection: MediaProjection +): AudioSource() { + + private val microphoneVolumeEffect = VolumeEffect() + private val internalVolumeEffect = VolumeEffect() + private val microphone = MicrophoneSource().apply { setAudioEffect(microphoneVolumeEffect) } + private val internal = InternalAudioSource(mediaProjection).apply { setAudioEffect(internalVolumeEffect) } + + private val scope = CoroutineScope(Dispatchers.IO) + private val microphoneQueue: BlockingQueue = LinkedBlockingQueue(500) + private val internalQueue: BlockingQueue = LinkedBlockingQueue(500) + private var running = false + //We need read with a higher buffer to get enough time to mix it + private val inputSize = AudioEncoder.inputSize + + override fun create(sampleRate: Int, isStereo: Boolean, echoCanceler: Boolean, noiseSuppressor: Boolean): Boolean { + return microphone.init(sampleRate, isStereo, echoCanceler, noiseSuppressor) && internal.init(sampleRate, isStereo, echoCanceler, noiseSuppressor) + } + + override fun start(getMicrophoneData: GetMicrophoneData) { + this.getMicrophoneData = getMicrophoneData + if (!isRunning()) { + microphoneQueue.clear() + internalQueue.clear() + microphone.start(callback1) + internal.start(callback2) + running = true + scope.launch { + val min = Byte.MIN_VALUE.toInt() + val max = Byte.MAX_VALUE.toInt() + + while (running) { + runCatching { + val frame1 = async { runInterruptible { microphoneQueue.poll(1, TimeUnit.SECONDS) } } + val frame2 = async { runInterruptible { internalQueue.poll(1, TimeUnit.SECONDS) } } + val r = awaitAll(frame1, frame2) + async { + val microphoneBuffer = r[0]?.buffer ?: return@async + val internalBuffer = r[1]?.buffer ?: return@async + val mixBuffer = ByteArray(inputSize) + for (i in mixBuffer.indices) { //mix buffers with same config + mixBuffer[i] = (microphoneBuffer[i] + internalBuffer[i]).coerceIn(min, max).toByte() + } + getMicrophoneData.inputPCMData(Frame(mixBuffer, 0, mixBuffer.size, r[0].timeStamp)) + } + }.exceptionOrNull() + } + } + } + } + + override fun stop() { + if (isRunning()) { + getMicrophoneData = null + microphone.stop() + internal.stop() + running = false + } + } + + override fun release() { + microphone.release() + internal.release() + } + + override fun isRunning(): Boolean = running + + private val callback1 = object: GetMicrophoneData { + override fun inputPCMData(frame: Frame) { + microphoneQueue.trySend(frame) + } + } + + private val callback2 = object: GetMicrophoneData { + override fun inputPCMData(frame: Frame) { + internalQueue.trySend(frame) + } + } + + var microphoneVolume: Float + set(value) { microphoneVolumeEffect.volume = value } + get() = microphoneVolumeEffect.volume + + var internalVolume: Float + set(value) { internalVolumeEffect.volume = value } + get() = internalVolumeEffect.volume + + fun muteMicrophone() { + microphone.mute() + } + + fun unMuteMicrophone() { + microphone.unMute() + } + + fun isMicrophoneMuted(): Boolean = microphone.isMuted() + + fun muteInternalAudio() { + internal.mute() + } + + fun unMuteInternalAudio() { + internal.unMute() + } + + fun isInternalAudioMuted(): Boolean = internal.isMuted() +} \ No newline at end of file diff --git a/library/src/main/java/com/pedro/library/util/sources/audio/NoAudioSource.kt b/library/src/main/java/com/pedro/library/util/sources/audio/NoAudioSource.kt index b40da58d1..b522d6d95 100644 --- a/library/src/main/java/com/pedro/library/util/sources/audio/NoAudioSource.kt +++ b/library/src/main/java/com/pedro/library/util/sources/audio/NoAudioSource.kt @@ -43,7 +43,4 @@ class NoAudioSource: AudioSource() { override fun release() {} override fun isRunning(): Boolean = running - override fun getMaxInputSize(): Int = 0 - - override fun setMaxInputSize(size: Int) {} } \ No newline at end of file