Skip to content

Commit

Permalink
Hooks up new FaceTrackingManager
Browse files Browse the repository at this point in the history
  • Loading branch information
@IanCrossCD
IanCrossCD committed Nov 21, 2023
1 parent 2211b40 commit 5da3d58
Show file tree
Hide file tree
Showing 7 changed files with 90 additions and 276 deletions.
216 changes: 25 additions & 191 deletions app/src/main/java/com/willowtree/vocable/MainActivity.kt
View file
Original file line number Diff line number Diff line change
@@ -1,52 +1,36 @@
package com.willowtree.vocable

import android.Manifest
import android.app.ActivityManager
import android.content.Context
import android.graphics.Rect
import android.hardware.display.DisplayManager
import android.os.Bundle
import android.util.DisplayMetrics
import android.view.Surface
import android.view.View
import android.view.ViewGroup
import android.widget.Toast
import androidx.appcompat.app.AppCompatActivity
import androidx.core.view.children
import androidx.core.view.isVisible
import com.google.ar.core.ArCoreApk
import com.vmadalin.easypermissions.EasyPermissions
import com.vmadalin.easypermissions.dialogs.SettingsDialog
import com.willowtree.vocable.customviews.PointerListener
import com.willowtree.vocable.customviews.PointerView
import com.willowtree.vocable.databinding.ActivityMainBinding
import com.willowtree.vocable.facetracking.FaceTrackFragment
import com.willowtree.vocable.facetracking.FaceTrackingViewModel
import com.willowtree.vocable.settings.selectionmode.HeadTrackingPermissionState
import com.willowtree.vocable.settings.selectionmode.SelectionModeViewModel
import com.willowtree.vocable.utils.VocableSharedPreferences
import com.willowtree.vocable.utils.FaceTrackingManager
import com.willowtree.vocable.utils.FaceTrackingPointerUpdates
import com.willowtree.vocable.utils.IVocableSharedPreferences
import com.willowtree.vocable.utils.VocableTextToSpeech
import io.github.inflationx.viewpump.ViewPumpContextWrapper
import org.koin.android.ext.android.inject
import org.koin.androidx.scope.ScopeActivity
import org.koin.androidx.viewmodel.ViewModelOwner
import org.koin.androidx.viewmodel.ext.android.viewModel
import timber.log.Timber

class MainActivity : AppCompatActivity(),
EasyPermissions.PermissionCallbacks {
class MainActivity : ScopeActivity() {

private val minOpenGlVersion = 3.0
private val displayMetrics = DisplayMetrics()
private var currentView: View? = null
private var paused = false
private lateinit var binding: ActivityMainBinding
private val sharedPrefs: VocableSharedPreferences by inject()
private var hasSetupAr: Boolean = false
private val sharedPrefs: IVocableSharedPreferences by inject()
private val allViews = mutableListOf<View>()

private val selectionModeViewModel: SelectionModeViewModel by viewModel(owner = {
ViewModelOwner.from(this)
})
private val faceTrackingManager: FaceTrackingManager by inject()

private val faceTrackingViewModel: FaceTrackingViewModel by viewModel(owner = {
ViewModelOwner.from(this)
})
Expand All @@ -58,24 +42,14 @@ class MainActivity : AppCompatActivity(),
binding.pointerView.isVisible = false
setContentView(binding.root)

val canUseHeadTracking = BuildConfig.USE_HEAD_TRACKING
val isSupportedDevice = checkIsSupportedDeviceOrFinish()

if (canUseHeadTracking && isSupportedDevice) {
selectionModeViewModel.headTrackingPermissionState.observe(this) { headTrackingState ->
when (headTrackingState) {
HeadTrackingPermissionState.PermissionRequested -> requestPermissions()
HeadTrackingPermissionState.Enabled -> {
togglePointerVisible(true)
setupArTracking()
}

HeadTrackingPermissionState.Disabled -> {
togglePointerVisible(false)
}
faceTrackingManager.initialize(
activity = this,
object : FaceTrackingPointerUpdates {
override fun toggleVisibility(visible: Boolean) {
binding.pointerView.isVisible = visible
}
}
}
)

faceTrackingViewModel.showError.observe(this) { showError ->
if (!sharedPrefs.getHeadTrackingEnabled()) {
Expand All @@ -90,6 +64,11 @@ class MainActivity : AppCompatActivity(),
getPointerView().isVisible = !showError
}


faceTrackingViewModel.pointerLocation.observe(this) {
updatePointer(it.x, it.y)
}

supportActionBar?.hide()
VocableTextToSpeech.initialize(this)

Expand All @@ -98,75 +77,6 @@ class MainActivity : AppCompatActivity(),
}
}

private fun togglePointerVisible(visible: Boolean) {
binding.pointerView.isVisible = if (!BuildConfig.USE_HEAD_TRACKING) false else visible
}

private fun setupArTracking() {

if (!hasSetupAr) {

hasSetupAr = true

if (supportFragmentManager.findFragmentById(R.id.face_fragment) == null) {
supportFragmentManager
.beginTransaction()
.replace(R.id.face_fragment, FaceTrackFragment())
.commitAllowingStateLoss()
}

faceTrackingViewModel.pointerLocation.observe(this) {
updatePointer(it.x, it.y)
}

windowManager.defaultDisplay.getMetrics(displayMetrics)

val displayListener = object : DisplayManager.DisplayListener {

private var orientation = windowManager.defaultDisplay.rotation

override fun onDisplayChanged(displayId: Int) {
val newOrientation = windowManager.defaultDisplay.rotation
// Only reset FaceTrackFragment if device is rotated 180 degrees
when (orientation) {
Surface.ROTATION_0 -> {
if (newOrientation == Surface.ROTATION_180) {
resetFaceTrackFragment("${Surface.ROTATION_180}")
}
}

Surface.ROTATION_90 -> {
if (newOrientation == Surface.ROTATION_270) {
resetFaceTrackFragment("${Surface.ROTATION_270}")
}
}

Surface.ROTATION_180 -> {
if (newOrientation == Surface.ROTATION_0) {
resetFaceTrackFragment("${Surface.ROTATION_0}")
}
}

Surface.ROTATION_270 -> {
if (newOrientation == Surface.ROTATION_90) {
resetFaceTrackFragment("${Surface.ROTATION_90}")
}
}
}
orientation = newOrientation
}

override fun onDisplayAdded(displayId: Int) = Unit

override fun onDisplayRemoved(displayId: Int) = Unit
}

val displayManager = getSystemService(Context.DISPLAY_SERVICE) as DisplayManager
displayManager.registerDisplayListener(displayListener, null)

}
}

override fun attachBaseContext(newBase: Context) {
super.attachBaseContext(ViewPumpContextWrapper.wrap(newBase))
}
Expand Down Expand Up @@ -210,35 +120,19 @@ class MainActivity : AppCompatActivity(),
}
}

/**
* If the device rotates 180 degrees (portrait to portrait/landscape to landscape), the
* activity won't be destroyed and recreated. This means that the FaceTrackFragment will not
* reset its camera positioning. The only way to reset it currently is to create a new
* instance of the fragment and add it to the activity.
* @param tag The tag to use for the FaceTrackFragment, should be unique to the orientation
*/
private fun resetFaceTrackFragment(tag: String) {
if (!supportFragmentManager.isDestroyed && supportFragmentManager.findFragmentByTag(tag) == null) {
supportFragmentManager
.beginTransaction()
.replace(R.id.face_fragment, FaceTrackFragment(), tag)
.commitAllowingStateLoss()
}
}

private fun updatePointer(x: Float, y: Float) {
var newX = x
var newY = y
if (x < 0) {
newX = 0f
} else if (x > displayMetrics.widthPixels) {
newX = displayMetrics.widthPixels.toFloat()
} else if (x > faceTrackingManager.displayMetrics.widthPixels) {
newX = faceTrackingManager.displayMetrics.widthPixels.toFloat()
}

if (y < 0) {
newY = 0f
} else if (y > displayMetrics.heightPixels) {
newY = displayMetrics.heightPixels.toFloat()
} else if (y > faceTrackingManager.displayMetrics.heightPixels) {
newY = faceTrackingManager.displayMetrics.heightPixels.toFloat()
}
getPointerView().updatePointerPosition(newX, newY)
getPointerView().bringToFront()
Expand Down Expand Up @@ -289,73 +183,13 @@ class MainActivity : AppCompatActivity(),
return rect.contains(view2Rect.centerX(), view2Rect.centerY())
}

/**
* Returns false and displays an error message if Sceneform can not run, true if Sceneform can run
* on this device.
*
*
* Sceneform requires Android N on the device as well as OpenGL 3.0 capabilities.
*
*
* Finishes the activity if Sceneform can not run
*/
private fun checkIsSupportedDeviceOrFinish(): Boolean {
if (ArCoreApk.getInstance().checkAvailability(this) === ArCoreApk.Availability.UNSUPPORTED_DEVICE_NOT_CAPABLE) {
Timber.e("TAG", "Augmented Faces requires ARCore.")
Toast.makeText(this, "Augmented Faces requires ARCore", Toast.LENGTH_LONG).show()
finish()
return false
}
val openGlVersionString =
(getSystemService(Context.ACTIVITY_SERVICE) as ActivityManager)
.deviceConfigurationInfo
.glEsVersion
if (java.lang.Double.parseDouble(openGlVersionString) < minOpenGlVersion) {
Timber.e("TAG", "Sceneform requires OpenGL ES 3.0 later")
Toast.makeText(this, "Sceneform requires OpenGL ES 3.0 or later", Toast.LENGTH_LONG)
.show()
finish()
return false
}
return true
}

/**
* PERMISSIONS
*/

private fun requestPermissions() {
if (EasyPermissions.hasPermissions(this, Manifest.permission.CAMERA)) {
selectionModeViewModel.enableHeadTracking()
} else {
// Do not have permissions, request them now
EasyPermissions.requestPermissions(
host = this,
rationale = "Allow camera permissions to enable Head Tracking.",
requestCode = REQUEST_CAMERA_PERMISSION_CODE,
perms = arrayOf(Manifest.permission.CAMERA)
)
}
}

override fun onRequestPermissionsResult(requestCode: Int, permissions: Array<out String>, grantResults: IntArray) {
super.onRequestPermissionsResult(requestCode, permissions, grantResults)
// EasyPermissions handles the request result.
EasyPermissions.onRequestPermissionsResult(requestCode, permissions, grantResults, this)
faceTrackingManager.onRequestPermissionsResult(requestCode, permissions, grantResults)
}

override fun onPermissionsDenied(requestCode: Int, perms: List<String>) {
if (EasyPermissions.somePermissionPermanentlyDenied(this@MainActivity, perms)) {
SettingsDialog.Builder(this@MainActivity).build().show()
} else {
selectionModeViewModel.disableHeadTracking()
}
}

override fun onPermissionsGranted(requestCode: Int, perms: List<String>) {
requestPermissions()
}

}

const val REQUEST_CAMERA_PERMISSION_CODE = 5504
}
13 changes: 2 additions & 11 deletions app/src/main/java/com/willowtree/vocable/facetracking/FaceTrackFragment.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,10 @@ import com.google.ar.core.Config
import com.google.ar.core.Session
import com.google.ar.sceneform.ux.ArFragment
import com.vmadalin.easypermissions.EasyPermissions
import com.willowtree.vocable.settings.selectionmode.HeadTrackingPermissionState
import com.willowtree.vocable.settings.selectionmode.SelectionModeViewModel
import org.koin.androidx.viewmodel.ViewModelOwner
import org.koin.androidx.viewmodel.ext.android.viewModel
import java.util.EnumSet


class FaceTrackFragment : ArFragment() {

private val selectionModeViewModel: SelectionModeViewModel by viewModel(owner = {
ViewModelOwner.from(this)
})
private val viewModel: FaceTrackingViewModel by activityViewModels()

override fun getSessionConfiguration(session: Session): Config {
Expand All @@ -34,9 +26,8 @@ class FaceTrackFragment : ArFragment() {
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)

selectionModeViewModel.headTrackingPermissionState.observe(this) { headTrackingPermissionState ->
if (headTrackingPermissionState == HeadTrackingPermissionState.PermissionRequested) return@observe
enableFaceTracking(headTrackingPermissionState == HeadTrackingPermissionState.Enabled)
viewModel.headTrackingEnabledLd.observe(this) {
enableFaceTracking(it)
}
}

Expand Down
7 changes: 7 additions & 0 deletions app/src/main/java/com/willowtree/vocable/facetracking/FaceTrackingViewModel.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ import androidx.lifecycle.*
import com.google.ar.core.AugmentedFace
import com.google.ar.sceneform.math.Vector3
import com.willowtree.vocable.R
import com.willowtree.vocable.utils.IFaceTrackingPermissions
import com.willowtree.vocable.utils.VocableSharedPreferences
import com.willowtree.vocable.utils.isEnabled
import kotlinx.coroutines.*
import org.koin.core.component.KoinComponent
import org.koin.core.component.get
Expand All @@ -19,6 +21,9 @@ class FaceTrackingViewModel : ViewModel(), LifecycleObserver, KoinComponent {
private const val FACE_DETECTION_TIMEOUT = 1000
}

private val headTrackingPermissions: IFaceTrackingPermissions by inject()
val headTrackingEnabledLd = headTrackingPermissions.permissionState.asLiveData().map { it.isEnabled() }

private val viewModelJob = SupervisorJob()
private val backgroundScope = CoroutineScope(viewModelJob + Dispatchers.IO)

Expand All @@ -31,6 +36,7 @@ class FaceTrackingViewModel : ViewModel(), LifecycleObserver, KoinComponent {
VocableSharedPreferences.KEY_SENSITIVITY -> {
sensitivity = sharedPrefs.getSensitivity()
}

VocableSharedPreferences.KEY_HEAD_TRACKING_ENABLED -> {
headTrackingEnabled = sharedPrefs.getHeadTrackingEnabled()
}
Expand Down Expand Up @@ -105,6 +111,7 @@ class FaceTrackingViewModel : ViewModel(), LifecycleObserver, KoinComponent {
oldVector = Vector3(x, y, z)
liveAdjustedVector.postValue(oldVector)
}

else -> {
if (!isTablet) {
y *= 2F
Expand Down
9 changes: 3 additions & 6 deletions app/src/main/java/com/willowtree/vocable/settings/selectionmode/SelectionModeFragment.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,12 @@ import androidx.navigation.fragment.findNavController
import com.willowtree.vocable.BaseFragment
import com.willowtree.vocable.BindingInflater
import com.willowtree.vocable.databinding.FragmentSelectionModeBinding
import org.koin.androidx.viewmodel.ViewModelOwner
import org.koin.androidx.viewmodel.ext.android.viewModel

class SelectionModeFragment : BaseFragment<FragmentSelectionModeBinding>() {

override val bindingInflater: BindingInflater<FragmentSelectionModeBinding> = FragmentSelectionModeBinding::inflate
private val viewModel: SelectionModeViewModel by viewModel(owner = {
ViewModelOwner.from(requireActivity())
})
private val viewModel: SelectionModeViewModel by viewModel()

override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
super.onViewCreated(view, savedInstanceState)
Expand All @@ -23,8 +20,8 @@ class SelectionModeFragment : BaseFragment<FragmentSelectionModeBinding>() {
findNavController().popBackStack()
}

viewModel.headTrackingPermissionState.observe(viewLifecycleOwner) {
binding.selectionModeOptions.headTrackingSwitch.isChecked = it == HeadTrackingPermissionState.Enabled
viewModel.headTrackingEnabled.observe(viewLifecycleOwner) {
binding.selectionModeOptions.headTrackingSwitch.isChecked = it
}

binding.selectionModeOptions.headTrackingContainer.setOnClickListener {
Expand Down
Loading

0 comments on commit 5da3d58

Please sign in to comment.