Docs & API

Integrate into your Android app

Register an SDK Key

Get your free SDK key on https://flow.org.es, usage limits may apply. SDK Keys are linked to your bundle ID, please check Key before distributing to the App Store.

Installing the SDK

Maven

Step 1: Open your app's build.gradle. Step 2: Add the Maven package and it's dependencies: 
    
dependencies {
      // recommended for starting floworg
      implementation ("androidx.lifecycle:lifecycle-runtime-ktx:2.6.1'")
      
      // CameraX core library
      val camerax_version = ("1.4.1")
      
      implementation("androidx.camera:camera-core:$camerax_version")
      implementation("androidx.camera:camera-camera2:$camerax_version")
      implementation("androidx.camera:camera-lifecycle:$camerax_version")
      implementation("androidx.camera:camera-view:$camerax_version")

      implementation("com.google.flogger:flogger:latest.release")
      implementation("com.google.flogger:flogger-system-backend:latest.release")
      implementation("com.google.guava:guava:27.0.1-android")
      implementation("com.google.protobuf:protobuf-javalite:3.19.1")

      implementation("com.microsoft.onnxruntime:onnxruntime-android:latest.release")
      implementation("flow.org.es:floworg-mp:0.1")
      implementation("flow.org.es:floworg-core:0.5")
}

Add Camera Permissions Check To Main App

Next, you have to explicitly request access to the camera, which will provide the Android standard camera permission prompt. This is only a demo implementation, as you'd typically want to give the user an idea of what your app does first and why camera permissions help them. 
    
class MainActivity : ComponentActivity() {
  private var hasPermissions = mutableStateOf(false)

  override fun onCreate(savedInstanceState: Bundle?) {
      hasPermissions.value = PermissionsHelper.checkAndRequestCameraPermissions(this)
  }

  override fun onResume() {
      super.onResume()

      hasPermissions.value = PermissionsHelper.checkAndRequestCameraPermissions(this)
      if (!hasPermissions.value) {
          return
      }
  }
We use this standard permission helper: 
    
public class PermissionsHelper {
  companion object {
      fun checkAndRequestCameraPermissions(context: Activity): Boolean {
          val hasPermission = cameraPermissionsGranted(context)
          if (!hasPermission) {
              ActivityCompat.requestPermissions(context, arrayOf(Manifest.permission.CAMERA), 0);
          }
          return hasPermission
      }

      fun cameraPermissionsGranted(context: Activity): Boolean {
          return ContextCompat.checkSelfPermission(
              context,
              Manifest.permission.CAMERA
          ) == PackageManager.PERMISSION_GRANTED
      }
  }
}

Initialize the SDK

    
private var floworg: Floworg =
      Floworg(this, sdkKey = "YOUR SDK KEY HERE") 
      // register for your free key at https://flow.org.es/contact-us

Attach SDK to Views

This is our standard boilerplate implentation providing:
  1. Camera Permission Checking
  2. A fullscreen camera display.
  3. A canvas overlay showing the AI user's landmarks.
  4. Sensible memory releasing when the view is no longer visible.
Declare an xml definition the floworg camera and overlay stack. 
    
<FrameLayout
  android:id="@+id/floworg_camera_and_overlay_view"
  android:layout_width="match_parent"
  android:layout_height="0dp"
  app:layout_constraintTop_toTopOf="parent"
  app:layout_constraintBottom_toBottomOf="parent"
  app:layout_constraintStart_toStartOf="parent"
  app:layout_constraintEnd_toEndOf="parent">
</FrameLayout>
Declare a reference to the camera and overlay view group. 
    
private var cameraAndOverlay: ViewGroup? = null
Plus a reference to the camera view explicitly. 
    
private var cameraSwitchView: FloworgCameraSwitchView? = null
Connect up the views 
    
override fun onCreate(savedInstanceState: Bundle?) {
        cameraAndOverlay = findViewById(R.id.floworg_camera_and_overlay_view)
        cameraSwitchView = FloworgCameraSwitchView(this, floworg)
        cameraAndOverlay?.addView(cameraSwitchView)
And on resume, start the camera and overlay stack and floworg. This is a bit simplified as you need to request camera permissions. 
    
override fun onResume() {
  super.onResume()
  lifecycleScope.launch {
      cameraSwitchView?.start(useFrontCamera)!!
      floworg.start(
              arrayOf(Feature.RangeOfMotion(RangeOfMotion.Shoulder(Side.LEFT, false))),
              onFrame = { status, overlay, features, feedback, landmarks ->
                  landmarks?.let { println(it.allLandmarksForBody()[0].x) }
              }
      )
  }
}
For Compose apps, apps we assume you're using a ComponentActivity. We provide a supported wrapper around the legacy camera view 
    
@Composable
fun FloworgCamera(
        floworg: Floworg,
        useFrontCamera: Boolean = true,
) {
    val coroutineScope = rememberCoroutineScope()
    AndroidView(
            factory = { context ->
                val cameraSwitchView = FloworgCameraSwitchView(context, floworg)
                coroutineScope.launch { cameraSwitchView.start(useFrontCamera) }
                cameraSwitchView
            }
    )
}
Which you can use in your Compose UI like this: 
    
setContent {
    BasicDemoTheme {
        // assuming you have camera permissions
        FloworgCamera(floworg = Floworg)
    }
}
    
    
    
class MainActivity : ComponentActivity() {
  private var statusText = mutableStateOf("Powered by flow.org.es")
  private var floworg: Floworg =
          Floworg(
                  this,
                  sdkKey = "YOUR SDK KEY HERE"
          ) // register for your free key at https://flow.org.es/contact-us

  private var hasPermissions = mutableStateOf(false)

  override fun onCreate(savedInstanceState: Bundle?) {
      enableEdgeToEdge()
      super.onCreate(savedInstanceState)
      WindowInsetsCompat.Type.navigationBars()

      hasPermissions.value = PermissionsHelper.checkAndRequestCameraPermissions(this)

      setContent {
          BasicDemoTheme {
              Scaffold(modifier = Modifier.fillMaxSize()) { padding ->
                  Box(modifier = Modifier.fillMaxSize().padding(padding)) {
                      if (hasPermissions.value) {
                          FloworgCamera(floworg = floworg)
                      } else {
                          Text(
                                  text = "Camera permissions are required",
                                  color = Color.White,
                                  fontSize = 16.sp,
                                  modifier = Modifier.align(Alignment.Center),
                                  textAlign = TextAlign.Center
                          )
                      }

                      Box(
                              modifier = Modifier.fillMaxSize().padding(bottom = 64.dp),
                              contentAlignment = Alignment.BottomCenter
                      ) {
                          Text(
                                  text = statusText.value,
                                  color = Color.White,
                                  fontSize = 16.sp,
                                  modifier = Modifier.fillMaxWidth(),
                                  textAlign = TextAlign.Center
                          )
                      }
                  }
              }
          }
      }
  }

  override fun onResume() {
      super.onResume()

      hasPermissions.value = PermissionsHelper.checkAndRequestCameraPermissions(this)
      if (!hasPermissions.value) {
          return
      }

      window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
      window.setDecorFitsSystemWindows(false)
      val windowInsetsController = WindowInsetsControllerCompat(window, window.decorView)
      windowInsetsController.hide(android.view.WindowInsets.Type.navigationBars())

      lifecycleScope.launch {
          floworg.start(
                  arrayOf(Feature.Overlay(Landmarks.Group.WholeBody())),
                  onFrame = { status, overlay, features, feedback, landmarks ->
                      if (status is Status.Success) {
                          runOnUiThread {
                              statusText.value =
                                      "Powered by flow.org.es v$ {floworg.floworgVersion()}
$ {status.fps} fps"
                          }
                      }
                  }
          )
      }
  }

  override fun onPause() {
      super.onPause()
      window.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
      floworg.stop()
  }
}

@Composable
fun FloworgCamera(
      floworg: Floworg,
      useFrontCamera: Boolean = true,
) {
  val coroutineScope = rememberCoroutineScope()
  AndroidView(
          factory = { context ->
              val cameraSwitchView = FloworgCameraSwitchView(context, floworg)
              coroutineScope.launch { cameraSwitchView.start(useFrontCamera) }
              cameraSwitchView
          }
  )
}

View the raw landmarks

    
landmarks?.let { println(it.allLandmarksForBody()[0]) }

Access the overlay image.

For performance reasons, the overlay is returned as a canvas. This differs from iOS, use PixelCopy to efficiently convert the canvas to a bitmap. 
    
floworg.start(
    arrayOf(Feature.Overlay(Landmarks.Group.WholeBody())),
    onFrame = { status, overlay, features, feedback, landmarks ->         
        overlay?.let { outputOverlay ->
            val bitmap =
                    Bitmap.createBitmap(
                            outputOverlay.width,
                            outputOverlay.height,
                            Bitmap.Config.ARGB_8888
                    )

            PixelCopy.request(
                    outputOverlay,
                    bitmap,
                    { copyResult ->
                        if (copyResult == PixelCopy.SUCCESS) {
                            println(
                                    "Saved Bitmap $ {bitmap.height}, $ {bitmap.width}"
                            )
                        }
                    },
                    Handler(Looper.getMainLooper())
            )
        }
    }
)