diff --git a/.gitignore b/.gitignore index 7e65384..db0eb68 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ local.properties .gradle app/.cxx app/src/main/assets/*/ +build/ +app/release/ diff --git a/app/src/main/cpp/native-lib.cpp b/app/src/main/cpp/native-lib.cpp index 7ed866f..a2671e3 100644 --- a/app/src/main/cpp/native-lib.cpp +++ b/app/src/main/cpp/native-lib.cpp @@ -21,7 +21,10 @@ namespace { - struct image_info { + struct image { + uint8_t * input_image_data0; + uint8_t * input_image_data1; + uint8_t * input_image_data2; int width; int height; int row_stride0; @@ -30,11 +33,7 @@ namespace int pixel_stride0; int pixel_stride1; int pixel_stride2; - int input_orientation; - int output_orientation; int pixel_format; - int image_format; - bool require_mirroring; }; int get_int_field(const char* field_name, JNIEnv* env, jobject object, jclass object_class) @@ -43,34 +42,36 @@ namespace return env->GetIntField(object, field_id); } - bool get_bool_field(const char* field_name, JNIEnv* env, jobject object, jclass object_class) + uint8_t * get_byte_buffer_field(const char* field_name, JNIEnv* env, jobject object, jclass object_class) { - jfieldID field_id = env->GetFieldID(object_class, field_name, "Z"); - return env->GetBooleanField(object, field_id); + jfieldID fid = env-> GetFieldID (object_class, field_name, "Ljava/nio/ByteBuffer;"); + jobject data = env-> GetObjectField (object, fid); + return static_cast(env->GetDirectBufferAddress(data)); } - image_info get_image_info(JNIEnv* env, jobject image_info) + image get_image(JNIEnv* env, jobject jimage) { - jclass cls = env->GetObjectClass(image_info); + jclass image_class = env->GetObjectClass(jimage); + return { - get_int_field("width", env, image_info, cls), - get_int_field("height", env, image_info, cls), - get_int_field("rowStride0", env, image_info, cls), - get_int_field("rowStride1", env, image_info, cls), - get_int_field("rowStride2", env, image_info, cls), - get_int_field("pixelStride0", env, image_info, cls), - get_int_field("pixelStride1", env, image_info, cls), - get_int_field("pixelStride2", env, image_info, cls), - get_int_field("inputOrientation", env, image_info, cls), - get_int_field("outputOrientation", env, image_info, cls), - get_int_field("pixelFormat", env, image_info, cls), - get_int_field("imageFormat", env, image_info, cls), - get_bool_field("requireMirroring", env, image_info, cls) + get_byte_buffer_field("mPlane0", env, jimage, image_class), + get_byte_buffer_field("mPlane1", env, jimage, image_class), + get_byte_buffer_field("mPlane2", env, jimage, image_class), + get_int_field("mWidth", env, jimage, image_class), + get_int_field("mHeight", env, jimage, image_class), + get_int_field("mRowStride0", env, jimage, image_class), + get_int_field("mRowStride1", env, jimage, image_class), + get_int_field("mRowStride2", env, jimage, image_class), + get_int_field("mPixelStride0", env, jimage, image_class), + get_int_field("mPixelStride1", env, jimage, image_class), + get_int_field("mPixelStride2", env, jimage, image_class), + get_int_field("mPixelFormat", env, jimage, image_class) }; } - bnb::oep::interfaces::image_format get_image_format(int output_image_format) { - switch (output_image_format) { + bnb::oep::interfaces::image_format get_image_format(jint jimage_format) { + int image_format = static_cast(jimage_format); + switch (image_format) { case 1: return bnb::oep::interfaces::image_format::nv12_bt601_full; case 2: @@ -81,11 +82,7 @@ namespace } std::vector create_planes_from_format( - uint8_t* input_image_data0, - uint8_t* input_image_data1, - uint8_t* input_image_data2, - bnb::oep::interfaces::image_format format, - const image_info& image_info) + const image& img, bnb::oep::interfaces::image_format format) { // only nv12_bt601_full and i420_bt601_full are supported assert(format == bnb::oep::interfaces::image_format::nv12_bt601_full @@ -93,43 +90,43 @@ namespace using ns_pb = bnb::oep::interfaces::pixel_buffer; if(format == bnb::oep::interfaces::image_format::nv12_bt601_full) { - int y_size = image_info.row_stride0 * image_info.height; - int uv_size = image_info.row_stride1 * image_info.height / 2; + int y_size = img.row_stride0 * img.height; + int uv_size = img.row_stride1 * img.height / 2; - ns_pb::plane_sptr y_plane_data(new ns_pb::plane_sptr::element_type[y_size], [](uint8_t* ptr) { /* DO NOTHING */ }); - std::memcpy(y_plane_data.get(), input_image_data0, y_size); + ns_pb::plane_sptr y_plane_data(new ns_pb::plane_sptr::element_type[y_size], [](uint8_t* ptr) { delete[] ptr; }); + std::memcpy(y_plane_data.get(), img.input_image_data0, y_size); - ns_pb::plane_sptr uv_plane_data(new ns_pb::plane_sptr::element_type[uv_size], [](uint8_t* ptr) { /* DO NOTHING */ }); - std::memcpy(uv_plane_data.get(), input_image_data1, uv_size); + ns_pb::plane_sptr uv_plane_data(new ns_pb::plane_sptr::element_type[uv_size], [](uint8_t* ptr) { delete[] ptr; }); + std::memcpy(uv_plane_data.get(), img.input_image_data1, uv_size); - ns_pb::plane_data y_plane{std::move(y_plane_data), static_cast(y_size), image_info.row_stride0}; - ns_pb::plane_data uv_plane{std::move(uv_plane_data), static_cast(uv_size), image_info.row_stride1}; + ns_pb::plane_data y_plane{std::move(y_plane_data), static_cast(y_size), img.row_stride0}; + ns_pb::plane_data uv_plane{std::move(uv_plane_data), static_cast(uv_size), img.row_stride1}; std::vector planes{std::move(y_plane), std::move(uv_plane)}; return planes; } if(format == bnb::oep::interfaces::image_format::i420_bt601_full) { - int y_size = image_info.row_stride0 * image_info.height; - int u_size = image_info.row_stride1 * image_info.height / 4; - int v_size = image_info.row_stride2 * image_info.height/ 4; + int y_size = img.row_stride0 * img.height; + int u_size = img.row_stride1 * img.height / 4; + int v_size = img.row_stride2 * img.height/ 4; - ns_pb::plane_sptr y_plane_data(new ns_pb::plane_sptr::element_type[y_size], [](uint8_t* ptr) { /* DO NOTHING */ }); - std::memcpy(y_plane_data.get(), input_image_data0, y_size); + ns_pb::plane_sptr y_plane_data(new ns_pb::plane_sptr::element_type[y_size], [](uint8_t* ptr) { delete[] ptr;}); + std::memcpy(y_plane_data.get(), img.input_image_data0, y_size); - ns_pb::plane_sptr u_plane_data(new ns_pb::plane_sptr::element_type[u_size], [](uint8_t* ptr) { /* DO NOTHING */ }); - ns_pb::plane_sptr v_plane_data(new ns_pb::plane_sptr::element_type[v_size], [](uint8_t* ptr) { /* DO NOTHING */ }); + ns_pb::plane_sptr u_plane_data(new ns_pb::plane_sptr::element_type[u_size], [](uint8_t* ptr) { delete[] ptr; }); + ns_pb::plane_sptr v_plane_data(new ns_pb::plane_sptr::element_type[v_size], [](uint8_t* ptr) { delete[] ptr; }); auto ptr_u = u_plane_data.get(); auto ptr_v = v_plane_data.get(); - for (unsigned row = 0; row < image_info.height * image_info.row_stride1 / 2; row += 2) { - *ptr_u++ = input_image_data1[row]; - *ptr_v++ = input_image_data1[row + 1]; + for (unsigned row = 0; row < img.height * img.row_stride1 / 2; row += 2) { + *ptr_u++ = img.input_image_data1[row]; + *ptr_v++ = img.input_image_data1[row + 1]; } - ns_pb::plane_data y_plane{std::move(y_plane_data), static_cast(y_size), image_info.row_stride0}; - ns_pb::plane_data u_plane{std::move(u_plane_data), static_cast(u_size), image_info.row_stride1 / 2}; - ns_pb::plane_data v_plane{std::move(v_plane_data), static_cast(v_size), image_info.row_stride2 / 2}; + ns_pb::plane_data y_plane{std::move(y_plane_data), static_cast(y_size), img.row_stride0}; + ns_pb::plane_data u_plane{std::move(u_plane_data), static_cast(u_size), img.row_stride1 / 2}; + ns_pb::plane_data v_plane{std::move(v_plane_data), static_cast(v_size), img.row_stride2 / 2}; std::vector planes{std::move(y_plane), std::move(u_plane), std::move(v_plane)}; return planes; } @@ -137,16 +134,12 @@ namespace return {}; } - pixel_buffer_sptr create_pixel_buffer(JNIEnv* env, jobject jimageY, jobject jimageU, jobject jimageV, - const image_info& image_info, bnb::oep::interfaces::image_format image_format) + pixel_buffer_sptr create_pixel_buffer(JNIEnv* env, jobject jimage, bnb::oep::interfaces::image_format image_format) { - uint8_t* input_image_data0 = static_cast(env->GetDirectBufferAddress(jimageY)); - uint8_t* input_image_data1 = static_cast(env->GetDirectBufferAddress(jimageU)); - uint8_t* input_image_data2 = static_cast(env->GetDirectBufferAddress(jimageV)); - auto width = static_cast(image_info.width); - auto height = static_cast(image_info.height); - auto planes = create_planes_from_format(input_image_data0, input_image_data1, input_image_data2, image_format, image_info); - return bnb::oep::interfaces::pixel_buffer::create(planes, image_format, width, height, [](auto* pb) { delete pb; }); + image img = get_image(env, jimage); + + auto planes = create_planes_from_format(img, image_format); + return bnb::oep::interfaces::pixel_buffer::create(planes, image_format, img.width, img.height, [](auto* pb) { delete pb; }); } struct banuba_sdk_manager @@ -182,8 +175,9 @@ namespace return ret; } - bnb::oep::interfaces::rotation java_rotation_to_oep_rotation(int rotation) + bnb::oep::interfaces::rotation java_rotation_to_oep_rotation(jint orientation) { + int rotation = static_cast(orientation); switch (rotation) { case 0: return bnb::oep::interfaces::rotation::deg0; @@ -207,6 +201,107 @@ namespace } return oep; } + + jobject get_image_from_pixel_buffer(pixel_buffer_sptr image, + bnb::oep::interfaces::image_format image_format, + JNIEnv* env, jclass image_class) { + switch(image_format) { + case bnb::oep::interfaces::image_format::bpc8_rgba: { + auto size = image->get_width() * image->get_height() * image->get_bytes_per_pixel(); + jbyteArray byte_array0 = env->NewByteArray(size); + env->SetByteArrayRegion(byte_array0, 0, size, reinterpret_cast((uint8_t*) image->get_base_sptr().get())); + if(byte_array0 == nullptr) { + print_message("GetEnv: error in getting data"); + return nullptr; + } + auto image_constructor_id = env->GetMethodID(image_class, "", "([BII)V"); + return env->NewObject( + image_class, image_constructor_id, byte_array0, + image->get_width(), image->get_height()); + break; + } + case bnb::oep::interfaces::image_format::nv12_bt601_full: { + auto size0 = image->get_width() * image->get_height() * image->get_bytes_per_pixel(); + auto size1 = image->get_width() * image->get_height() * image->get_bytes_per_pixel() / 2; + void* buf0 = reinterpret_cast((void*) image->get_base_sptr_of_plane(0).get()); + void* buf1 = reinterpret_cast((void*) image->get_base_sptr_of_plane(1).get()); + + jbyteArray byte_array0 = env->NewByteArray(size0); + jbyteArray byte_array1 = env->NewByteArray(size1); + env->SetByteArrayRegion(byte_array0, 0, size0, reinterpret_cast(buf0)); + env->SetByteArrayRegion(byte_array1, 0, size1, reinterpret_cast(buf1)); + if(byte_array0 == nullptr || byte_array1 == nullptr) { + print_message("GetEnv: error in getting data"); + return nullptr; + } + auto image_constructor_id = env->GetMethodID(image_class, "", "([B[BII)V"); + return env->NewObject( + image_class, image_constructor_id, byte_array0, byte_array1, + image->get_width(), image->get_height()); + break; + } + case bnb::oep::interfaces::image_format::i420_bt601_full: { + auto size0 = image->get_width_of_plane(0) * image->get_height() * image->get_bytes_per_pixel_of_plane(0); + auto size1 = image->get_width_of_plane(1) * image->get_height() * image->get_bytes_per_pixel_of_plane(1); + auto size2 = image->get_width_of_plane(2) * image->get_height() * image->get_bytes_per_pixel_of_plane(2); + void* buf0 = reinterpret_cast((void*) image->get_base_sptr_of_plane(0).get()); + void* buf1 = reinterpret_cast((void*) image->get_base_sptr_of_plane(1).get()); + void* buf2 = reinterpret_cast((void*) image->get_base_sptr_of_plane(2).get()); + + jbyteArray byte_array0 = env->NewByteArray(size0); + jbyteArray byte_array1 = env->NewByteArray(size1); + jbyteArray byte_array2 = env->NewByteArray(size2); + env->SetByteArrayRegion(byte_array0, 0, size0, reinterpret_cast(buf0)); + int u_width = image->get_width_of_plane(1); + int u_stride = image->get_bytes_per_row_of_plane(1); + + for(int i = 0; i < image->get_height_of_plane(1); ++i) { + env->SetByteArrayRegion(byte_array1, u_width * i, u_width, reinterpret_cast(buf1) + u_stride * i); + } + + int v_width = image->get_width_of_plane(2); + int v_stride = image->get_bytes_per_row_of_plane(2); + + for(int i = 0; i < image->get_height_of_plane(2); ++i) { + env->SetByteArrayRegion(byte_array2, v_width * i, v_width, reinterpret_cast(buf2) + v_stride * i); + } + if(byte_array0 == nullptr && byte_array1 == nullptr && byte_array2 == nullptr) { + print_message("GetEnv: error in getting data"); + return nullptr; + } + auto image_constructor_id = env->GetMethodID(image_class, "", "([B[B[BII)V"); + return env->NewObject( + image_class, image_constructor_id, byte_array0, byte_array1, byte_array2, + image->get_width(), image->get_height()); + break; + } + default: + break; + } + return nullptr; + } + + void draw_image_from_pixel_buffer(pixel_buffer_sptr pixel_buffer, + bnb::oep::interfaces::image_format image_format, + JNIEnv* env, + jobject this_ref, jclass image_class_ref) { + + jobject image = get_image_from_pixel_buffer(std::move(pixel_buffer), image_format, env, image_class_ref); + + if(image == nullptr) { + print_message("GetEnv: unsupported output image format"); + return; + } + + jclass jcallback_class = env->GetObjectClass(this_ref); + jmethodID jcallback_method = env->GetMethodID(jcallback_class, + "onDataReady", "(Lcom/banuba/quickstart_c_api/OffscreenEffectPlayerImage;)V"); + env->CallVoidMethod(this_ref, jcallback_method, image); + + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); + } + } } /* namespace */ extern "C" @@ -256,27 +351,33 @@ extern "C" /* OffscreenEffectPlayer::externalProcessImageAsync - java interface */ JNIEXPORT void JNICALL Java_com_banuba_quickstart_1c_1api_OffscreenEffectPlayer_externalProcessImageAsync( JNIEnv* env, jobject thiz, jlong jsdk, - jobject jimageY, jobject jimageU, jobject jimageV, - jobject jimage_info) + jobject jimage, + jint jinput_orientation, + jboolean jis_required_mirroring, + jint joutput_orientation, + jint jimage_format) { auto oep = get_offscreen_effect_player_from_jlong(jsdk); if (oep == nullptr) { return; } - auto image_info = get_image_info(env, jimage_info); - auto image_format = get_image_format(image_info.image_format); - auto pb_image = create_pixel_buffer(env, jimageY, jimageU, jimageV, image_info, image_format); + auto image_format = get_image_format(jimage_format); + auto pb_image = create_pixel_buffer(env, jimage, image_format); + + jclass image_class = env->FindClass("com/banuba/quickstart_c_api/OffscreenEffectPlayerImage"); + auto image_class_ref = (jclass) env->NewGlobalRef(image_class); + + jobject this_ref = env->NewGlobalRef(thiz); JavaVM* jvm; env->GetJavaVM(&jvm); - jobject this_ref = env->NewGlobalRef(thiz); // Callback for received pixel buffer from the offscreen effect player - auto get_pixel_buffer_callback = [this_ref, jvm, image_format](image_processing_result_sptr result) { + auto get_pixel_buffer_callback = [this_ref, jvm, image_format, image_class_ref](image_processing_result_sptr result) { if (result != nullptr) { // Callback for update data in render thread - auto get_image_callback = [this_ref, jvm, image_format](pixel_buffer_sptr image) { + auto get_image_callback = [this_ref, jvm, image_format, image_class_ref](pixel_buffer_sptr pb_image) { JNIEnv* env = nullptr; // double check it's all ok @@ -284,100 +385,39 @@ extern "C" if (getEnvStat == JNI_EDETACHED) { if (jvm->AttachCurrentThread((JNIEnv**) &env, nullptr) != JNI_OK) { print_message("GetEnv: Failed to attach"); + env->DeleteGlobalRef(image_class_ref); + env->DeleteGlobalRef(this_ref); return; } } else if (getEnvStat == JNI_EVERSION) { print_message("GetEnv: version not supported"); - return; - } - - if (image == nullptr) { - print_message("GetEnv: image is null"); + env->DeleteGlobalRef(image_class_ref); env->DeleteGlobalRef(this_ref); - jvm->DetachCurrentThread(); return; } - jbyteArray byte_array0 = nullptr; - jbyteArray byte_array1 = nullptr; - jbyteArray byte_array2 = nullptr; - switch(image_format) { - case bnb::oep::interfaces::image_format::bpc8_rgba: { - auto size = image->get_width() * image->get_height() * image->get_bytes_per_pixel(); - byte_array0 = env->NewByteArray(size); - env->SetByteArrayRegion(byte_array0, 0, size, reinterpret_cast((uint8_t*) image->get_base_sptr().get())); - break; - } - case bnb::oep::interfaces::image_format::nv12_bt601_full: { - auto size0 = image->get_width() * image->get_height() * image->get_bytes_per_pixel(); - auto size1 = image->get_width() * image->get_height() * image->get_bytes_per_pixel() / 2; - void* buf0 = reinterpret_cast((void*) image->get_base_sptr_of_plane(0).get()); - void* buf1 = reinterpret_cast((void*) image->get_base_sptr_of_plane(1).get()); - - byte_array0 = env->NewByteArray(size0); - byte_array1 = env->NewByteArray(size1); - byte_array2 = env->NewByteArray(0); - env->SetByteArrayRegion(byte_array0, 0, size0, reinterpret_cast(buf0)); - env->SetByteArrayRegion(byte_array1, 0, size1, reinterpret_cast(buf1)); - break; - } - case bnb::oep::interfaces::image_format::i420_bt601_full: { - auto size0 = image->get_width_of_plane(0) * image->get_height() * image->get_bytes_per_pixel_of_plane(0); - auto size1 = image->get_width_of_plane(1) * image->get_height() * image->get_bytes_per_pixel_of_plane(1); - auto size2 = image->get_width_of_plane(2) * image->get_height() * image->get_bytes_per_pixel_of_plane(2); - void* buf0 = reinterpret_cast((void*) image->get_base_sptr_of_plane(0).get()); - void* buf1 = reinterpret_cast((void*) image->get_base_sptr_of_plane(1).get()); - void* buf2 = reinterpret_cast((void*) image->get_base_sptr_of_plane(2).get()); - - byte_array0 = env->NewByteArray(size0); - byte_array1 = env->NewByteArray(size1); - byte_array2 = env->NewByteArray(size2); - env->SetByteArrayRegion(byte_array0, 0, size0, reinterpret_cast(buf0)); - int u_width = image->get_width_of_plane(1); - int u_stride = image->get_bytes_per_row_of_plane(1); - - for(int i = 0; i < image->get_height_of_plane(1); ++i) { - env->SetByteArrayRegion(byte_array1, u_width * i, u_width, reinterpret_cast(buf1) + u_stride * i); - } - - int v_width = image->get_width_of_plane(2); - int v_stride = image->get_bytes_per_row_of_plane(2); - - for(int i = 0; i < image->get_height_of_plane(2); ++i) { - env->SetByteArrayRegion(byte_array2, v_width * i, v_width, reinterpret_cast(buf2) + v_stride * i); - } - break; - } - default: - break; - } - - if(byte_array0 == nullptr && byte_array1 == nullptr && byte_array2 == nullptr) { - print_message("GetEnv: unsupported output image format"); + if (pb_image == nullptr) { + print_message("GetEnv: image is null"); + env->DeleteGlobalRef(image_class_ref); env->DeleteGlobalRef(this_ref); jvm->DetachCurrentThread(); return; } - jclass jcallback_class = env->GetObjectClass(this_ref); - jmethodID jcallback_method = env->GetMethodID(jcallback_class, "onDataReady", "([B[B[BII)V"); - - // call callback - env->CallVoidMethod(this_ref, jcallback_method, byte_array0, byte_array1, byte_array2, image->get_width(), image->get_height()); - if (env->ExceptionCheck()) { - env->ExceptionDescribe(); - } + draw_image_from_pixel_buffer(std::move(pb_image), image_format, env, this_ref, + image_class_ref); + env->DeleteGlobalRef(image_class_ref); env->DeleteGlobalRef(this_ref); - jvm->DetachCurrentThread(); }; // Get image from effect_player and return it in the callback result->get_image(image_format, get_image_callback); } }; - auto in_rotation = java_rotation_to_oep_rotation(image_info.input_orientation); - auto out_rotation = java_rotation_to_oep_rotation(image_info.output_orientation); - oep->process_image_async(pb_image, in_rotation, image_info.require_mirroring, get_pixel_buffer_callback, out_rotation); + auto in_rotation = java_rotation_to_oep_rotation(jinput_orientation); + auto out_rotation = java_rotation_to_oep_rotation(joutput_orientation); + auto require_mirroring = static_cast(jis_required_mirroring); + oep->process_image_async(pb_image, in_rotation, require_mirroring, get_pixel_buffer_callback, out_rotation); } /* OffscreenEffectPlayer::externalSurfaceChanged - java interface */ diff --git a/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java b/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java index b9674bb..0dc173b 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java @@ -28,8 +28,6 @@ import com.google.common.util.concurrent.ListenableFuture; import java.io.File; -import java.util.Arrays; -import java.util.List; public class MainActivity extends AppCompatActivity { private static int CAMERA_PERMISSION_REQUEST = 12345; @@ -40,7 +38,6 @@ public class MainActivity extends AppCompatActivity { private OffscreenEffectPlayer oep = null; private GLSurfaceView glView = null; private GLRenderer renderer = null; - private OffscreenEffectPlayerImage mImage = null; // Changing mImageOutputFormat will cause the format's changing (input and output image of OEP) private ImageFormat mImageFormat = ImageFormat.NV12; @@ -61,11 +58,10 @@ void createRenderer() { void createOEP() { oep = new OffscreenEffectPlayer(size.getWidth(), size.getHeight()); - oep.loadEffect("effects/") + oep.loadEffect("effects/"); oep.setDataReadyCallback( - (image0,image1, image2, width, height) -> { - List planes = Arrays.asList(image0, image1, image2); - renderer.drawImage(planes, width, height); + (offscreenEffectPlayerImage) -> { + renderer.drawImage(offscreenEffectPlayerImage); glView.requestRender(); }); } @@ -94,7 +90,6 @@ protected void onCreate(@Nullable Bundle savedInstanceState) { /* Create offscreen effect player */ createOEP(); - mImage = new OffscreenEffectPlayerImage(); requestCameraPermissionAndStart(); } @@ -154,8 +149,11 @@ private void startCamera() { imageAnalysis.setAnalyzer( ContextCompat.getMainExecutor(MainActivity.this), proxy -> { - updateImage(proxy); - oep.processImageAsync(mImage); + int rotation = getRotation(MainActivity.this); + int imageFormat = mImageFormat.ordinal(); + OffscreenEffectPlayerImage image = new OffscreenEffectPlayerImage(proxy); + oep.processImageAsync(image, getInputOrientation(rotation), + false, getOutputOrientation(rotation), imageFormat); proxy.close(); }); cameraProvider.bindToLifecycle(MainActivity.this, cameraSelector, imageAnalysis); @@ -191,34 +189,6 @@ private int getOutputOrientation(int rotation) { } return 0; } - - private void updateImage(ImageProxy imageProxy) { - int rotation = getRotation(MainActivity.this); - - mImage.mImageInfo.width = imageProxy.getWidth(); - mImage.mImageInfo.height = imageProxy.getHeight(); - mImage.mImageInfo.inputOrientation = getInputOrientation(rotation); - mImage.mImageInfo.outputOrientation = getOutputOrientation(rotation); - mImage.mImageInfo.pixelFormat = imageProxy.getImage().getFormat(); - mImage.mImageInfo.requireMirroring = false; - mImage.mImageInfo.imageFormat = mImageFormat.ordinal(); - - mImage.mImageInfo.rowStride0 = imageProxy.getPlanes()[0].getRowStride(); - mImage.mImageZero = imageProxy.getPlanes()[0].getBuffer(); - mImage.mImageInfo.pixelStride0 = imageProxy.getPlanes()[0].getPixelStride(); - - int planesNumber = imageProxy.getPlanes().length; - if (planesNumber > 1) { - mImage.mImageInfo.rowStride1 = imageProxy.getPlanes()[1].getRowStride(); - mImage.mImageFirst = imageProxy.getPlanes()[1].getBuffer(); - mImage.mImageInfo.pixelStride1 = imageProxy.getPlanes()[1].getPixelStride(); - if (planesNumber > 2) { - mImage.mImageInfo.rowStride2 = imageProxy.getPlanes()[2].getRowStride(); - mImage.mImageSecond = imageProxy.getPlanes()[2].getBuffer(); - mImage.mImageInfo.pixelStride2 = imageProxy.getPlanes()[2].getPixelStride(); - } - } - } } enum ImageFormat { diff --git a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java index 5da5749..eff3d65 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java @@ -1,7 +1,5 @@ package com.banuba.quickstart_c_api; -import java.nio.ByteBuffer; - class OffscreenEffectPlayer { private long mOep = 0; private DataReadyCallback mDataReadyCallback = null; @@ -41,9 +39,14 @@ public void destroy() { } } - /* image must be NV12 format */ - public void processImageAsync(OffscreenEffectPlayerImage image) { - externalProcessImageAsync(mOep, image.mImageZero, image.mImageFirst, image.mImageSecond, image.mImageInfo); + /* image must be NV12 or i420 format */ + public void processImageAsync(OffscreenEffectPlayerImage image, + int inputOrientation, + boolean isRequiredMirroring, + int outputOrientation, + int imageFormat) { + externalProcessImageAsync(mOep, image, inputOrientation, isRequiredMirroring, + outputOrientation, imageFormat); } public void surfaceChanged(int width, int height) { @@ -79,16 +82,16 @@ public void evalJs(String script) { } public interface DataReadyCallback { - void onDataReady(byte[] image0, byte[] image1, byte[] image2, int width, int height); + void onDataReady(OffscreenEffectPlayerImage image); } public void setDataReadyCallback(DataReadyCallback callback) { mDataReadyCallback = callback; } - private void onDataReady(byte[] image0, byte[] image1, byte[] image2, int width, int height) { + private void onDataReady(OffscreenEffectPlayerImage image) { if (mDataReadyCallback != null) { - mDataReadyCallback.onDataReady(image0, image1, image2, width, height); + mDataReadyCallback.onDataReady(image); } } @@ -97,7 +100,11 @@ private void onDataReady(byte[] image0, byte[] image1, byte[] image2, int width, private static native void externalDeinit(); private native long externalCreate(int width, int height); private native void externalDestroy(long oep); - private native void externalProcessImageAsync(long oep, ByteBuffer imageY, ByteBuffer imageU, ByteBuffer imageV, ImageInfo info); + private native void externalProcessImageAsync(long oep, OffscreenEffectPlayerImage image, + int inputOrientation, + boolean isRequiredMirroring, + int outputOrientation, + int imageFormat); private native void externalSurfaceChanged(long oep, int width, int height); private native void externalLoadEffect(long oep, String effectPath); private native void externalUnloadEffect(long oep); @@ -111,3 +118,4 @@ private void onDataReady(byte[] image0, byte[] image1, byte[] image2, int width, System.loadLibrary("native-lib"); } } + diff --git a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java index 304bce2..4718ead 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java @@ -1,45 +1,125 @@ package com.banuba.quickstart_c_api; +import androidx.camera.core.ImageProxy; + import java.nio.ByteBuffer; public class OffscreenEffectPlayerImage { - OffscreenEffectPlayerImage() { - mImageInfo = new ImageInfo(); + + OffscreenEffectPlayerImage(ImageProxy imageProxy) { + mWidth = imageProxy.getWidth(); + mHeight = imageProxy.getHeight(); + mPixelFormat = imageProxy.getImage().getFormat(); + + ImageProxy.PlaneProxy planeProxy = imageProxy.getPlanes()[0]; + mRowStride0 = planeProxy.getRowStride(); + mPlane0 = planeProxy.getBuffer(); + mPixelStride0 = planeProxy.getPixelStride(); + int planesNumber = imageProxy.getPlanes().length; + if (planesNumber > 1) { + planeProxy = imageProxy.getPlanes()[1]; + mRowStride1 = planeProxy.getRowStride(); + mPlane1 = planeProxy.getBuffer(); + mPixelStride1 = planeProxy.getPixelStride(); + if (planesNumber > 2) { + planeProxy = imageProxy.getPlanes()[2]; + mRowStride2 = planeProxy.getRowStride(); + mPlane2 = planeProxy.getBuffer(); + mPixelStride2 = planeProxy.getPixelStride(); + } else { + mRowStride2 = 0; + mPlane2 = null; + mPixelStride2 = 0; + } + } else { + mRowStride1 = 0; + mPlane1 = null; + mPixelStride1 = 0; + mRowStride2 = 0; + mPlane2 = null; + mPixelStride2 = 0; + } } - public ImageInfo mImageInfo; - public ByteBuffer mImageZero = null; - public ByteBuffer mImageFirst = null; - public ByteBuffer mImageSecond = null; -} + OffscreenEffectPlayerImage(byte[] rgbPlane, int width, int height) { + assert rgbPlane != null; + mPlane0 = ByteBuffer.wrap(rgbPlane); + mPlane1 = null; + mPlane2 = null; + mWidth = width; + mHeight = height; + + mRowStride0 = 0; + mRowStride1 = 0; + mRowStride2 = 0; + mPixelStride0 = 0; + mPixelStride1 = 0; + mPixelStride2 = 0; + mPixelFormat = 0; + } + + OffscreenEffectPlayerImage(byte[] yPlane, byte[] uvPlane, int width, int height) { + assert yPlane != null && uvPlane != null; + mPlane0 = ByteBuffer.wrap(yPlane); + mPlane1 = ByteBuffer.wrap(uvPlane); + mPlane2 = null; + mWidth = width; + mHeight = height; + + mRowStride0 = 0; + mRowStride1 = 0; + mRowStride2 = 0; + mPixelStride0 = 0; + mPixelStride1 = 0; + mPixelStride2 = 0; + mPixelFormat = 0; + } -class ImageInfo { - ImageInfo() { - width = 0; - height = 0; - inputOrientation = 0; - outputOrientation = 0; - requireMirroring = false; - rowStride0 = 0; - rowStride1 = 0; - rowStride2 = 0; - pixelStride0 = 0; - pixelStride1 = 0; - pixelStride2 = 0; - pixelFormat = 0; - imageFormat = 0; + OffscreenEffectPlayerImage(byte[] yPlane, byte[] uPlane, byte[] vPlane, int width, int height) { + assert yPlane != null && uPlane != null && vPlane != null; + mPlane0 = ByteBuffer.wrap(yPlane); + mPlane1 = ByteBuffer.wrap(uPlane); + mPlane2 = ByteBuffer.wrap(vPlane); + mWidth = width; + mHeight = height; + + mRowStride0 = 0; + mRowStride1 = 0; + mRowStride2 = 0; + mPixelStride0 = 0; + mPixelStride1 = 0; + mPixelStride2 = 0; + mPixelFormat = 0; + } + + public ByteBuffer getPlane(int planeNumber) { + switch (planeNumber) { + case 1: + return mPlane1; + case 2: + return mPlane2; + default: + return mPlane0; + } } - public int width; - public int height; - public int inputOrientation; - public int outputOrientation; - public int rowStride0; - public int rowStride1; - public int rowStride2; - public int pixelStride0; - public int pixelStride1; - public int pixelStride2; - public int pixelFormat; - public int imageFormat; - public boolean requireMirroring; -} \ No newline at end of file + + public int getWidth() { return mWidth; } + + public int getHeight() { + return mHeight; + } + + final private ByteBuffer mPlane0; + final private ByteBuffer mPlane1; + final private ByteBuffer mPlane2; + + final private int mWidth; + final private int mHeight; + final private int mRowStride0; + final private int mRowStride1; + final private int mRowStride2; + final private int mPixelStride0; + final private int mPixelStride1; + final private int mPixelStride2; + final private int mPixelFormat; +} diff --git a/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java b/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java index 691ca1b..dd8c7fb 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java @@ -12,6 +12,8 @@ import javax.microedition.khronos.egl.EGLConfig; import javax.microedition.khronos.opengles.GL10; +import com.banuba.quickstart_c_api.OffscreenEffectPlayerImage; + public class GLRenderer implements GLSurfaceView.Renderer { /* shaders */ @@ -178,10 +180,16 @@ public void scaleMatrix() { mMat4[5] = yScale; } - public void drawImage(List imageDataPlanes, int width, int height) { - mImageDataPlanes = imageDataPlanes; - mImageWidth = width; - mImageHeight = height; + public void drawImage(OffscreenEffectPlayerImage image) { + mImageDataPlanes.clear(); + for(int i = 0; i < mTexturesCount; ++i) { + ByteBuffer plane = image.getPlane(i); + if(plane != null) { + mImageDataPlanes.add(plane.array()); + } + } + mImageWidth = image.getWidth(); + mImageHeight = image.getHeight(); } /* destructor */