From 1544113d77f6baaa29465db0bb6213bcace51da0 Mon Sep 17 00:00:00 2001 From: romanbylina Date: Thu, 4 Aug 2022 18:26:49 +0300 Subject: [PATCH 1/8] fix memory leak --- app/src/main/cpp/native-lib.cpp | 10 +++++----- .../quickstart_c_api/OffscreenEffectPlayerImage.java | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/src/main/cpp/native-lib.cpp b/app/src/main/cpp/native-lib.cpp index 7ed866f..0f4d844 100644 --- a/app/src/main/cpp/native-lib.cpp +++ b/app/src/main/cpp/native-lib.cpp @@ -96,10 +96,10 @@ namespace int y_size = image_info.row_stride0 * image_info.height; int uv_size = image_info.row_stride1 * image_info.height / 2; - ns_pb::plane_sptr y_plane_data(new ns_pb::plane_sptr::element_type[y_size], [](uint8_t* ptr) { /* DO NOTHING */ }); + ns_pb::plane_sptr y_plane_data(new ns_pb::plane_sptr::element_type[y_size], [](uint8_t* ptr) { delete[] ptr; }); std::memcpy(y_plane_data.get(), input_image_data0, y_size); - ns_pb::plane_sptr uv_plane_data(new ns_pb::plane_sptr::element_type[uv_size], [](uint8_t* ptr) { /* DO NOTHING */ }); + ns_pb::plane_sptr uv_plane_data(new ns_pb::plane_sptr::element_type[uv_size], [](uint8_t* ptr) { delete[] ptr; }); std::memcpy(uv_plane_data.get(), input_image_data1, uv_size); ns_pb::plane_data y_plane{std::move(y_plane_data), static_cast(y_size), image_info.row_stride0}; @@ -113,11 +113,11 @@ namespace int u_size = image_info.row_stride1 * image_info.height / 4; int v_size = image_info.row_stride2 * image_info.height/ 4; - ns_pb::plane_sptr y_plane_data(new ns_pb::plane_sptr::element_type[y_size], [](uint8_t* ptr) { /* DO NOTHING */ }); + ns_pb::plane_sptr y_plane_data(new ns_pb::plane_sptr::element_type[y_size], [](uint8_t* ptr) { delete[] ptr;}); std::memcpy(y_plane_data.get(), input_image_data0, y_size); - ns_pb::plane_sptr u_plane_data(new ns_pb::plane_sptr::element_type[u_size], [](uint8_t* ptr) { /* DO NOTHING */ }); - ns_pb::plane_sptr v_plane_data(new ns_pb::plane_sptr::element_type[v_size], [](uint8_t* ptr) { /* DO NOTHING */ }); + ns_pb::plane_sptr u_plane_data(new ns_pb::plane_sptr::element_type[u_size], [](uint8_t* ptr) { delete[] ptr; }); + ns_pb::plane_sptr v_plane_data(new ns_pb::plane_sptr::element_type[v_size], [](uint8_t* ptr) { delete[] ptr; }); auto ptr_u = u_plane_data.get(); auto ptr_v = v_plane_data.get(); diff --git a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java index 304bce2..f592311 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java @@ -42,4 +42,4 @@ class ImageInfo { public int pixelFormat; public int imageFormat; public boolean requireMirroring; -} \ No newline at end of file +} From 5590cbaae3ff3aab69c7c44143b30b544c444097 Mon Sep 17 00:00:00 2001 From: romanbylina Date: Fri, 5 Aug 2022 13:59:06 +0300 Subject: [PATCH 2/8] can disable OEP --- app/src/main/cpp/native-lib.cpp | 84 ++++++++++++++++++- .../banuba/quickstart_c_api/MainActivity.java | 5 +- .../OffscreenEffectPlayer.java | 6 +- 3 files changed, 89 insertions(+), 6 deletions(-) diff --git a/app/src/main/cpp/native-lib.cpp b/app/src/main/cpp/native-lib.cpp index 0f4d844..f8e8457 100644 --- a/app/src/main/cpp/native-lib.cpp +++ b/app/src/main/cpp/native-lib.cpp @@ -253,11 +253,85 @@ extern "C" delete sdk; } + void draw_image_without_processing(pixel_buffer_sptr image, + bnb::oep::interfaces::image_format image_format, + JNIEnv* env, + jobject this_ref) { + jbyteArray byte_array0 = nullptr; + jbyteArray byte_array1 = nullptr; + jbyteArray byte_array2 = nullptr; + switch(image_format) { + case bnb::oep::interfaces::image_format::bpc8_rgba: { + auto size = image->get_width() * image->get_height() * image->get_bytes_per_pixel(); + byte_array0 = env->NewByteArray(size); + env->SetByteArrayRegion(byte_array0, 0, size, reinterpret_cast((uint8_t*) image->get_base_sptr().get())); + break; + } + case bnb::oep::interfaces::image_format::nv12_bt601_full: { + auto size0 = image->get_width() * image->get_height() * image->get_bytes_per_pixel(); + auto size1 = image->get_width() * image->get_height() * image->get_bytes_per_pixel() / 2; + void* buf0 = reinterpret_cast((void*) image->get_base_sptr_of_plane(0).get()); + void* buf1 = reinterpret_cast((void*) image->get_base_sptr_of_plane(1).get()); + + byte_array0 = env->NewByteArray(size0); + byte_array1 = env->NewByteArray(size1); + byte_array2 = env->NewByteArray(0); + env->SetByteArrayRegion(byte_array0, 0, size0, reinterpret_cast(buf0)); + env->SetByteArrayRegion(byte_array1, 0, size1, reinterpret_cast(buf1)); + break; + } + case bnb::oep::interfaces::image_format::i420_bt601_full: { + auto size0 = image->get_width_of_plane(0) * image->get_height() * image->get_bytes_per_pixel_of_plane(0); + auto size1 = image->get_width_of_plane(1) * image->get_height() * image->get_bytes_per_pixel_of_plane(1); + auto size2 = image->get_width_of_plane(2) * image->get_height() * image->get_bytes_per_pixel_of_plane(2); + void* buf0 = reinterpret_cast((void*) image->get_base_sptr_of_plane(0).get()); + void* buf1 = reinterpret_cast((void*) image->get_base_sptr_of_plane(1).get()); + void* buf2 = reinterpret_cast((void*) image->get_base_sptr_of_plane(2).get()); + + byte_array0 = env->NewByteArray(size0); + byte_array1 = env->NewByteArray(size1); + byte_array2 = env->NewByteArray(size2); + env->SetByteArrayRegion(byte_array0, 0, size0, reinterpret_cast(buf0)); + int u_width = image->get_width_of_plane(1); + int u_stride = image->get_bytes_per_row_of_plane(1); + + for(int i = 0; i < image->get_height_of_plane(1); ++i) { + env->SetByteArrayRegion(byte_array1, u_width * i, u_width, reinterpret_cast(buf1) + u_stride * i); + } + + int v_width = image->get_width_of_plane(2); + int v_stride = image->get_bytes_per_row_of_plane(2); + + for(int i = 0; i < image->get_height_of_plane(2); ++i) { + env->SetByteArrayRegion(byte_array2, v_width * i, v_width, reinterpret_cast(buf2) + v_stride * i); + } + break; + } + default: + break; + } + + if(byte_array0 == nullptr && byte_array1 == nullptr && byte_array2 == nullptr) { + print_message("GetEnv: unsupported output image format"); + return; + } + + jclass jcallback_class = env->GetObjectClass(this_ref); + jmethodID jcallback_method = env->GetMethodID(jcallback_class, "onDataReady", "([B[B[BII)V"); + + // call callback + env->CallVoidMethod(this_ref, jcallback_method, byte_array0, byte_array1, byte_array2, image->get_width(), image->get_height()); + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); + } + env->DeleteGlobalRef(this_ref); + } + /* OffscreenEffectPlayer::externalProcessImageAsync - java interface */ JNIEXPORT void JNICALL Java_com_banuba_quickstart_1c_1api_OffscreenEffectPlayer_externalProcessImageAsync( JNIEnv* env, jobject thiz, jlong jsdk, jobject jimageY, jobject jimageU, jobject jimageV, - jobject jimage_info) + jobject jimage_info, jboolean jis_process_image) { auto oep = get_offscreen_effect_player_from_jlong(jsdk); if (oep == nullptr) { @@ -267,10 +341,16 @@ extern "C" auto image_info = get_image_info(env, jimage_info); auto image_format = get_image_format(image_info.image_format); auto pb_image = create_pixel_buffer(env, jimageY, jimageU, jimageV, image_info, image_format); + bool is_process_image = static_cast(jis_process_image); + + jobject this_ref = env->NewGlobalRef(thiz); + if(!is_process_image) { + draw_image_without_processing(pb_image, image_format, env, this_ref); + return; + } JavaVM* jvm; env->GetJavaVM(&jvm); - jobject this_ref = env->NewGlobalRef(thiz); // Callback for received pixel buffer from the offscreen effect player auto get_pixel_buffer_callback = [this_ref, jvm, image_format](image_processing_result_sptr result) { diff --git a/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java b/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java index b9674bb..b189467 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java @@ -45,6 +45,9 @@ public class MainActivity extends AppCompatActivity { // Changing mImageOutputFormat will cause the format's changing (input and output image of OEP) private ImageFormat mImageFormat = ImageFormat.NV12; + // OEP doesn't work if mIsOEPEnabled = false; + private boolean mIsOEPEnabled = true; + void createRenderer() { switch (mImageFormat) { case NV12: @@ -155,7 +158,7 @@ private void startCamera() { ContextCompat.getMainExecutor(MainActivity.this), proxy -> { updateImage(proxy); - oep.processImageAsync(mImage); + oep.processImageAsync(mImage, mIsOEPEnabled); proxy.close(); }); cameraProvider.bindToLifecycle(MainActivity.this, cameraSelector, imageAnalysis); diff --git a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java index 5da5749..9db41ca 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java @@ -42,8 +42,8 @@ public void destroy() { } /* image must be NV12 format */ - public void processImageAsync(OffscreenEffectPlayerImage image) { - externalProcessImageAsync(mOep, image.mImageZero, image.mImageFirst, image.mImageSecond, image.mImageInfo); + public void processImageAsync(OffscreenEffectPlayerImage image, boolean isProcessImage) { + externalProcessImageAsync(mOep, image.mImageZero, image.mImageFirst, image.mImageSecond, image.mImageInfo, isProcessImage); } public void surfaceChanged(int width, int height) { @@ -97,7 +97,7 @@ private void onDataReady(byte[] image0, byte[] image1, byte[] image2, int width, private static native void externalDeinit(); private native long externalCreate(int width, int height); private native void externalDestroy(long oep); - private native void externalProcessImageAsync(long oep, ByteBuffer imageY, ByteBuffer imageU, ByteBuffer imageV, ImageInfo info); + private native void externalProcessImageAsync(long oep, ByteBuffer imageY, ByteBuffer imageU, ByteBuffer imageV, ImageInfo info, boolean isProcessImage); private native void externalSurfaceChanged(long oep, int width, int height); private native void externalLoadEffect(long oep, String effectPath); private native void externalUnloadEffect(long oep); From 3c38bb845029f23422ecc6400e0a12f4871d9104 Mon Sep 17 00:00:00 2001 From: romanbylina Date: Wed, 17 Aug 2022 14:11:12 +0300 Subject: [PATCH 3/8] return Image object from JNI --- .gitignore | 2 + app/src/main/cpp/native-lib.cpp | 215 +++++++----------- .../com/banuba/quickstart_c_api/Image.java | 19 ++ .../banuba/quickstart_c_api/MainActivity.java | 10 +- .../OffscreenEffectPlayer.java | 9 +- .../rendering/GLRenderer.java | 11 +- 6 files changed, 125 insertions(+), 141 deletions(-) create mode 100644 app/src/main/java/com/banuba/quickstart_c_api/Image.java diff --git a/.gitignore b/.gitignore index 7e65384..db0eb68 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ local.properties .gradle app/.cxx app/src/main/assets/*/ +build/ +app/release/ diff --git a/app/src/main/cpp/native-lib.cpp b/app/src/main/cpp/native-lib.cpp index f8e8457..6feda23 100644 --- a/app/src/main/cpp/native-lib.cpp +++ b/app/src/main/cpp/native-lib.cpp @@ -207,56 +207,10 @@ namespace } return oep; } -} /* namespace */ - -extern "C" -{ - static utility_manager_holder_t* utility = nullptr; - /* OffscreenEffectPlayer::externalInit - java interface */ - JNIEXPORT void JNICALL Java_com_banuba_quickstart_1c_1api_OffscreenEffectPlayer_externalInit(JNIEnv* env, jclass clazz, jstring jpath_to_resources, jstring jtoken) - { - assert(utility == nullptr); - std::vector path{jstring_to_string(env, jpath_to_resources)}; - auto token = jstring_to_string(env, jtoken); - std::unique_ptr res_paths = std::make_unique(path.size() + 1); - std::transform(path.begin(), path.end(), res_paths.get(), [](const auto& s) { return s.c_str(); }); - res_paths.get()[path.size()] = nullptr; - utility = bnb_utility_manager_init(res_paths.get(), token.c_str(), nullptr); - } - - /* OffscreenEffectPlayer::externalDeinit - java interface */ - JNIEXPORT void JNICALL Java_com_banuba_quickstart_1c_1api_OffscreenEffectPlayer_externalDeinit(JNIEnv* env, jclass clazz) - { - assert(utility != nullptr); - bnb_utility_manager_release(utility, nullptr); - utility = nullptr; - } - - /* OffscreenEffectPlayer::externalCreate - java interface */ - JNIEXPORT jlong JNICALL Java_com_banuba_quickstart_1c_1api_OffscreenEffectPlayer_externalCreate(JNIEnv* env, jobject thiz, jint jwidth, jint jheight) - { - int32_t width = jwidth; - int32_t height = jheight; - auto oep = new banuba_sdk_manager(width, height); - return reinterpret_cast(oep); - } - - /* OffscreenEffectPlayer::externalDestroy - java interface */ - JNIEXPORT void JNICALL Java_com_banuba_quickstart_1c_1api_OffscreenEffectPlayer_externalDestroy(JNIEnv* env, jobject thiz, jlong jsdk) - { - auto sdk = reinterpret_cast(jsdk); - if (sdk == nullptr) { - print_message("error: OffscreenEffectPlayer::externalDestroy(): sdk == nullptr\n"); - return; - } - delete sdk; - } - - void draw_image_without_processing(pixel_buffer_sptr image, - bnb::oep::interfaces::image_format image_format, - JNIEnv* env, - jobject this_ref) { + jobject get_image_from_pixel_buffer(pixel_buffer_sptr image, + bnb::oep::interfaces::image_format image_format, + JNIEnv* env, jclass image_class) { jbyteArray byte_array0 = nullptr; jbyteArray byte_array1 = nullptr; jbyteArray byte_array2 = nullptr; @@ -312,19 +266,80 @@ extern "C" } if(byte_array0 == nullptr && byte_array1 == nullptr && byte_array2 == nullptr) { + print_message("GetEnv: unsupported output image format"); + return nullptr; + } + auto image_constructor_id = env->GetMethodID(image_class, "", "([B[B[BII)V"); + return env->NewObject( + image_class, image_constructor_id, byte_array0, byte_array1, byte_array2, + image->get_width(), image->get_height()); + } + + void draw_image_from_pixel_buffer(pixel_buffer_sptr pixel_buffer, + bnb::oep::interfaces::image_format image_format, + JNIEnv* env, + jobject this_ref, jclass image_class_ref) { + + jobject image = get_image_from_pixel_buffer(std::move(pixel_buffer), image_format, env, image_class_ref); + + if(image == nullptr) { print_message("GetEnv: unsupported output image format"); return; } jclass jcallback_class = env->GetObjectClass(this_ref); - jmethodID jcallback_method = env->GetMethodID(jcallback_class, "onDataReady", "([B[B[BII)V"); + jmethodID jcallback_method = env->GetMethodID(jcallback_class, + "onDataReady", "(Lcom/banuba/quickstart_c_api/Image;)V"); + env->CallVoidMethod(this_ref, jcallback_method, image); - // call callback - env->CallVoidMethod(this_ref, jcallback_method, byte_array0, byte_array1, byte_array2, image->get_width(), image->get_height()); if (env->ExceptionCheck()) { env->ExceptionDescribe(); } - env->DeleteGlobalRef(this_ref); + } +} /* namespace */ + +extern "C" +{ + static utility_manager_holder_t* utility = nullptr; + + /* OffscreenEffectPlayer::externalInit - java interface */ + JNIEXPORT void JNICALL Java_com_banuba_quickstart_1c_1api_OffscreenEffectPlayer_externalInit(JNIEnv* env, jclass clazz, jstring jpath_to_resources, jstring jtoken) + { + assert(utility == nullptr); + std::vector path{jstring_to_string(env, jpath_to_resources)}; + auto token = jstring_to_string(env, jtoken); + std::unique_ptr res_paths = std::make_unique(path.size() + 1); + std::transform(path.begin(), path.end(), res_paths.get(), [](const auto& s) { return s.c_str(); }); + res_paths.get()[path.size()] = nullptr; + utility = bnb_utility_manager_init(res_paths.get(), token.c_str(), nullptr); + } + + /* OffscreenEffectPlayer::externalDeinit - java interface */ + JNIEXPORT void JNICALL Java_com_banuba_quickstart_1c_1api_OffscreenEffectPlayer_externalDeinit(JNIEnv* env, jclass clazz) + { + assert(utility != nullptr); + bnb_utility_manager_release(utility, nullptr); + utility = nullptr; + } + + /* OffscreenEffectPlayer::externalCreate - java interface */ + JNIEXPORT jlong JNICALL Java_com_banuba_quickstart_1c_1api_OffscreenEffectPlayer_externalCreate(JNIEnv* env, jobject thiz, jint jwidth, jint jheight) + { + int32_t width = jwidth; + int32_t height = jheight; + auto oep = new banuba_sdk_manager(width, height); + return reinterpret_cast(oep); + } + + /* OffscreenEffectPlayer::externalDestroy - java interface */ + JNIEXPORT void JNICALL Java_com_banuba_quickstart_1c_1api_OffscreenEffectPlayer_externalDestroy(JNIEnv* env, jobject thiz, jlong jsdk) + { + auto sdk = reinterpret_cast(jsdk); + if (sdk == nullptr) { + print_message("error: OffscreenEffectPlayer::externalDestroy(): sdk == nullptr\n"); + return; + } + delete sdk; } /* OffscreenEffectPlayer::externalProcessImageAsync - java interface */ @@ -341,11 +356,17 @@ extern "C" auto image_info = get_image_info(env, jimage_info); auto image_format = get_image_format(image_info.image_format); auto pb_image = create_pixel_buffer(env, jimageY, jimageU, jimageV, image_info, image_format); - bool is_process_image = static_cast(jis_process_image); + + jclass image_class = env->FindClass("com/banuba/quickstart_c_api/Image"); + auto image_class_ref = (jclass) env->NewGlobalRef(image_class); jobject this_ref = env->NewGlobalRef(thiz); + + bool is_process_image = static_cast(jis_process_image); if(!is_process_image) { - draw_image_without_processing(pb_image, image_format, env, this_ref); + draw_image_from_pixel_buffer(pb_image, image_format, env, this_ref, image_class_ref); + env->DeleteGlobalRef(image_class_ref); + env->DeleteGlobalRef(this_ref); return; } @@ -353,10 +374,10 @@ extern "C" env->GetJavaVM(&jvm); // Callback for received pixel buffer from the offscreen effect player - auto get_pixel_buffer_callback = [this_ref, jvm, image_format](image_processing_result_sptr result) { + auto get_pixel_buffer_callback = [this_ref, jvm, image_format, image_class_ref](image_processing_result_sptr result) { if (result != nullptr) { // Callback for update data in render thread - auto get_image_callback = [this_ref, jvm, image_format](pixel_buffer_sptr image) { + auto get_image_callback = [this_ref, jvm, image_format, image_class_ref](pixel_buffer_sptr pb_image) { JNIEnv* env = nullptr; // double check it's all ok @@ -364,91 +385,29 @@ extern "C" if (getEnvStat == JNI_EDETACHED) { if (jvm->AttachCurrentThread((JNIEnv**) &env, nullptr) != JNI_OK) { print_message("GetEnv: Failed to attach"); + env->DeleteGlobalRef(image_class_ref); + env->DeleteGlobalRef(this_ref); return; } } else if (getEnvStat == JNI_EVERSION) { print_message("GetEnv: version not supported"); - return; - } - - if (image == nullptr) { - print_message("GetEnv: image is null"); + env->DeleteGlobalRef(image_class_ref); env->DeleteGlobalRef(this_ref); - jvm->DetachCurrentThread(); return; } - jbyteArray byte_array0 = nullptr; - jbyteArray byte_array1 = nullptr; - jbyteArray byte_array2 = nullptr; - switch(image_format) { - case bnb::oep::interfaces::image_format::bpc8_rgba: { - auto size = image->get_width() * image->get_height() * image->get_bytes_per_pixel(); - byte_array0 = env->NewByteArray(size); - env->SetByteArrayRegion(byte_array0, 0, size, reinterpret_cast((uint8_t*) image->get_base_sptr().get())); - break; - } - case bnb::oep::interfaces::image_format::nv12_bt601_full: { - auto size0 = image->get_width() * image->get_height() * image->get_bytes_per_pixel(); - auto size1 = image->get_width() * image->get_height() * image->get_bytes_per_pixel() / 2; - void* buf0 = reinterpret_cast((void*) image->get_base_sptr_of_plane(0).get()); - void* buf1 = reinterpret_cast((void*) image->get_base_sptr_of_plane(1).get()); - - byte_array0 = env->NewByteArray(size0); - byte_array1 = env->NewByteArray(size1); - byte_array2 = env->NewByteArray(0); - env->SetByteArrayRegion(byte_array0, 0, size0, reinterpret_cast(buf0)); - env->SetByteArrayRegion(byte_array1, 0, size1, reinterpret_cast(buf1)); - break; - } - case bnb::oep::interfaces::image_format::i420_bt601_full: { - auto size0 = image->get_width_of_plane(0) * image->get_height() * image->get_bytes_per_pixel_of_plane(0); - auto size1 = image->get_width_of_plane(1) * image->get_height() * image->get_bytes_per_pixel_of_plane(1); - auto size2 = image->get_width_of_plane(2) * image->get_height() * image->get_bytes_per_pixel_of_plane(2); - void* buf0 = reinterpret_cast((void*) image->get_base_sptr_of_plane(0).get()); - void* buf1 = reinterpret_cast((void*) image->get_base_sptr_of_plane(1).get()); - void* buf2 = reinterpret_cast((void*) image->get_base_sptr_of_plane(2).get()); - - byte_array0 = env->NewByteArray(size0); - byte_array1 = env->NewByteArray(size1); - byte_array2 = env->NewByteArray(size2); - env->SetByteArrayRegion(byte_array0, 0, size0, reinterpret_cast(buf0)); - int u_width = image->get_width_of_plane(1); - int u_stride = image->get_bytes_per_row_of_plane(1); - - for(int i = 0; i < image->get_height_of_plane(1); ++i) { - env->SetByteArrayRegion(byte_array1, u_width * i, u_width, reinterpret_cast(buf1) + u_stride * i); - } - - int v_width = image->get_width_of_plane(2); - int v_stride = image->get_bytes_per_row_of_plane(2); - - for(int i = 0; i < image->get_height_of_plane(2); ++i) { - env->SetByteArrayRegion(byte_array2, v_width * i, v_width, reinterpret_cast(buf2) + v_stride * i); - } - break; - } - default: - break; - } - - if(byte_array0 == nullptr && byte_array1 == nullptr && byte_array2 == nullptr) { - print_message("GetEnv: unsupported output image format"); + if (pb_image == nullptr) { + print_message("GetEnv: image is null"); + env->DeleteGlobalRef(image_class_ref); env->DeleteGlobalRef(this_ref); jvm->DetachCurrentThread(); return; } - jclass jcallback_class = env->GetObjectClass(this_ref); - jmethodID jcallback_method = env->GetMethodID(jcallback_class, "onDataReady", "([B[B[BII)V"); - - // call callback - env->CallVoidMethod(this_ref, jcallback_method, byte_array0, byte_array1, byte_array2, image->get_width(), image->get_height()); - if (env->ExceptionCheck()) { - env->ExceptionDescribe(); - } + draw_image_from_pixel_buffer(std::move(pb_image), image_format, env, this_ref, + image_class_ref); + env->DeleteGlobalRef(image_class_ref); env->DeleteGlobalRef(this_ref); - jvm->DetachCurrentThread(); }; // Get image from effect_player and return it in the callback diff --git a/app/src/main/java/com/banuba/quickstart_c_api/Image.java b/app/src/main/java/com/banuba/quickstart_c_api/Image.java new file mode 100644 index 0000000..8946109 --- /dev/null +++ b/app/src/main/java/com/banuba/quickstart_c_api/Image.java @@ -0,0 +1,19 @@ +package com.banuba.quickstart_c_api; + +public class Image { + // List imageDataPlanes; + + Image(byte[] image0, byte[] image1, byte[] image2, int w, int h) { + mImage0 = image0; + mImage1 = image1; + mImage2 = image2; + mWidth = w; + mHeight = h; + + } + public byte[] mImage0; + public byte[] mImage1; + public byte[] mImage2; + public int mWidth; + public int mHeight; +} diff --git a/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java b/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java index b189467..0a4751c 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java @@ -46,7 +46,7 @@ public class MainActivity extends AppCompatActivity { private ImageFormat mImageFormat = ImageFormat.NV12; // OEP doesn't work if mIsOEPEnabled = false; - private boolean mIsOEPEnabled = true; + private boolean mIsOEPEnabled = false; void createRenderer() { switch (mImageFormat) { @@ -64,11 +64,10 @@ void createRenderer() { void createOEP() { oep = new OffscreenEffectPlayer(size.getWidth(), size.getHeight()); - oep.loadEffect("effects/") + oep.loadEffect("effects/"); oep.setDataReadyCallback( - (image0,image1, image2, width, height) -> { - List planes = Arrays.asList(image0, image1, image2); - renderer.drawImage(planes, width, height); + (image) -> { + renderer.drawImage(image); glView.requestRender(); }); } @@ -159,6 +158,7 @@ private void startCamera() { proxy -> { updateImage(proxy); oep.processImageAsync(mImage, mIsOEPEnabled); + proxy.close(); }); cameraProvider.bindToLifecycle(MainActivity.this, cameraSelector, imageAnalysis); diff --git a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java index 9db41ca..09607ea 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java @@ -41,7 +41,7 @@ public void destroy() { } } - /* image must be NV12 format */ + /* image must be NV12 or i420 format */ public void processImageAsync(OffscreenEffectPlayerImage image, boolean isProcessImage) { externalProcessImageAsync(mOep, image.mImageZero, image.mImageFirst, image.mImageSecond, image.mImageInfo, isProcessImage); } @@ -79,16 +79,16 @@ public void evalJs(String script) { } public interface DataReadyCallback { - void onDataReady(byte[] image0, byte[] image1, byte[] image2, int width, int height); + void onDataReady(Image image); } public void setDataReadyCallback(DataReadyCallback callback) { mDataReadyCallback = callback; } - private void onDataReady(byte[] image0, byte[] image1, byte[] image2, int width, int height) { + private void onDataReady(Image image) { if (mDataReadyCallback != null) { - mDataReadyCallback.onDataReady(image0, image1, image2, width, height); + mDataReadyCallback.onDataReady(image); } } @@ -111,3 +111,4 @@ private void onDataReady(byte[] image0, byte[] image1, byte[] image2, int width, System.loadLibrary("native-lib"); } } + diff --git a/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java b/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java index 691ca1b..bfdf30b 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java @@ -7,11 +7,14 @@ import java.nio.ByteBuffer; import java.nio.FloatBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import javax.microedition.khronos.egl.EGLConfig; import javax.microedition.khronos.opengles.GL10; +import com.banuba.quickstart_c_api.Image; + public class GLRenderer implements GLSurfaceView.Renderer { /* shaders */ @@ -178,10 +181,10 @@ public void scaleMatrix() { mMat4[5] = yScale; } - public void drawImage(List imageDataPlanes, int width, int height) { - mImageDataPlanes = imageDataPlanes; - mImageWidth = width; - mImageHeight = height; + public void drawImage(Image image) { + mImageDataPlanes = Arrays.asList(image.mImage0, image.mImage1, image.mImage2);; + mImageWidth = image.mWidth; + mImageHeight = image.mHeight; } /* destructor */ From 604bf267d7e049008985dd133d8bb90b20fdfedb Mon Sep 17 00:00:00 2001 From: romanbylina Date: Thu, 18 Aug 2022 14:11:27 +0300 Subject: [PATCH 4/8] put Image object to OEP --- app/src/main/cpp/native-lib.cpp | 126 +++++++++--------- .../com/banuba/quickstart_c_api/Image.java | 28 ++-- .../banuba/quickstart_c_api/MainActivity.java | 37 +++-- .../OffscreenEffectPlayer.java | 17 ++- .../OffscreenEffectPlayerImage.java | 45 ------- .../rendering/GLRenderer.java | 3 +- 6 files changed, 116 insertions(+), 140 deletions(-) delete mode 100644 app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java diff --git a/app/src/main/cpp/native-lib.cpp b/app/src/main/cpp/native-lib.cpp index 6feda23..aa90fa2 100644 --- a/app/src/main/cpp/native-lib.cpp +++ b/app/src/main/cpp/native-lib.cpp @@ -21,7 +21,10 @@ namespace { - struct image_info { + struct image { + uint8_t * input_image_data0; + uint8_t * input_image_data1; + uint8_t * input_image_data2; int width; int height; int row_stride0; @@ -30,11 +33,7 @@ namespace int pixel_stride0; int pixel_stride1; int pixel_stride2; - int input_orientation; - int output_orientation; int pixel_format; - int image_format; - bool require_mirroring; }; int get_int_field(const char* field_name, JNIEnv* env, jobject object, jclass object_class) @@ -43,34 +42,36 @@ namespace return env->GetIntField(object, field_id); } - bool get_bool_field(const char* field_name, JNIEnv* env, jobject object, jclass object_class) + uint8_t * get_byte_buffer_field(const char* field_name, JNIEnv* env, jobject object, jclass object_class) { - jfieldID field_id = env->GetFieldID(object_class, field_name, "Z"); - return env->GetBooleanField(object, field_id); + jfieldID fid = env-> GetFieldID (object_class, field_name, "Ljava/nio/ByteBuffer;"); + jobject data = env-> GetObjectField (object, fid); + return static_cast(env->GetDirectBufferAddress(data)); } - image_info get_image_info(JNIEnv* env, jobject image_info) + image get_image(JNIEnv* env, jobject jimage) { - jclass cls = env->GetObjectClass(image_info); + jclass image_class = env->GetObjectClass(jimage); + return { - get_int_field("width", env, image_info, cls), - get_int_field("height", env, image_info, cls), - get_int_field("rowStride0", env, image_info, cls), - get_int_field("rowStride1", env, image_info, cls), - get_int_field("rowStride2", env, image_info, cls), - get_int_field("pixelStride0", env, image_info, cls), - get_int_field("pixelStride1", env, image_info, cls), - get_int_field("pixelStride2", env, image_info, cls), - get_int_field("inputOrientation", env, image_info, cls), - get_int_field("outputOrientation", env, image_info, cls), - get_int_field("pixelFormat", env, image_info, cls), - get_int_field("imageFormat", env, image_info, cls), - get_bool_field("requireMirroring", env, image_info, cls) + get_byte_buffer_field("mImageZero", env, jimage, image_class), + get_byte_buffer_field("mImageFirst", env, jimage, image_class), + get_byte_buffer_field("mImageSecond", env, jimage, image_class), + get_int_field("mWidth", env, jimage, image_class), + get_int_field("mHeight", env, jimage, image_class), + get_int_field("rowStrideZero", env, jimage, image_class), + get_int_field("rowStrideFirst", env, jimage, image_class), + get_int_field("rowStrideSecond", env, jimage, image_class), + get_int_field("pixelStrideZero", env, jimage, image_class), + get_int_field("pixelStrideFirst", env, jimage, image_class), + get_int_field("pixelStrideSecond", env, jimage, image_class), + get_int_field("pixelFormat", env, jimage, image_class) }; } - bnb::oep::interfaces::image_format get_image_format(int output_image_format) { - switch (output_image_format) { + bnb::oep::interfaces::image_format get_image_format(jint jimage_format) { + int image_format = static_cast(jimage_format); + switch (image_format) { case 1: return bnb::oep::interfaces::image_format::nv12_bt601_full; case 2: @@ -81,11 +82,7 @@ namespace } std::vector create_planes_from_format( - uint8_t* input_image_data0, - uint8_t* input_image_data1, - uint8_t* input_image_data2, - bnb::oep::interfaces::image_format format, - const image_info& image_info) + const image& img, bnb::oep::interfaces::image_format format) { // only nv12_bt601_full and i420_bt601_full are supported assert(format == bnb::oep::interfaces::image_format::nv12_bt601_full @@ -93,28 +90,28 @@ namespace using ns_pb = bnb::oep::interfaces::pixel_buffer; if(format == bnb::oep::interfaces::image_format::nv12_bt601_full) { - int y_size = image_info.row_stride0 * image_info.height; - int uv_size = image_info.row_stride1 * image_info.height / 2; + int y_size = img.row_stride0 * img.height; + int uv_size = img.row_stride1 * img.height / 2; ns_pb::plane_sptr y_plane_data(new ns_pb::plane_sptr::element_type[y_size], [](uint8_t* ptr) { delete[] ptr; }); - std::memcpy(y_plane_data.get(), input_image_data0, y_size); + std::memcpy(y_plane_data.get(), img.input_image_data0, y_size); ns_pb::plane_sptr uv_plane_data(new ns_pb::plane_sptr::element_type[uv_size], [](uint8_t* ptr) { delete[] ptr; }); - std::memcpy(uv_plane_data.get(), input_image_data1, uv_size); + std::memcpy(uv_plane_data.get(), img.input_image_data1, uv_size); - ns_pb::plane_data y_plane{std::move(y_plane_data), static_cast(y_size), image_info.row_stride0}; - ns_pb::plane_data uv_plane{std::move(uv_plane_data), static_cast(uv_size), image_info.row_stride1}; + ns_pb::plane_data y_plane{std::move(y_plane_data), static_cast(y_size), img.row_stride0}; + ns_pb::plane_data uv_plane{std::move(uv_plane_data), static_cast(uv_size), img.row_stride1}; std::vector planes{std::move(y_plane), std::move(uv_plane)}; return planes; } if(format == bnb::oep::interfaces::image_format::i420_bt601_full) { - int y_size = image_info.row_stride0 * image_info.height; - int u_size = image_info.row_stride1 * image_info.height / 4; - int v_size = image_info.row_stride2 * image_info.height/ 4; + int y_size = img.row_stride0 * img.height; + int u_size = img.row_stride1 * img.height / 4; + int v_size = img.row_stride2 * img.height/ 4; ns_pb::plane_sptr y_plane_data(new ns_pb::plane_sptr::element_type[y_size], [](uint8_t* ptr) { delete[] ptr;}); - std::memcpy(y_plane_data.get(), input_image_data0, y_size); + std::memcpy(y_plane_data.get(), img.input_image_data0, y_size); ns_pb::plane_sptr u_plane_data(new ns_pb::plane_sptr::element_type[u_size], [](uint8_t* ptr) { delete[] ptr; }); ns_pb::plane_sptr v_plane_data(new ns_pb::plane_sptr::element_type[v_size], [](uint8_t* ptr) { delete[] ptr; }); @@ -122,14 +119,14 @@ namespace auto ptr_u = u_plane_data.get(); auto ptr_v = v_plane_data.get(); - for (unsigned row = 0; row < image_info.height * image_info.row_stride1 / 2; row += 2) { - *ptr_u++ = input_image_data1[row]; - *ptr_v++ = input_image_data1[row + 1]; + for (unsigned row = 0; row < img.height * img.row_stride1 / 2; row += 2) { + *ptr_u++ = img.input_image_data1[row]; + *ptr_v++ = img.input_image_data1[row + 1]; } - ns_pb::plane_data y_plane{std::move(y_plane_data), static_cast(y_size), image_info.row_stride0}; - ns_pb::plane_data u_plane{std::move(u_plane_data), static_cast(u_size), image_info.row_stride1 / 2}; - ns_pb::plane_data v_plane{std::move(v_plane_data), static_cast(v_size), image_info.row_stride2 / 2}; + ns_pb::plane_data y_plane{std::move(y_plane_data), static_cast(y_size), img.row_stride0}; + ns_pb::plane_data u_plane{std::move(u_plane_data), static_cast(u_size), img.row_stride1 / 2}; + ns_pb::plane_data v_plane{std::move(v_plane_data), static_cast(v_size), img.row_stride2 / 2}; std::vector planes{std::move(y_plane), std::move(u_plane), std::move(v_plane)}; return planes; } @@ -137,16 +134,12 @@ namespace return {}; } - pixel_buffer_sptr create_pixel_buffer(JNIEnv* env, jobject jimageY, jobject jimageU, jobject jimageV, - const image_info& image_info, bnb::oep::interfaces::image_format image_format) + pixel_buffer_sptr create_pixel_buffer(JNIEnv* env, jobject jimage, bnb::oep::interfaces::image_format image_format) { - uint8_t* input_image_data0 = static_cast(env->GetDirectBufferAddress(jimageY)); - uint8_t* input_image_data1 = static_cast(env->GetDirectBufferAddress(jimageU)); - uint8_t* input_image_data2 = static_cast(env->GetDirectBufferAddress(jimageV)); - auto width = static_cast(image_info.width); - auto height = static_cast(image_info.height); - auto planes = create_planes_from_format(input_image_data0, input_image_data1, input_image_data2, image_format, image_info); - return bnb::oep::interfaces::pixel_buffer::create(planes, image_format, width, height, [](auto* pb) { delete pb; }); + image img = get_image(env, jimage); + + auto planes = create_planes_from_format(img, image_format); + return bnb::oep::interfaces::pixel_buffer::create(planes, image_format, img.width, img.height, [](auto* pb) { delete pb; }); } struct banuba_sdk_manager @@ -182,8 +175,9 @@ namespace return ret; } - bnb::oep::interfaces::rotation java_rotation_to_oep_rotation(int rotation) + bnb::oep::interfaces::rotation java_rotation_to_oep_rotation(jint orientation) { + int rotation = static_cast(orientation); switch (rotation) { case 0: return bnb::oep::interfaces::rotation::deg0; @@ -345,17 +339,20 @@ extern "C" /* OffscreenEffectPlayer::externalProcessImageAsync - java interface */ JNIEXPORT void JNICALL Java_com_banuba_quickstart_1c_1api_OffscreenEffectPlayer_externalProcessImageAsync( JNIEnv* env, jobject thiz, jlong jsdk, - jobject jimageY, jobject jimageU, jobject jimageV, - jobject jimage_info, jboolean jis_process_image) + jobject jimage, + jint jinput_orientation, + jboolean jis_required_mirroring, + jint joutput_orientation, + jint jimage_format, + jboolean jis_process_image) { auto oep = get_offscreen_effect_player_from_jlong(jsdk); if (oep == nullptr) { return; } - auto image_info = get_image_info(env, jimage_info); - auto image_format = get_image_format(image_info.image_format); - auto pb_image = create_pixel_buffer(env, jimageY, jimageU, jimageV, image_info, image_format); + auto image_format = get_image_format(jimage_format); + auto pb_image = create_pixel_buffer(env, jimage, image_format); jclass image_class = env->FindClass("com/banuba/quickstart_c_api/Image"); auto image_class_ref = (jclass) env->NewGlobalRef(image_class); @@ -414,9 +411,10 @@ extern "C" result->get_image(image_format, get_image_callback); } }; - auto in_rotation = java_rotation_to_oep_rotation(image_info.input_orientation); - auto out_rotation = java_rotation_to_oep_rotation(image_info.output_orientation); - oep->process_image_async(pb_image, in_rotation, image_info.require_mirroring, get_pixel_buffer_callback, out_rotation); + auto in_rotation = java_rotation_to_oep_rotation(jinput_orientation); + auto out_rotation = java_rotation_to_oep_rotation(joutput_orientation); + auto require_mirroring = static_cast(jis_required_mirroring); + oep->process_image_async(pb_image, in_rotation, require_mirroring, get_pixel_buffer_callback, out_rotation); } /* OffscreenEffectPlayer::externalSurfaceChanged - java interface */ diff --git a/app/src/main/java/com/banuba/quickstart_c_api/Image.java b/app/src/main/java/com/banuba/quickstart_c_api/Image.java index 8946109..984f727 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/Image.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/Image.java @@ -1,19 +1,31 @@ package com.banuba.quickstart_c_api; +import java.nio.ByteBuffer; + public class Image { - // List imageDataPlanes; + + Image() {} Image(byte[] image0, byte[] image1, byte[] image2, int w, int h) { - mImage0 = image0; - mImage1 = image1; - mImage2 = image2; + mImageZero = ByteBuffer.wrap(image0); + mImageFirst = ByteBuffer.wrap(image1); + mImageSecond = ByteBuffer.wrap(image2); + mWidth = w; mHeight = h; - } - public byte[] mImage0; - public byte[] mImage1; - public byte[] mImage2; + + public ByteBuffer mImageZero; + public ByteBuffer mImageFirst; + public ByteBuffer mImageSecond; + public int mWidth; public int mHeight; + public int rowStrideZero; + public int rowStrideFirst; + public int rowStrideSecond; + public int pixelStrideZero; + public int pixelStrideFirst; + public int pixelStrideSecond; + public int pixelFormat; } diff --git a/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java b/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java index 0a4751c..ff262f2 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java @@ -28,8 +28,6 @@ import com.google.common.util.concurrent.ListenableFuture; import java.io.File; -import java.util.Arrays; -import java.util.List; public class MainActivity extends AppCompatActivity { private static int CAMERA_PERMISSION_REQUEST = 12345; @@ -40,7 +38,7 @@ public class MainActivity extends AppCompatActivity { private OffscreenEffectPlayer oep = null; private GLSurfaceView glView = null; private GLRenderer renderer = null; - private OffscreenEffectPlayerImage mImage = null; + private Image mImage = null; // Changing mImageOutputFormat will cause the format's changing (input and output image of OEP) private ImageFormat mImageFormat = ImageFormat.NV12; @@ -96,7 +94,7 @@ protected void onCreate(@Nullable Bundle savedInstanceState) { /* Create offscreen effect player */ createOEP(); - mImage = new OffscreenEffectPlayerImage(); + mImage = new Image(); requestCameraPermissionAndStart(); } @@ -157,7 +155,12 @@ private void startCamera() { ContextCompat.getMainExecutor(MainActivity.this), proxy -> { updateImage(proxy); - oep.processImageAsync(mImage, mIsOEPEnabled); + + int rotation = getRotation(MainActivity.this); + int imageFormat = mImageFormat.ordinal(); + + oep.processImageAsync(mImage, getInputOrientation(rotation), + false, getOutputOrientation(rotation), imageFormat, mIsOEPEnabled); proxy.close(); }); @@ -196,29 +199,25 @@ private int getOutputOrientation(int rotation) { } private void updateImage(ImageProxy imageProxy) { - int rotation = getRotation(MainActivity.this); - mImage.mImageInfo.width = imageProxy.getWidth(); - mImage.mImageInfo.height = imageProxy.getHeight(); - mImage.mImageInfo.inputOrientation = getInputOrientation(rotation); - mImage.mImageInfo.outputOrientation = getOutputOrientation(rotation); - mImage.mImageInfo.pixelFormat = imageProxy.getImage().getFormat(); - mImage.mImageInfo.requireMirroring = false; - mImage.mImageInfo.imageFormat = mImageFormat.ordinal(); + mImage.mWidth = imageProxy.getWidth(); + mImage.mHeight = imageProxy.getHeight(); + + mImage.pixelFormat = imageProxy.getImage().getFormat(); - mImage.mImageInfo.rowStride0 = imageProxy.getPlanes()[0].getRowStride(); + mImage.rowStrideZero = imageProxy.getPlanes()[0].getRowStride(); mImage.mImageZero = imageProxy.getPlanes()[0].getBuffer(); - mImage.mImageInfo.pixelStride0 = imageProxy.getPlanes()[0].getPixelStride(); + mImage.pixelStrideZero = imageProxy.getPlanes()[0].getPixelStride(); int planesNumber = imageProxy.getPlanes().length; if (planesNumber > 1) { - mImage.mImageInfo.rowStride1 = imageProxy.getPlanes()[1].getRowStride(); + mImage.rowStrideFirst = imageProxy.getPlanes()[1].getRowStride(); mImage.mImageFirst = imageProxy.getPlanes()[1].getBuffer(); - mImage.mImageInfo.pixelStride1 = imageProxy.getPlanes()[1].getPixelStride(); + mImage.pixelStrideFirst = imageProxy.getPlanes()[1].getPixelStride(); if (planesNumber > 2) { - mImage.mImageInfo.rowStride2 = imageProxy.getPlanes()[2].getRowStride(); + mImage.rowStrideSecond = imageProxy.getPlanes()[2].getRowStride(); mImage.mImageSecond = imageProxy.getPlanes()[2].getBuffer(); - mImage.mImageInfo.pixelStride2 = imageProxy.getPlanes()[2].getPixelStride(); + mImage.pixelStrideSecond = imageProxy.getPlanes()[2].getPixelStride(); } } } diff --git a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java index 09607ea..c51e564 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java @@ -42,8 +42,14 @@ public void destroy() { } /* image must be NV12 or i420 format */ - public void processImageAsync(OffscreenEffectPlayerImage image, boolean isProcessImage) { - externalProcessImageAsync(mOep, image.mImageZero, image.mImageFirst, image.mImageSecond, image.mImageInfo, isProcessImage); + public void processImageAsync(Image image, + int inputOrientation, + boolean isRequiredMirroring, + int outputOrientation, + int imageFormat, + boolean isProcessedImage) { + externalProcessImageAsync(mOep, image, inputOrientation, isRequiredMirroring, + outputOrientation, imageFormat, isProcessedImage); } public void surfaceChanged(int width, int height) { @@ -97,7 +103,12 @@ private void onDataReady(Image image) { private static native void externalDeinit(); private native long externalCreate(int width, int height); private native void externalDestroy(long oep); - private native void externalProcessImageAsync(long oep, ByteBuffer imageY, ByteBuffer imageU, ByteBuffer imageV, ImageInfo info, boolean isProcessImage); + private native void externalProcessImageAsync(long oep, Image image, + int inputOrientation, + boolean isRequiredMirroring, + int outputOrientation, + int imageFormat, + boolean isProcessedImage); private native void externalSurfaceChanged(long oep, int width, int height); private native void externalLoadEffect(long oep, String effectPath); private native void externalUnloadEffect(long oep); diff --git a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java deleted file mode 100644 index f592311..0000000 --- a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.banuba.quickstart_c_api; - -import java.nio.ByteBuffer; - -public class OffscreenEffectPlayerImage { - OffscreenEffectPlayerImage() { - mImageInfo = new ImageInfo(); - } - - public ImageInfo mImageInfo; - public ByteBuffer mImageZero = null; - public ByteBuffer mImageFirst = null; - public ByteBuffer mImageSecond = null; -} - -class ImageInfo { - ImageInfo() { - width = 0; - height = 0; - inputOrientation = 0; - outputOrientation = 0; - requireMirroring = false; - rowStride0 = 0; - rowStride1 = 0; - rowStride2 = 0; - pixelStride0 = 0; - pixelStride1 = 0; - pixelStride2 = 0; - pixelFormat = 0; - imageFormat = 0; - } - public int width; - public int height; - public int inputOrientation; - public int outputOrientation; - public int rowStride0; - public int rowStride1; - public int rowStride2; - public int pixelStride0; - public int pixelStride1; - public int pixelStride2; - public int pixelFormat; - public int imageFormat; - public boolean requireMirroring; -} diff --git a/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java b/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java index bfdf30b..84abf98 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java @@ -182,7 +182,8 @@ public void scaleMatrix() { } public void drawImage(Image image) { - mImageDataPlanes = Arrays.asList(image.mImage0, image.mImage1, image.mImage2);; + mImageDataPlanes = Arrays.asList(image.mImageZero.array(), + image.mImageFirst.array(), image.mImageSecond.array()); mImageWidth = image.mWidth; mImageHeight = image.mHeight; } From f1b7815d5ef3c5d52db9aba3fd64ac324d9901e9 Mon Sep 17 00:00:00 2001 From: romanbylina Date: Thu, 18 Aug 2022 14:35:23 +0300 Subject: [PATCH 5/8] small fix --- app/src/main/cpp/native-lib.cpp | 1 - .../main/java/com/banuba/quickstart_c_api/Image.java | 12 +++++++++--- .../quickstart_c_api/rendering/GLRenderer.java | 12 ++++++++++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/app/src/main/cpp/native-lib.cpp b/app/src/main/cpp/native-lib.cpp index aa90fa2..0739a60 100644 --- a/app/src/main/cpp/native-lib.cpp +++ b/app/src/main/cpp/native-lib.cpp @@ -223,7 +223,6 @@ namespace byte_array0 = env->NewByteArray(size0); byte_array1 = env->NewByteArray(size1); - byte_array2 = env->NewByteArray(0); env->SetByteArrayRegion(byte_array0, 0, size0, reinterpret_cast(buf0)); env->SetByteArrayRegion(byte_array1, 0, size1, reinterpret_cast(buf1)); break; diff --git a/app/src/main/java/com/banuba/quickstart_c_api/Image.java b/app/src/main/java/com/banuba/quickstart_c_api/Image.java index 984f727..1e022c3 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/Image.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/Image.java @@ -7,9 +7,15 @@ public class Image { Image() {} Image(byte[] image0, byte[] image1, byte[] image2, int w, int h) { - mImageZero = ByteBuffer.wrap(image0); - mImageFirst = ByteBuffer.wrap(image1); - mImageSecond = ByteBuffer.wrap(image2); + if(image0 != null) { + mImageZero = ByteBuffer.wrap(image0); + } + if(image1 != null) { + mImageFirst = ByteBuffer.wrap(image1); + } + if(image2 != null) { + mImageSecond = ByteBuffer.wrap(image2); + } mWidth = w; mHeight = h; diff --git a/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java b/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java index 84abf98..7d6be40 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java @@ -182,8 +182,16 @@ public void scaleMatrix() { } public void drawImage(Image image) { - mImageDataPlanes = Arrays.asList(image.mImageZero.array(), - image.mImageFirst.array(), image.mImageSecond.array()); + mImageDataPlanes.clear(); + if(image.mImageZero != null) { + mImageDataPlanes.add(image.mImageZero.array()); + } + if(image.mImageFirst != null) { + mImageDataPlanes.add(image.mImageFirst.array()); + } + if(image.mImageSecond != null) { + mImageDataPlanes.add(image.mImageSecond.array()); + } mImageWidth = image.mWidth; mImageHeight = image.mHeight; } From bab5375e58d5bc6461c3b05a87fe53156cfece2b Mon Sep 17 00:00:00 2001 From: romanbylina Date: Thu, 18 Aug 2022 14:40:27 +0300 Subject: [PATCH 6/8] delete code for disabled OEP --- app/src/main/cpp/native-lib.cpp | 11 +---------- .../com/banuba/quickstart_c_api/MainActivity.java | 7 ++----- .../quickstart_c_api/OffscreenEffectPlayer.java | 8 +++----- 3 files changed, 6 insertions(+), 20 deletions(-) diff --git a/app/src/main/cpp/native-lib.cpp b/app/src/main/cpp/native-lib.cpp index 0739a60..d865986 100644 --- a/app/src/main/cpp/native-lib.cpp +++ b/app/src/main/cpp/native-lib.cpp @@ -342,8 +342,7 @@ extern "C" jint jinput_orientation, jboolean jis_required_mirroring, jint joutput_orientation, - jint jimage_format, - jboolean jis_process_image) + jint jimage_format) { auto oep = get_offscreen_effect_player_from_jlong(jsdk); if (oep == nullptr) { @@ -358,14 +357,6 @@ extern "C" jobject this_ref = env->NewGlobalRef(thiz); - bool is_process_image = static_cast(jis_process_image); - if(!is_process_image) { - draw_image_from_pixel_buffer(pb_image, image_format, env, this_ref, image_class_ref); - env->DeleteGlobalRef(image_class_ref); - env->DeleteGlobalRef(this_ref); - return; - } - JavaVM* jvm; env->GetJavaVM(&jvm); diff --git a/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java b/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java index ff262f2..047fe2b 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java @@ -41,10 +41,7 @@ public class MainActivity extends AppCompatActivity { private Image mImage = null; // Changing mImageOutputFormat will cause the format's changing (input and output image of OEP) - private ImageFormat mImageFormat = ImageFormat.NV12; - - // OEP doesn't work if mIsOEPEnabled = false; - private boolean mIsOEPEnabled = false; + private ImageFormat mImageFormat = ImageFormat.i420; void createRenderer() { switch (mImageFormat) { @@ -160,7 +157,7 @@ private void startCamera() { int imageFormat = mImageFormat.ordinal(); oep.processImageAsync(mImage, getInputOrientation(rotation), - false, getOutputOrientation(rotation), imageFormat, mIsOEPEnabled); + false, getOutputOrientation(rotation), imageFormat); proxy.close(); }); diff --git a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java index c51e564..dfe3eda 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java @@ -46,10 +46,9 @@ public void processImageAsync(Image image, int inputOrientation, boolean isRequiredMirroring, int outputOrientation, - int imageFormat, - boolean isProcessedImage) { + int imageFormat) { externalProcessImageAsync(mOep, image, inputOrientation, isRequiredMirroring, - outputOrientation, imageFormat, isProcessedImage); + outputOrientation, imageFormat); } public void surfaceChanged(int width, int height) { @@ -107,8 +106,7 @@ private native void externalProcessImageAsync(long oep, Image image, int inputOrientation, boolean isRequiredMirroring, int outputOrientation, - int imageFormat, - boolean isProcessedImage); + int imageFormat); private native void externalSurfaceChanged(long oep, int width, int height); private native void externalLoadEffect(long oep, String effectPath); private native void externalUnloadEffect(long oep); From 8ccc8277cf890a23220db08933609d18f4df750c Mon Sep 17 00:00:00 2001 From: romanbylina Date: Wed, 24 Aug 2022 10:10:05 +0300 Subject: [PATCH 7/8] review changes --- app/src/main/cpp/native-lib.cpp | 24 +++--- .../com/banuba/quickstart_c_api/Image.java | 37 --------- .../banuba/quickstart_c_api/MainActivity.java | 41 +++++----- .../OffscreenEffectPlayer.java | 10 +-- .../OffscreenEffectPlayerImage.java | 82 +++++++++++++++++++ .../rendering/GLRenderer.java | 22 ++--- 6 files changed, 127 insertions(+), 89 deletions(-) delete mode 100644 app/src/main/java/com/banuba/quickstart_c_api/Image.java create mode 100644 app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java diff --git a/app/src/main/cpp/native-lib.cpp b/app/src/main/cpp/native-lib.cpp index d865986..23f45cd 100644 --- a/app/src/main/cpp/native-lib.cpp +++ b/app/src/main/cpp/native-lib.cpp @@ -54,18 +54,18 @@ namespace jclass image_class = env->GetObjectClass(jimage); return { - get_byte_buffer_field("mImageZero", env, jimage, image_class), - get_byte_buffer_field("mImageFirst", env, jimage, image_class), - get_byte_buffer_field("mImageSecond", env, jimage, image_class), + get_byte_buffer_field("mPlane0", env, jimage, image_class), + get_byte_buffer_field("mPlane1", env, jimage, image_class), + get_byte_buffer_field("mPlane2", env, jimage, image_class), get_int_field("mWidth", env, jimage, image_class), get_int_field("mHeight", env, jimage, image_class), - get_int_field("rowStrideZero", env, jimage, image_class), - get_int_field("rowStrideFirst", env, jimage, image_class), - get_int_field("rowStrideSecond", env, jimage, image_class), - get_int_field("pixelStrideZero", env, jimage, image_class), - get_int_field("pixelStrideFirst", env, jimage, image_class), - get_int_field("pixelStrideSecond", env, jimage, image_class), - get_int_field("pixelFormat", env, jimage, image_class) + get_int_field("mRowStride0", env, jimage, image_class), + get_int_field("mRowStride1", env, jimage, image_class), + get_int_field("mRowStride2", env, jimage, image_class), + get_int_field("mPixelStride0", env, jimage, image_class), + get_int_field("mPixelStride1", env, jimage, image_class), + get_int_field("mPixelStride2", env, jimage, image_class), + get_int_field("mPixelFormat", env, jimage, image_class) }; } @@ -282,7 +282,7 @@ namespace jclass jcallback_class = env->GetObjectClass(this_ref); jmethodID jcallback_method = env->GetMethodID(jcallback_class, - "onDataReady", "(Lcom/banuba/quickstart_c_api/Image;)V"); + "onDataReady", "(Lcom/banuba/quickstart_c_api/OffscreenEffectPlayerImage;)V"); env->CallVoidMethod(this_ref, jcallback_method, image); if (env->ExceptionCheck()) { @@ -352,7 +352,7 @@ extern "C" auto image_format = get_image_format(jimage_format); auto pb_image = create_pixel_buffer(env, jimage, image_format); - jclass image_class = env->FindClass("com/banuba/quickstart_c_api/Image"); + jclass image_class = env->FindClass("com/banuba/quickstart_c_api/OffscreenEffectPlayerImage"); auto image_class_ref = (jclass) env->NewGlobalRef(image_class); jobject this_ref = env->NewGlobalRef(thiz); diff --git a/app/src/main/java/com/banuba/quickstart_c_api/Image.java b/app/src/main/java/com/banuba/quickstart_c_api/Image.java deleted file mode 100644 index 1e022c3..0000000 --- a/app/src/main/java/com/banuba/quickstart_c_api/Image.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.banuba.quickstart_c_api; - -import java.nio.ByteBuffer; - -public class Image { - - Image() {} - - Image(byte[] image0, byte[] image1, byte[] image2, int w, int h) { - if(image0 != null) { - mImageZero = ByteBuffer.wrap(image0); - } - if(image1 != null) { - mImageFirst = ByteBuffer.wrap(image1); - } - if(image2 != null) { - mImageSecond = ByteBuffer.wrap(image2); - } - - mWidth = w; - mHeight = h; - } - - public ByteBuffer mImageZero; - public ByteBuffer mImageFirst; - public ByteBuffer mImageSecond; - - public int mWidth; - public int mHeight; - public int rowStrideZero; - public int rowStrideFirst; - public int rowStrideSecond; - public int pixelStrideZero; - public int pixelStrideFirst; - public int pixelStrideSecond; - public int pixelFormat; -} diff --git a/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java b/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java index 047fe2b..f507d89 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java @@ -38,7 +38,7 @@ public class MainActivity extends AppCompatActivity { private OffscreenEffectPlayer oep = null; private GLSurfaceView glView = null; private GLRenderer renderer = null; - private Image mImage = null; + private OffscreenEffectPlayerImage mOffscreenEffectPlayerImage = null; // Changing mImageOutputFormat will cause the format's changing (input and output image of OEP) private ImageFormat mImageFormat = ImageFormat.i420; @@ -61,8 +61,8 @@ void createOEP() { oep = new OffscreenEffectPlayer(size.getWidth(), size.getHeight()); oep.loadEffect("effects/"); oep.setDataReadyCallback( - (image) -> { - renderer.drawImage(image); + (offscreenEffectPlayerImage) -> { + renderer.drawImage(offscreenEffectPlayerImage); glView.requestRender(); }); } @@ -91,7 +91,7 @@ protected void onCreate(@Nullable Bundle savedInstanceState) { /* Create offscreen effect player */ createOEP(); - mImage = new Image(); + mOffscreenEffectPlayerImage = new OffscreenEffectPlayerImage(); requestCameraPermissionAndStart(); } @@ -156,7 +156,7 @@ private void startCamera() { int rotation = getRotation(MainActivity.this); int imageFormat = mImageFormat.ordinal(); - oep.processImageAsync(mImage, getInputOrientation(rotation), + oep.processImageAsync(mOffscreenEffectPlayerImage, getInputOrientation(rotation), false, getOutputOrientation(rotation), imageFormat); proxy.close(); @@ -195,26 +195,25 @@ private int getOutputOrientation(int rotation) { return 0; } - private void updateImage(ImageProxy imageProxy) { - - mImage.mWidth = imageProxy.getWidth(); - mImage.mHeight = imageProxy.getHeight(); - - mImage.pixelFormat = imageProxy.getImage().getFormat(); - - mImage.rowStrideZero = imageProxy.getPlanes()[0].getRowStride(); - mImage.mImageZero = imageProxy.getPlanes()[0].getBuffer(); - mImage.pixelStrideZero = imageProxy.getPlanes()[0].getPixelStride(); + private void updateImagePlane(ImageProxy imageProxy, int planeNumber) { + ImageProxy.PlaneProxy planeProxy = imageProxy.getPlanes()[planeNumber]; + mOffscreenEffectPlayerImage.setPlane( + planeNumber, + planeProxy.getRowStride(), + planeProxy.getBuffer(), + planeProxy.getPixelStride()); + } + private void updateImage(ImageProxy imageProxy) { + mOffscreenEffectPlayerImage.setWidth(imageProxy.getWidth()); + mOffscreenEffectPlayerImage.setHeight(imageProxy.getHeight()); + mOffscreenEffectPlayerImage.setPixelFormat(imageProxy.getImage().getFormat()); + updateImagePlane(imageProxy, 0); int planesNumber = imageProxy.getPlanes().length; if (planesNumber > 1) { - mImage.rowStrideFirst = imageProxy.getPlanes()[1].getRowStride(); - mImage.mImageFirst = imageProxy.getPlanes()[1].getBuffer(); - mImage.pixelStrideFirst = imageProxy.getPlanes()[1].getPixelStride(); + updateImagePlane(imageProxy, 1); if (planesNumber > 2) { - mImage.rowStrideSecond = imageProxy.getPlanes()[2].getRowStride(); - mImage.mImageSecond = imageProxy.getPlanes()[2].getBuffer(); - mImage.pixelStrideSecond = imageProxy.getPlanes()[2].getPixelStride(); + updateImagePlane(imageProxy, 2); } } } diff --git a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java index dfe3eda..eff3d65 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java @@ -1,7 +1,5 @@ package com.banuba.quickstart_c_api; -import java.nio.ByteBuffer; - class OffscreenEffectPlayer { private long mOep = 0; private DataReadyCallback mDataReadyCallback = null; @@ -42,7 +40,7 @@ public void destroy() { } /* image must be NV12 or i420 format */ - public void processImageAsync(Image image, + public void processImageAsync(OffscreenEffectPlayerImage image, int inputOrientation, boolean isRequiredMirroring, int outputOrientation, @@ -84,14 +82,14 @@ public void evalJs(String script) { } public interface DataReadyCallback { - void onDataReady(Image image); + void onDataReady(OffscreenEffectPlayerImage image); } public void setDataReadyCallback(DataReadyCallback callback) { mDataReadyCallback = callback; } - private void onDataReady(Image image) { + private void onDataReady(OffscreenEffectPlayerImage image) { if (mDataReadyCallback != null) { mDataReadyCallback.onDataReady(image); } @@ -102,7 +100,7 @@ private void onDataReady(Image image) { private static native void externalDeinit(); private native long externalCreate(int width, int height); private native void externalDestroy(long oep); - private native void externalProcessImageAsync(long oep, Image image, + private native void externalProcessImageAsync(long oep, OffscreenEffectPlayerImage image, int inputOrientation, boolean isRequiredMirroring, int outputOrientation, diff --git a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java new file mode 100644 index 0000000..367f828 --- /dev/null +++ b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java @@ -0,0 +1,82 @@ +package com.banuba.quickstart_c_api; + +import java.nio.ByteBuffer; + +public class OffscreenEffectPlayerImage { + + OffscreenEffectPlayerImage() {} + + OffscreenEffectPlayerImage(byte[] yPlane, byte[] uPlane, byte[] vPlane, int width, int height) { + if(yPlane != null) { + mPlane0 = ByteBuffer.wrap(yPlane); + } + if(uPlane != null) { + mPlane1 = ByteBuffer.wrap(uPlane); + } + if(vPlane != null) { + mPlane2 = ByteBuffer.wrap(vPlane); + } + + mWidth = width; + mHeight = height; + } + + void setWidth(int width) { mWidth = width; } + + void setHeight(int height) { mHeight = height; } + + void setPixelFormat(int pixelFormat) { + mPixelFormat = pixelFormat; + } + + void setPlane(int planeNumber, int planeWidth, ByteBuffer plane, int pixelStride) { + switch (planeNumber) { + case 1: + mRowStride1 = planeWidth; + mPlane1 = plane; + mPixelStride1 = pixelStride; + break; + case 2: + mRowStride2 = planeWidth; + mPlane2 = plane; + mPixelStride2 = pixelStride; + break; + default: + mRowStride0 = planeWidth; + mPlane0 = plane; + mPixelStride0 = pixelStride; + break; + } + } + + public ByteBuffer getPlane(int planeNumber) { + switch (planeNumber) { + case 1: + return mPlane1; + case 2: + return mPlane2; + default: + return mPlane0; + } + } + + public int getWidth() { return mWidth; } + + public int getHeight() { + return mHeight; + } + + private ByteBuffer mPlane0; + private ByteBuffer mPlane1; + private ByteBuffer mPlane2; + + private int mWidth; + private int mHeight; + private int mRowStride0; + private int mRowStride1; + private int mRowStride2; + private int mPixelStride0; + private int mPixelStride1; + private int mPixelStride2; + private int mPixelFormat; +} diff --git a/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java b/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java index 7d6be40..dd8c7fb 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java @@ -7,13 +7,12 @@ import java.nio.ByteBuffer; import java.nio.FloatBuffer; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import javax.microedition.khronos.egl.EGLConfig; import javax.microedition.khronos.opengles.GL10; -import com.banuba.quickstart_c_api.Image; +import com.banuba.quickstart_c_api.OffscreenEffectPlayerImage; public class GLRenderer implements GLSurfaceView.Renderer { @@ -181,19 +180,16 @@ public void scaleMatrix() { mMat4[5] = yScale; } - public void drawImage(Image image) { + public void drawImage(OffscreenEffectPlayerImage image) { mImageDataPlanes.clear(); - if(image.mImageZero != null) { - mImageDataPlanes.add(image.mImageZero.array()); - } - if(image.mImageFirst != null) { - mImageDataPlanes.add(image.mImageFirst.array()); - } - if(image.mImageSecond != null) { - mImageDataPlanes.add(image.mImageSecond.array()); + for(int i = 0; i < mTexturesCount; ++i) { + ByteBuffer plane = image.getPlane(i); + if(plane != null) { + mImageDataPlanes.add(plane.array()); + } } - mImageWidth = image.mWidth; - mImageHeight = image.mHeight; + mImageWidth = image.getWidth(); + mImageHeight = image.getHeight(); } /* destructor */ From 382fb9444b66e74b019e61c7734591650c93c08b Mon Sep 17 00:00:00 2001 From: romanbylina Date: Fri, 26 Aug 2022 08:31:01 +0300 Subject: [PATCH 8/8] review changes --- app/src/main/cpp/native-lib.cpp | 49 ++++--- .../banuba/quickstart_c_api/MainActivity.java | 34 +---- .../OffscreenEffectPlayerImage.java | 135 ++++++++++++------ 3 files changed, 123 insertions(+), 95 deletions(-) diff --git a/app/src/main/cpp/native-lib.cpp b/app/src/main/cpp/native-lib.cpp index 23f45cd..a2671e3 100644 --- a/app/src/main/cpp/native-lib.cpp +++ b/app/src/main/cpp/native-lib.cpp @@ -205,14 +205,19 @@ namespace jobject get_image_from_pixel_buffer(pixel_buffer_sptr image, bnb::oep::interfaces::image_format image_format, JNIEnv* env, jclass image_class) { - jbyteArray byte_array0 = nullptr; - jbyteArray byte_array1 = nullptr; - jbyteArray byte_array2 = nullptr; switch(image_format) { case bnb::oep::interfaces::image_format::bpc8_rgba: { auto size = image->get_width() * image->get_height() * image->get_bytes_per_pixel(); - byte_array0 = env->NewByteArray(size); + jbyteArray byte_array0 = env->NewByteArray(size); env->SetByteArrayRegion(byte_array0, 0, size, reinterpret_cast((uint8_t*) image->get_base_sptr().get())); + if(byte_array0 == nullptr) { + print_message("GetEnv: error in getting data"); + return nullptr; + } + auto image_constructor_id = env->GetMethodID(image_class, "", "([BII)V"); + return env->NewObject( + image_class, image_constructor_id, byte_array0, + image->get_width(), image->get_height()); break; } case bnb::oep::interfaces::image_format::nv12_bt601_full: { @@ -221,10 +226,18 @@ namespace void* buf0 = reinterpret_cast((void*) image->get_base_sptr_of_plane(0).get()); void* buf1 = reinterpret_cast((void*) image->get_base_sptr_of_plane(1).get()); - byte_array0 = env->NewByteArray(size0); - byte_array1 = env->NewByteArray(size1); + jbyteArray byte_array0 = env->NewByteArray(size0); + jbyteArray byte_array1 = env->NewByteArray(size1); env->SetByteArrayRegion(byte_array0, 0, size0, reinterpret_cast(buf0)); env->SetByteArrayRegion(byte_array1, 0, size1, reinterpret_cast(buf1)); + if(byte_array0 == nullptr || byte_array1 == nullptr) { + print_message("GetEnv: error in getting data"); + return nullptr; + } + auto image_constructor_id = env->GetMethodID(image_class, "", "([B[BII)V"); + return env->NewObject( + image_class, image_constructor_id, byte_array0, byte_array1, + image->get_width(), image->get_height()); break; } case bnb::oep::interfaces::image_format::i420_bt601_full: { @@ -235,9 +248,9 @@ namespace void* buf1 = reinterpret_cast((void*) image->get_base_sptr_of_plane(1).get()); void* buf2 = reinterpret_cast((void*) image->get_base_sptr_of_plane(2).get()); - byte_array0 = env->NewByteArray(size0); - byte_array1 = env->NewByteArray(size1); - byte_array2 = env->NewByteArray(size2); + jbyteArray byte_array0 = env->NewByteArray(size0); + jbyteArray byte_array1 = env->NewByteArray(size1); + jbyteArray byte_array2 = env->NewByteArray(size2); env->SetByteArrayRegion(byte_array0, 0, size0, reinterpret_cast(buf0)); int u_width = image->get_width_of_plane(1); int u_stride = image->get_bytes_per_row_of_plane(1); @@ -252,20 +265,20 @@ namespace for(int i = 0; i < image->get_height_of_plane(2); ++i) { env->SetByteArrayRegion(byte_array2, v_width * i, v_width, reinterpret_cast(buf2) + v_stride * i); } + if(byte_array0 == nullptr && byte_array1 == nullptr && byte_array2 == nullptr) { + print_message("GetEnv: error in getting data"); + return nullptr; + } + auto image_constructor_id = env->GetMethodID(image_class, "", "([B[B[BII)V"); + return env->NewObject( + image_class, image_constructor_id, byte_array0, byte_array1, byte_array2, + image->get_width(), image->get_height()); break; } default: break; } - - if(byte_array0 == nullptr && byte_array1 == nullptr && byte_array2 == nullptr) { - print_message("GetEnv: unsupported output image format"); - return nullptr; - } - auto image_constructor_id = env->GetMethodID(image_class, "", "([B[B[BII)V"); - return env->NewObject( - image_class, image_constructor_id, byte_array0, byte_array1, byte_array2, - image->get_width(), image->get_height()); + return nullptr; } void draw_image_from_pixel_buffer(pixel_buffer_sptr pixel_buffer, diff --git a/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java b/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java index f507d89..0dc173b 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java @@ -38,10 +38,9 @@ public class MainActivity extends AppCompatActivity { private OffscreenEffectPlayer oep = null; private GLSurfaceView glView = null; private GLRenderer renderer = null; - private OffscreenEffectPlayerImage mOffscreenEffectPlayerImage = null; // Changing mImageOutputFormat will cause the format's changing (input and output image of OEP) - private ImageFormat mImageFormat = ImageFormat.i420; + private ImageFormat mImageFormat = ImageFormat.NV12; void createRenderer() { switch (mImageFormat) { @@ -91,7 +90,6 @@ protected void onCreate(@Nullable Bundle savedInstanceState) { /* Create offscreen effect player */ createOEP(); - mOffscreenEffectPlayerImage = new OffscreenEffectPlayerImage(); requestCameraPermissionAndStart(); } @@ -151,14 +149,11 @@ private void startCamera() { imageAnalysis.setAnalyzer( ContextCompat.getMainExecutor(MainActivity.this), proxy -> { - updateImage(proxy); - int rotation = getRotation(MainActivity.this); int imageFormat = mImageFormat.ordinal(); - - oep.processImageAsync(mOffscreenEffectPlayerImage, getInputOrientation(rotation), + OffscreenEffectPlayerImage image = new OffscreenEffectPlayerImage(proxy); + oep.processImageAsync(image, getInputOrientation(rotation), false, getOutputOrientation(rotation), imageFormat); - proxy.close(); }); cameraProvider.bindToLifecycle(MainActivity.this, cameraSelector, imageAnalysis); @@ -194,29 +189,6 @@ private int getOutputOrientation(int rotation) { } return 0; } - - private void updateImagePlane(ImageProxy imageProxy, int planeNumber) { - ImageProxy.PlaneProxy planeProxy = imageProxy.getPlanes()[planeNumber]; - mOffscreenEffectPlayerImage.setPlane( - planeNumber, - planeProxy.getRowStride(), - planeProxy.getBuffer(), - planeProxy.getPixelStride()); - } - - private void updateImage(ImageProxy imageProxy) { - mOffscreenEffectPlayerImage.setWidth(imageProxy.getWidth()); - mOffscreenEffectPlayerImage.setHeight(imageProxy.getHeight()); - mOffscreenEffectPlayerImage.setPixelFormat(imageProxy.getImage().getFormat()); - updateImagePlane(imageProxy, 0); - int planesNumber = imageProxy.getPlanes().length; - if (planesNumber > 1) { - updateImagePlane(imageProxy, 1); - if (planesNumber > 2) { - updateImagePlane(imageProxy, 2); - } - } - } } enum ImageFormat { diff --git a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java index 367f828..4718ead 100644 --- a/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java +++ b/app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java @@ -1,52 +1,95 @@ package com.banuba.quickstart_c_api; +import androidx.camera.core.ImageProxy; + import java.nio.ByteBuffer; public class OffscreenEffectPlayerImage { - OffscreenEffectPlayerImage() {} + OffscreenEffectPlayerImage(ImageProxy imageProxy) { + mWidth = imageProxy.getWidth(); + mHeight = imageProxy.getHeight(); + mPixelFormat = imageProxy.getImage().getFormat(); - OffscreenEffectPlayerImage(byte[] yPlane, byte[] uPlane, byte[] vPlane, int width, int height) { - if(yPlane != null) { - mPlane0 = ByteBuffer.wrap(yPlane); - } - if(uPlane != null) { - mPlane1 = ByteBuffer.wrap(uPlane); - } - if(vPlane != null) { - mPlane2 = ByteBuffer.wrap(vPlane); + ImageProxy.PlaneProxy planeProxy = imageProxy.getPlanes()[0]; + mRowStride0 = planeProxy.getRowStride(); + mPlane0 = planeProxy.getBuffer(); + mPixelStride0 = planeProxy.getPixelStride(); + int planesNumber = imageProxy.getPlanes().length; + if (planesNumber > 1) { + planeProxy = imageProxy.getPlanes()[1]; + mRowStride1 = planeProxy.getRowStride(); + mPlane1 = planeProxy.getBuffer(); + mPixelStride1 = planeProxy.getPixelStride(); + if (planesNumber > 2) { + planeProxy = imageProxy.getPlanes()[2]; + mRowStride2 = planeProxy.getRowStride(); + mPlane2 = planeProxy.getBuffer(); + mPixelStride2 = planeProxy.getPixelStride(); + } else { + mRowStride2 = 0; + mPlane2 = null; + mPixelStride2 = 0; + } + } else { + mRowStride1 = 0; + mPlane1 = null; + mPixelStride1 = 0; + mRowStride2 = 0; + mPlane2 = null; + mPixelStride2 = 0; } + } + OffscreenEffectPlayerImage(byte[] rgbPlane, int width, int height) { + assert rgbPlane != null; + mPlane0 = ByteBuffer.wrap(rgbPlane); + mPlane1 = null; + mPlane2 = null; mWidth = width; mHeight = height; - } - void setWidth(int width) { mWidth = width; } + mRowStride0 = 0; + mRowStride1 = 0; + mRowStride2 = 0; + mPixelStride0 = 0; + mPixelStride1 = 0; + mPixelStride2 = 0; + mPixelFormat = 0; + } - void setHeight(int height) { mHeight = height; } + OffscreenEffectPlayerImage(byte[] yPlane, byte[] uvPlane, int width, int height) { + assert yPlane != null && uvPlane != null; + mPlane0 = ByteBuffer.wrap(yPlane); + mPlane1 = ByteBuffer.wrap(uvPlane); + mPlane2 = null; + mWidth = width; + mHeight = height; - void setPixelFormat(int pixelFormat) { - mPixelFormat = pixelFormat; + mRowStride0 = 0; + mRowStride1 = 0; + mRowStride2 = 0; + mPixelStride0 = 0; + mPixelStride1 = 0; + mPixelStride2 = 0; + mPixelFormat = 0; } - void setPlane(int planeNumber, int planeWidth, ByteBuffer plane, int pixelStride) { - switch (planeNumber) { - case 1: - mRowStride1 = planeWidth; - mPlane1 = plane; - mPixelStride1 = pixelStride; - break; - case 2: - mRowStride2 = planeWidth; - mPlane2 = plane; - mPixelStride2 = pixelStride; - break; - default: - mRowStride0 = planeWidth; - mPlane0 = plane; - mPixelStride0 = pixelStride; - break; - } + OffscreenEffectPlayerImage(byte[] yPlane, byte[] uPlane, byte[] vPlane, int width, int height) { + assert yPlane != null && uPlane != null && vPlane != null; + mPlane0 = ByteBuffer.wrap(yPlane); + mPlane1 = ByteBuffer.wrap(uPlane); + mPlane2 = ByteBuffer.wrap(vPlane); + mWidth = width; + mHeight = height; + + mRowStride0 = 0; + mRowStride1 = 0; + mRowStride2 = 0; + mPixelStride0 = 0; + mPixelStride1 = 0; + mPixelStride2 = 0; + mPixelFormat = 0; } public ByteBuffer getPlane(int planeNumber) { @@ -66,17 +109,17 @@ public int getHeight() { return mHeight; } - private ByteBuffer mPlane0; - private ByteBuffer mPlane1; - private ByteBuffer mPlane2; - - private int mWidth; - private int mHeight; - private int mRowStride0; - private int mRowStride1; - private int mRowStride2; - private int mPixelStride0; - private int mPixelStride1; - private int mPixelStride2; - private int mPixelFormat; + final private ByteBuffer mPlane0; + final private ByteBuffer mPlane1; + final private ByteBuffer mPlane2; + + final private int mWidth; + final private int mHeight; + final private int mRowStride0; + final private int mRowStride1; + final private int mRowStride2; + final private int mPixelStride0; + final private int mPixelStride1; + final private int mPixelStride2; + final private int mPixelFormat; }