Skip to content
This repository was archived by the owner on Apr 16, 2025. It is now read-only.

Refactor sample #9

Open
sdk-banuba wants to merge 8 commits into
base: master
Choose a base branch
from
Open

Refactor sample #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
1544113
fix memory leak
bn-roma Aug 4, 2022
5590cba
can disable OEP
bn-roma Aug 5, 2022
3c38bb8
return Image object from JNI
bn-roma Aug 17, 2022
604bf26
put Image object to OEP
bn-roma Aug 18, 2022
f1b7815
small fix
bn-roma Aug 18, 2022
bab5375
delete code for disabled OEP
bn-roma Aug 18, 2022
8ccc827
review changes
bn-roma Aug 24, 2022
382fb94
review changes
bn-roma Aug 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,5 @@ local.properties
.gradle
app/.cxx
app/src/main/assets/*/
build/
app/release/
326 changes: 183 additions & 143 deletions app/src/main/cpp/native-lib.cpp
View file
Open in desktop

Large diffs are not rendered by default.

46 changes: 8 additions & 38 deletions app/src/main/java/com/banuba/quickstart_c_api/MainActivity.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,6 @@
import com.google.common.util.concurrent.ListenableFuture;

import java.io.File;
import java.util.Arrays;
import java.util.List;

public class MainActivity extends AppCompatActivity {
private static int CAMERA_PERMISSION_REQUEST = 12345;
Expand All @@ -40,7 +38,6 @@ public class MainActivity extends AppCompatActivity {
private OffscreenEffectPlayer oep = null;
private GLSurfaceView glView = null;
private GLRenderer renderer = null;
private OffscreenEffectPlayerImage mImage = null;

// Changing mImageOutputFormat will cause the format's changing (input and output image of OEP)
private ImageFormat mImageFormat = ImageFormat.NV12;
Expand All @@ -61,11 +58,10 @@ void createRenderer() {

void createOEP() {
oep = new OffscreenEffectPlayer(size.getWidth(), size.getHeight());
oep.loadEffect("effects/<!!! PLACE YOUR EFFECT NAME HERE !!!>")
oep.loadEffect("effects/<!!! PLACE YOUR EFFECT NAME HERE !!!>");
oep.setDataReadyCallback(
(image0,image1, image2, width, height) -> {
List<byte[]> planes = Arrays.asList(image0, image1, image2);
renderer.drawImage(planes, width, height);
(offscreenEffectPlayerImage) -> {

@PiotrBanuba PiotrBanuba Aug 26, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the name of variable is better to shorten to 'oepImage'

renderer.drawImage(offscreenEffectPlayerImage);
glView.requestRender();
});
}
Expand Down Expand Up @@ -94,7 +90,6 @@ protected void onCreate(@Nullable Bundle savedInstanceState) {
/* Create offscreen effect player */
createOEP();

mImage = new OffscreenEffectPlayerImage();
requestCameraPermissionAndStart();
}

Expand Down Expand Up @@ -154,8 +149,11 @@ private void startCamera() {
imageAnalysis.setAnalyzer(
ContextCompat.getMainExecutor(MainActivity.this),
proxy -> {
updateImage(proxy);
oep.processImageAsync(mImage);
int rotation = getRotation(MainActivity.this);
int imageFormat = mImageFormat.ordinal();
OffscreenEffectPlayerImage image = new OffscreenEffectPlayerImage(proxy);

@PiotrBanuba PiotrBanuba Aug 26, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

for example:
OffscreenEffectPlayerImage image = wrapToOffscreenEffectPlayerImage(proxy);

oep.processImageAsync(image, getInputOrientation(rotation),
false, getOutputOrientation(rotation), imageFormat);
proxy.close();
});
cameraProvider.bindToLifecycle(MainActivity.this, cameraSelector, imageAnalysis);
Expand Down Expand Up @@ -191,34 +189,6 @@ private int getOutputOrientation(int rotation) {
}
return 0;
}

private void updateImage(ImageProxy imageProxy) {
int rotation = getRotation(MainActivity.this);

mImage.mImageInfo.width = imageProxy.getWidth();
mImage.mImageInfo.height = imageProxy.getHeight();
mImage.mImageInfo.inputOrientation = getInputOrientation(rotation);
mImage.mImageInfo.outputOrientation = getOutputOrientation(rotation);
mImage.mImageInfo.pixelFormat = imageProxy.getImage().getFormat();
mImage.mImageInfo.requireMirroring = false;
mImage.mImageInfo.imageFormat = mImageFormat.ordinal();

mImage.mImageInfo.rowStride0 = imageProxy.getPlanes()[0].getRowStride();
mImage.mImageZero = imageProxy.getPlanes()[0].getBuffer();
mImage.mImageInfo.pixelStride0 = imageProxy.getPlanes()[0].getPixelStride();

int planesNumber = imageProxy.getPlanes().length;
if (planesNumber > 1) {
mImage.mImageInfo.rowStride1 = imageProxy.getPlanes()[1].getRowStride();
mImage.mImageFirst = imageProxy.getPlanes()[1].getBuffer();
mImage.mImageInfo.pixelStride1 = imageProxy.getPlanes()[1].getPixelStride();
if (planesNumber > 2) {
mImage.mImageInfo.rowStride2 = imageProxy.getPlanes()[2].getRowStride();
mImage.mImageSecond = imageProxy.getPlanes()[2].getBuffer();
mImage.mImageInfo.pixelStride2 = imageProxy.getPlanes()[2].getPixelStride();
}
}
}
}

enum ImageFormat {
Expand Down
26 changes: 17 additions & 9 deletions app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package com.banuba.quickstart_c_api;

import java.nio.ByteBuffer;

class OffscreenEffectPlayer {
private long mOep = 0;
private DataReadyCallback mDataReadyCallback = null;
Expand Down Expand Up @@ -41,9 +39,14 @@ public void destroy() {
}
}

/* image must be NV12 format */
public void processImageAsync(OffscreenEffectPlayerImage image) {
externalProcessImageAsync(mOep, image.mImageZero, image.mImageFirst, image.mImageSecond, image.mImageInfo);
/* image must be NV12 or i420 format */
public void processImageAsync(OffscreenEffectPlayerImage image,
int inputOrientation,
boolean isRequiredMirroring,
int outputOrientation,
int imageFormat) {

@PiotrBanuba PiotrBanuba Aug 26, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What does this variable mean ? input/output ?
If this is an output image format, then it's better to name it that 'outputImageFormat'

externalProcessImageAsync(mOep, image, inputOrientation, isRequiredMirroring,
outputOrientation, imageFormat);
}

public void surfaceChanged(int width, int height) {
Expand Down Expand Up @@ -79,16 +82,16 @@ public void evalJs(String script) {
}

public interface DataReadyCallback {
void onDataReady(byte[] image0, byte[] image1, byte[] image2, int width, int height);
void onDataReady(OffscreenEffectPlayerImage image);
}

public void setDataReadyCallback(DataReadyCallback callback) {
mDataReadyCallback = callback;
}

private void onDataReady(byte[] image0, byte[] image1, byte[] image2, int width, int height) {
private void onDataReady(OffscreenEffectPlayerImage image) {
if (mDataReadyCallback != null) {
mDataReadyCallback.onDataReady(image0, image1, image2, width, height);
mDataReadyCallback.onDataReady(image);
}
}

Expand All @@ -97,7 +100,11 @@ private void onDataReady(byte[] image0, byte[] image1, byte[] image2, int width,
private static native void externalDeinit();
private native long externalCreate(int width, int height);
private native void externalDestroy(long oep);
private native void externalProcessImageAsync(long oep, ByteBuffer imageY, ByteBuffer imageU, ByteBuffer imageV, ImageInfo info);
private native void externalProcessImageAsync(long oep, OffscreenEffectPlayerImage image,
int inputOrientation,
boolean isRequiredMirroring,
int outputOrientation,
int imageFormat);

@PiotrBanuba PiotrBanuba Aug 26, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

see previous comment

private native void externalSurfaceChanged(long oep, int width, int height);
private native void externalLoadEffect(long oep, String effectPath);
private native void externalUnloadEffect(long oep);
Expand All @@ -111,3 +118,4 @@ private void onDataReady(byte[] image0, byte[] image1, byte[] image2, int width,
System.loadLibrary("native-lib");
}
}

152 changes: 116 additions & 36 deletions app/src/main/java/com/banuba/quickstart_c_api/OffscreenEffectPlayerImage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,45 +1,125 @@
package com.banuba.quickstart_c_api;

import androidx.camera.core.ImageProxy;

@PiotrBanuba PiotrBanuba Aug 26, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove it


import java.nio.ByteBuffer;

public class OffscreenEffectPlayerImage {
OffscreenEffectPlayerImage() {
mImageInfo = new ImageInfo();

OffscreenEffectPlayerImage(ImageProxy imageProxy) {

@PiotrBanuba PiotrBanuba Aug 26, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The OffscreenEffectPlayerImage should not depends on ImageProxy. Remove it

@PiotrBanuba PiotrBanuba Aug 26, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add a method that returns OffscreenEffectPlayerImage from ImageProxy, to the class in which it is used

mWidth = imageProxy.getWidth();
mHeight = imageProxy.getHeight();
mPixelFormat = imageProxy.getImage().getFormat();

ImageProxy.PlaneProxy planeProxy = imageProxy.getPlanes()[0];
mRowStride0 = planeProxy.getRowStride();
mPlane0 = planeProxy.getBuffer();
mPixelStride0 = planeProxy.getPixelStride();

@PiotrBanuba PiotrBanuba Aug 26, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the mPixelStride0 is not use

int planesNumber = imageProxy.getPlanes().length;
if (planesNumber > 1) {
planeProxy = imageProxy.getPlanes()[1];
mRowStride1 = planeProxy.getRowStride();
mPlane1 = planeProxy.getBuffer();
mPixelStride1 = planeProxy.getPixelStride();
if (planesNumber > 2) {
planeProxy = imageProxy.getPlanes()[2];
mRowStride2 = planeProxy.getRowStride();
mPlane2 = planeProxy.getBuffer();
mPixelStride2 = planeProxy.getPixelStride();
} else {
mRowStride2 = 0;
mPlane2 = null;
mPixelStride2 = 0;
}
} else {
mRowStride1 = 0;
mPlane1 = null;
mPixelStride1 = 0;
mRowStride2 = 0;
mPlane2 = null;
mPixelStride2 = 0;
}
}

public ImageInfo mImageInfo;
public ByteBuffer mImageZero = null;
public ByteBuffer mImageFirst = null;
public ByteBuffer mImageSecond = null;
}
OffscreenEffectPlayerImage(byte[] rgbPlane, int width, int height) {

@PiotrBanuba PiotrBanuba Aug 26, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OffscreenEffectPlayerImage(final byte[] rgbPlane, final int rgbPlaneStride, final int width, final int height)

assert rgbPlane != null;
mPlane0 = ByteBuffer.wrap(rgbPlane);
mPlane1 = null;
mPlane2 = null;
mWidth = width;
mHeight = height;

mRowStride0 = 0;
mRowStride1 = 0;
mRowStride2 = 0;
mPixelStride0 = 0;
mPixelStride1 = 0;
mPixelStride2 = 0;
mPixelFormat = 0;
}

OffscreenEffectPlayerImage(byte[] yPlane, byte[] uvPlane, int width, int height) {

@PiotrBanuba PiotrBanuba Aug 26, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

change to:
OffscreenEffectPlayerImage(final byte[] yPlane, final int yPlaneStride, final byte[] uvPlane, final int uvPlaneStride, final int width, final int height)

@PiotrBanuba PiotrBanuba Aug 26, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this constructor will create an Image with nv12 format

assert yPlane != null && uvPlane != null;
mPlane0 = ByteBuffer.wrap(yPlane);
mPlane1 = ByteBuffer.wrap(uvPlane);
mPlane2 = null;
mWidth = width;
mHeight = height;

mRowStride0 = 0;
mRowStride1 = 0;
mRowStride2 = 0;
mPixelStride0 = 0;
mPixelStride1 = 0;
mPixelStride2 = 0;
mPixelFormat = 0;
}

class ImageInfo {
ImageInfo() {
width = 0;
height = 0;
inputOrientation = 0;
outputOrientation = 0;
requireMirroring = false;
rowStride0 = 0;
rowStride1 = 0;
rowStride2 = 0;
pixelStride0 = 0;
pixelStride1 = 0;
pixelStride2 = 0;
pixelFormat = 0;
imageFormat = 0;
OffscreenEffectPlayerImage(byte[] yPlane, byte[] uPlane, byte[] vPlane, int width, int height) {

@PiotrBanuba PiotrBanuba Aug 26, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

change to:
OffscreenEffectPlayerImage(final byte[] yPlane, final int yPlaneStride, final byte[] uPlane, final int uPlaneStride, final byte[] vPlane, final int vPlaneStride, int width, int height)

@PiotrBanuba PiotrBanuba Aug 26, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And this constructor will create an Image with i420 format

assert yPlane != null && uPlane != null && vPlane != null;
mPlane0 = ByteBuffer.wrap(yPlane);
mPlane1 = ByteBuffer.wrap(uPlane);
mPlane2 = ByteBuffer.wrap(vPlane);
mWidth = width;
mHeight = height;

mRowStride0 = 0;
mRowStride1 = 0;
mRowStride2 = 0;
mPixelStride0 = 0;

@PiotrBanuba PiotrBanuba Aug 26, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

mPixelStride is not use. Remove it

mPixelStride1 = 0;
mPixelStride2 = 0;
mPixelFormat = 0;
}

public ByteBuffer getPlane(int planeNumber) {
switch (planeNumber) {

@PiotrBanuba PiotrBanuba Aug 26, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if (planeNumber >= 0 && planeNumber < mNumberOfPlanes ) {
swich-case for planes 0, 1, 2: ...
} else { throw new ... }

case 1:
return mPlane1;
case 2:
return mPlane2;
default:
return mPlane0;
}
}
public int width;
public int height;
public int inputOrientation;
public int outputOrientation;
public int rowStride0;
public int rowStride1;
public int rowStride2;
public int pixelStride0;
public int pixelStride1;
public int pixelStride2;
public int pixelFormat;
public int imageFormat;
public boolean requireMirroring;
}

public int getWidth() { return mWidth; }

@PiotrBanuba PiotrBanuba Aug 26, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

reformat this line


public int getHeight() {
return mHeight;
}

final private ByteBuffer mPlane0;
final private ByteBuffer mPlane1;
final private ByteBuffer mPlane2;

@PiotrBanuba PiotrBanuba Aug 26, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add the variable:
final private mNumberOfPlanes;

final private int mWidth;
final private int mHeight;
final private int mRowStride0;
final private int mRowStride1;
final private int mRowStride2;
final private int mPixelStride0;
final private int mPixelStride1;
final private int mPixelStride2;
final private int mPixelFormat;
}
16 changes: 12 additions & 4 deletions app/src/main/java/com/banuba/quickstart_c_api/rendering/GLRenderer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10;

import com.banuba.quickstart_c_api.OffscreenEffectPlayerImage;

public class GLRenderer implements GLSurfaceView.Renderer {

/* shaders */
Expand Down Expand Up @@ -178,10 +180,16 @@ public void scaleMatrix() {
mMat4[5] = yScale;
}

public void drawImage(List<byte[]> imageDataPlanes, int width, int height) {
mImageDataPlanes = imageDataPlanes;
mImageWidth = width;
mImageHeight = height;
public void drawImage(OffscreenEffectPlayerImage image) {
mImageDataPlanes.clear();
for(int i = 0; i < mTexturesCount; ++i) {
ByteBuffer plane = image.getPlane(i);
if(plane != null) {
mImageDataPlanes.add(plane.array());
}
}
mImageWidth = image.getWidth();
mImageHeight = image.getHeight();
}

/* destructor */
Expand Down