added cropping of capture/background

app/background.cc

@@ -9,15 +9,28 @@
 #include <opencv2/imgproc.hpp>
 #include <opencv2/highgui.hpp>
 
+#include <lib/libbackscrub.h>
+
+#if CV_VERSION_MAJOR < 4 ||\
+    CV_VERSION_MAJOR == 4 && CV_VERSION_MINOR < 5 ||\
+    CV_VERSION_MAJOR == 4 && CV_VERSION_MINOR == 5 && CV_VERSION_REVISION < 3
+# define HAVE_IMCOUNT 0
+#else
+# define HAVE_IMCOUNT 1
+#endif
+
 // Internal state of background processing
 struct background_t {
     int debug;
     bool video;
     volatile bool run;
     cv::VideoCapture cap;
     int frame;
+    int cnt;
     double fps;
+    cv::Rect cropRegion;
     cv::Mat raw;
+    bool bg_stored;
     std::mutex rawmux;
     cv::Mat thumb;
     std::mutex thumbmux;
@@ -58,6 +71,11 @@ static void read_thread(std::weak_ptr<background_t> weak) {
                 grab.copyTo(pbkd->raw);
                 pbkd->frame += 1;
             }
+            // if last frame reached (gstreamer and GIF file) set to frame 0
+            if (pbkd->frame == pbkd->cnt) {
+                pbkd->cap.set(cv::CAP_PROP_POS_FRAMES, 0);
+                pbkd->frame = 0;
+            }
             // grab timing point
             auto now = std::chrono::steady_clock::now();
             // generate thumbnail frame with overlay info if double debug enabled
@@ -130,6 +148,14 @@ std::shared_ptr<background_t> load_background(const std::string& path, int debug
         pbkd->debug = debug;
         pbkd->video = false;
         pbkd->run = false;
+        pbkd->bg_stored = false;
+        pbkd->cropRegion = {-1, -1, -1, -1};
+
+#if HAVE_IMCOUNT
+        size_t images = cv::imcount(path.c_str());
+#else
+        size_t images = 0;
+#endif
         pbkd->cap.open(path, cv::CAP_ANY);    // explicitly ask for auto-detection of backend
         if (!pbkd->cap.isOpened()) {
             if (pbkd->debug) fprintf(stderr, "background: cap cannot open: %s\n", path.c_str());
@@ -139,23 +165,39 @@ std::shared_ptr<background_t> load_background(const std::string& path, int debug
         int fcc = (int)pbkd->cap.get(cv::CAP_PROP_FOURCC);
         pbkd->fps = pbkd->cap.get(cv::CAP_PROP_FPS);
         int cnt = (int)pbkd->cap.get(cv::CAP_PROP_FRAME_COUNT);
+        pbkd->cnt = cnt;
+#if !HAVE_IMCOUNT
+        if ( cnt < 0 )
+             images = 1;
+#endif
         // Here be the logic...
-        //  if: can read 2 video frames => it's a video
-        //  else: is loaded as an image => it's an image
+        //  if:  count > 1 it's a video
+        //  else: if cnt < 1 it´s is a still image
+        //  else: if cnt = 1 it´s is a static gif file
         //  else: it's not usable.
-        if (pbkd->cap.read(pbkd->raw) && pbkd->cap.read(pbkd->raw)) {
+        if (cnt > 1) {
+            if (!pbkd->cap.read(pbkd->raw)) {
+                if (pbkd->debug) fprintf(stderr, "background: read failed %s\n", path.c_str());
+                return nullptr;
+            }
             // it's a video, try a reset and start reader thread..
             if (pbkd->cap.set(cv::CAP_PROP_POS_FRAMES, 0))
                 pbkd->frame = 0;
             else
-                pbkd->frame = 2;    // unable to reset, so we're 2 frames in
+                pbkd->frame = 1;    // unable to reset, so we're 1 frames in
             pbkd->video = true;
             pbkd->run = true;
             pbkd->thread = std::thread(read_thread, std::weak_ptr<background_t>(pbkd));
         } else {
             // static image file, try loading..
-            pbkd->cap.release();
-            pbkd->raw = cv::imread(path);
+            if (cnt < 0) {
+                pbkd->cap.release();
+                pbkd->raw = cv::imread(path);
+            } else if (cnt == 1) {
+                // this is a static gif file with 1 frame
+                pbkd->cap.read(pbkd->raw);
+                pbkd->cap.release();
+            }
             if (pbkd->raw.empty()) {
                 if (pbkd->debug) fprintf(stderr, "background: imread cannot open: %s\n", path.c_str());
                 return nullptr;
@@ -179,15 +221,33 @@ int grab_background(std::shared_ptr<background_t> pbkd, int width, int height, c
     if (!pbkd)
         return -1;
     // static image or video?
-    int frm ;
+    int frm;
     if (pbkd->video) {
         // grab frame & frame no. under mutex
         std::unique_lock<std::mutex> hold(pbkd->rawmux);
-        cv::resize(pbkd->raw, out, cv::Size(width, height));
+        if ( pbkd->cropRegion.x < 0 ) { // calculate cropping if not already done
+            pbkd->cropRegion = bs_calc_cropping(pbkd->raw.cols, pbkd->raw.rows, width, height);
+        }
+        // Perform cropping if necessary
+        if ( pbkd->cropRegion.x || pbkd->cropRegion.y ) {
+            cv::Mat tmp;
+            pbkd->raw(pbkd->cropRegion).copyTo(tmp);
+            cv::resize(tmp, out, cv::Size(width, height));
+        } else {
+            cv::resize(pbkd->raw, out, cv::Size(width, height));
+        }
         frm = pbkd->frame;
     } else {
-        // resize still image as requested into out
-        cv::resize(pbkd->raw, out, cv::Size(width, height));
+        if (!pbkd->bg_stored) {
+            // resize still image as requested into out
+            cv::Rect crop = bs_calc_cropping(pbkd->raw.cols, pbkd->raw.rows, width, height);
+            // Under some circumstances we must do the job in two steps!
+            // Otherwise this resize(pbkd->raw(crop), pbkd->raw, ...) may fail.
+            pbkd->raw(crop).copyTo(pbkd->raw);
+            cv::resize(pbkd->raw, pbkd->raw, cv::Size(width, height));
+            pbkd->bg_stored = true;
+        }
+        out = pbkd->raw;
         frm = 1;
     }
     return frm;

app/deepseg.cc

@@ -84,7 +84,7 @@ std::optional<std::pair<size_t, size_t>> geometryFromString(const std::string& i
 }
 
 // OpenCV helper functions
-cv::Mat convert_rgb_to_yuyv( cv::Mat input ) {
+cv::Mat convert_rgb_to_yuyv(cv::Mat input) {
 	cv::Mat tmp;
 	cv::cvtColor(input, tmp, cv::COLOR_RGB2YUV);
 	std::vector<cv::Mat> yuv;
@@ -372,6 +372,7 @@ int main(int argc, char* argv[]) try {
 	bool flipVertical = false;
 	int fourcc = 0;
 	size_t blur_strength = 0;
+	cv::Rect crop_region(0, 0, 0, 0);
 
 	const char* modelname = "selfiesegmentation_mlkit-256x256-2021_01_19-v1215.f16.tflite";
 
@@ -568,6 +569,12 @@ int main(int argc, char* argv[]) try {
 	if (expWidth != vidGeo.value().first) {
 		fprintf(stderr, "Warning: virtual camera aspect ratio does not match capture device.\n");
 	}
+	// calculate crop region, only if result always smaller
+	if (capGeo != vidGeo) {
+		crop_region = bs_calc_cropping(
+		               capGeo->first, capGeo->second,
+		               vidGeo->first, vidGeo->second);
+	}
 
 	// dump settings..
 	printf("debug:  %d\n", debug);
@@ -600,7 +607,11 @@ int main(int argc, char* argv[]) try {
 		}
 	}
 	// default green screen background (at capture true geometry)
-	cv::Mat bg = cv::Mat(capGeo.value().second, capGeo.value().first, CV_8UC3, cv::Scalar(0, 255, 0));
+	std::pair<size_t, size_t> bg_dim = *capGeo;
+	if (crop_region.x || crop_region.y) {
+		bg_dim = {crop_region.width, crop_region.height};
+	}
+	cv::Mat bg(bg_dim.second, bg_dim.first, CV_8UC3, cv::Scalar(0, 255, 0));
 
 	// Virtual camera (at specified geometry)
 	int lbfd = loopback_init(s_vcam, vidGeo.value().first, vidGeo.value().second, debug);
@@ -613,11 +624,24 @@ int main(int argc, char* argv[]) try {
 		loopback_free(lbfd);
 	});
 
-
 	// Processing components, all at capture true geometry
-	cv::Mat mask(capGeo.value().second, capGeo.value().first, CV_8U);
+	std::pair<size_t, size_t> mask_dim = *capGeo;
+	if (crop_region.x || crop_region.y) {
+		mask_dim = {crop_region.width, crop_region.height};
+	}
+	cv::Mat mask(mask_dim.second, mask_dim.first, CV_8U);
+
 	cv::Mat raw;
-	CalcMask ai(s_model.value(), threads, capGeo.value().first, capGeo.value().second);
+	int aiw,aih;
+	if (!(crop_region.x || crop_region.y)) {
+		aiw=capGeo->first;
+		aih=capGeo->second;
+	} else {
+		aiw=crop_region.width;
+		aih=crop_region.height;
+	}
+	CalcMask ai(*s_model, threads, aiw, aih);
+
 	ti.lastns = timestamp();
 	printf("Startup: %ldns\n", diffnanosecs(ti.lastns,ti.bootns));
 
@@ -631,22 +655,35 @@ int main(int argc, char* argv[]) try {
 		// copy new frame to buffer
 		cap.retrieve(raw);
 		ti.retrns = timestamp();
+
+		if (raw.rows == 0 || raw.cols == 0) continue; // sanity check
+
+		if (crop_region.x || crop_region.y) {
+			raw(crop_region).copyTo(raw);
+		}
 		ai.set_input_frame(raw);
 		ti.copyns = timestamp();
 
-		if (raw.rows == 0 || raw.cols == 0) continue; // sanity check
+		// do background detection magic
+		ai.get_output_mask(mask);
+		ti.copyns = timestamp();
 
 		if (filterActive) {
-			// do background detection magic
-			ai.get_output_mask(mask);
-
 			// get background frame:
 			// - specified source if set
 			// - copy of input video if blur_strength != 0
 			// - default green (initial value)
 			bool canBlur = false;
 			if (pbk) {
-				if (grab_background(pbk, capGeo.value().first, capGeo.value().second, bg)<0)
+				int tw,th;
+				if (crop_region.x || crop_region.y) {
+					tw = crop_region.width;
+					th = crop_region.height;
+				} else {
+					tw = capGeo->first;
+					th = capGeo->second;
+				}
+				if (grab_background(pbk, tw, th, bg) < 0)
 					throw "Failed to read background frame";
 				canBlur = true;
 			} else if (blur_strength) {

lib/libbackscrub.cc

@@ -365,7 +365,11 @@ bool bs_maskgen_process(void *context, cv::Mat &frame, cv::Mat &mask) {
 
 	// scale up into full-sized mask
 	cv::Mat tmpbuf;
-	cv::resize(ctx.ofinal(ctx.in_roidim),tmpbuf,ctx.mroi.size());
+	// with body-pix-float-050-8.tflite the size of ctx.ofinal is 33x33
+	// and the wanted roi may be greater as 33x33 so we can crash with
+	// cv::resize(ctx.ofinal(ctx.in_roidim),tmpbuf,ctx.mroi.size());
+	ctx.ofinal.copyTo(tmpbuf);
+	cv::resize(tmpbuf, tmpbuf, ctx.mroi.size());
 
 	// blur at full size for maximum smoothness
 	cv::blur(tmpbuf,ctx.mroi,ctx.blur);
@@ -375,3 +379,24 @@ bool bs_maskgen_process(void *context, cv::Mat &frame, cv::Mat &mask) {
 	return true;
 }
 
+cv::Rect bs_calc_cropping(int inWidth, int inHeight, int targetWidth, int targetHigh) {
+	// if the input and output aspect ratio are not the same
+	// we can crop the source image. For example if the
+	// input image has a 16:9 (1280x720) ratio and the output is 4:3 (960x720)
+	// we will return the cropRegion set as x=160, width=960, y=0, height=720
+	// which is the centered part of the original image
+	cv::Rect cropRegion = {0, 0, 0, 0};
+	float sc = (float)targetWidth / inWidth;
+	float st = (float)targetHigh / inHeight;
+	sc = st > sc ? st : sc;
+
+	int sx = (int)(targetWidth / sc) - inWidth;
+	cropRegion.x =  (sx < 0 ? -sx : sx) / 2;
+
+	int sy = (int)(targetHigh / sc) - inHeight;
+	cropRegion.y =  (sy < 0 ? -sy : sy) / 2;
+
+	cropRegion.width = inWidth - cropRegion.x * 2;
+	cropRegion.height = inHeight - cropRegion.y * 2;
+	return cropRegion;
+}

lib/libbackscrub.h

@@ -38,4 +38,6 @@ extern void bs_maskgen_delete(void *context);
 // Process a video frame into a mask
 extern bool bs_maskgen_process(void *context, cv::Mat& frame, cv::Mat &mask);
 
+extern cv::Rect bs_calc_cropping(int inWidth, int inHeight, int targetWidth, int targetHight);
+
 #endif