fix(macos): #3433 cursor reappears correctly after typing

cmake/compile_definitions/macos.cmake

@@ -35,6 +35,7 @@ list(APPEND SUNSHINE_EXTERNAL_LIBRARIES
         ${CORE_MEDIA_LIBRARY}
         ${CORE_VIDEO_LIBRARY}
         ${FOUNDATION_LIBRARY}
+        ${SCREEN_CAPTURE_KIT_LIBRARY}
         ${VIDEO_TOOLBOX_LIBRARY})
 
 set(APPLE_PLIST_TEMPLATE "${SUNSHINE_SOURCE_ASSETS_DIR}/macos/build/Info.plist.in")
@@ -55,6 +56,8 @@ set(PLATFORM_TARGET_FILES
         "${CMAKE_SOURCE_DIR}/src/platform/macos/nv12_zero_device.cpp"
         "${CMAKE_SOURCE_DIR}/src/platform/macos/nv12_zero_device.h"
         "${CMAKE_SOURCE_DIR}/src/platform/macos/publish.cpp"
+        "${CMAKE_SOURCE_DIR}/src/platform/macos/sc_capture.h"
+        "${CMAKE_SOURCE_DIR}/src/platform/macos/sc_capture.m"
         "${CMAKE_SOURCE_DIR}/third-party/TPCircularBuffer/TPCircularBuffer.c"
         "${CMAKE_SOURCE_DIR}/third-party/TPCircularBuffer/TPCircularBuffer.h"
         ${APPLE_PLIST_FILE})

cmake/dependencies/macos.cmake

@@ -9,6 +9,7 @@ FIND_LIBRARY(CORE_AUDIO_LIBRARY CoreAudio)
 FIND_LIBRARY(CORE_MEDIA_LIBRARY CoreMedia)
 FIND_LIBRARY(CORE_VIDEO_LIBRARY CoreVideo)
 FIND_LIBRARY(FOUNDATION_LIBRARY Foundation)
+FIND_LIBRARY(SCREEN_CAPTURE_KIT_LIBRARY ScreenCaptureKit)
 FIND_LIBRARY(VIDEO_TOOLBOX_LIBRARY VideoToolbox)
 
 if(SUNSHINE_ENABLE_TRAY)

src/platform/macos/display.mm

@@ -2,6 +2,11 @@
  * @file src/platform/macos/display.mm
  * @brief Definitions for display capture on macOS.
  */
+// standard includes
+#include <cstdint>
+#include <cstring>
+#include <string_view>
+
 // local includes
 #include "src/config.h"
 #include "src/logging.h"
@@ -10,6 +15,8 @@
 #include "src/platform/macos/av_video.h"
 #include "src/platform/macos/misc.h"
 #include "src/platform/macos/nv12_zero_device.h"
+#import "src/platform/macos/sc_capture.h"
+#include "src/utility.h"
 
 // Avoid conflict between AVFoundation and libavutil both defining AVMediaType
 #define AVMediaType AVMediaType_FFmpeg
@@ -20,6 +27,105 @@
 
 namespace platf {
   using namespace std::literals;
+  static constexpr auto SCKIT_SCREENSHOT_POLL_INTERVAL_NS = NSEC_PER_SEC / 60;
+
+  static bool process_frame(CMSampleBufferRef sampleBuffer, img_t *img) {
+    auto pixel_buffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+    if (!pixel_buffer) {
+      return false;
+    }
+
+    auto new_sample_buffer = std::make_shared<av_sample_buf_t>(sampleBuffer);
+    auto new_pixel_buffer = std::make_shared<av_pixel_buf_t>(new_sample_buffer->buf);
+
+    auto av_img = (av_img_t *) img;
+
+    auto old_data_retainer = std::make_shared<temp_retain_av_img_t>(
+      av_img->sample_buffer,
+      av_img->pixel_buffer,
+      img->data
+    );
+
+    av_img->sample_buffer = new_sample_buffer;
+    av_img->pixel_buffer = new_pixel_buffer;
+    img->data = new_pixel_buffer->data();
+
+    img->width = (int) CVPixelBufferGetWidth(new_pixel_buffer->buf);
+    img->height = (int) CVPixelBufferGetHeight(new_pixel_buffer->buf);
+    img->row_pitch = CVPixelBufferIsPlanar(new_pixel_buffer->buf) ?
+                       (int) CVPixelBufferGetBytesPerRowOfPlane(new_pixel_buffer->buf, 0) :
+                       (int) CVPixelBufferGetBytesPerRow(new_pixel_buffer->buf);
+    img->pixel_pitch = img->row_pitch / img->width;
+
+    old_data_retainer = nullptr;
+    return true;
+  }
+
+  static void clear_pixel_buffer(CVPixelBufferRef pixel_buffer) {
+    CVPixelBufferLockBaseAddress(pixel_buffer, 0);
+
+    if (CVPixelBufferIsPlanar(pixel_buffer)) {
+      for (size_t plane = 0; plane < CVPixelBufferGetPlaneCount(pixel_buffer); ++plane) {
+        auto *base = static_cast<std::uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, plane));
+        auto bytes_per_row = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, plane);
+        auto height = CVPixelBufferGetHeightOfPlane(pixel_buffer, plane);
+        std::memset(base, 0, bytes_per_row * height);
+      }
+    } else {
+      auto *base = static_cast<std::uint8_t *>(CVPixelBufferGetBaseAddress(pixel_buffer));
+      std::memset(base, 0, CVPixelBufferGetBytesPerRow(pixel_buffer) * CVPixelBufferGetHeight(pixel_buffer));
+    }
+
+    CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
+  }
+
+  static int make_dummy_img(img_t *img, int width, int height, OSType pixel_format, std::string_view backend_name) {
+    CVPixelBufferRef pixel_buffer = nullptr;
+    NSDictionary *attrs = @{
+      (NSString *) kCVPixelBufferIOSurfacePropertiesKey: @ {},
+    };
+
+    auto status = CVPixelBufferCreate(
+      kCFAllocatorDefault,
+      width,
+      height,
+      pixel_format,
+      (__bridge CFDictionaryRef) attrs,
+      &pixel_buffer
+    );
+
+    if (status != kCVReturnSuccess || !pixel_buffer) {
+      BOOST_LOG(error) << backend_name << " dummy_img: failed to create pixel buffer"sv;
+      return 1;
+    }
+
+    clear_pixel_buffer(pixel_buffer);
+
+    CMVideoFormatDescriptionRef format_desc = nullptr;
+    status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixel_buffer, &format_desc);
+    if (status != noErr || !format_desc) {
+      CVPixelBufferRelease(pixel_buffer);
+      BOOST_LOG(error) << backend_name << " dummy_img: failed to create format description"sv;
+      return 1;
+    }
+
+    CMSampleTimingInfo timing = {kCMTimeInvalid, kCMTimeInvalid, kCMTimeInvalid};
+    CMSampleBufferRef sample_buffer = nullptr;
+    status = CMSampleBufferCreateForImageBuffer(kCFAllocatorDefault, pixel_buffer, YES, nullptr, nullptr, format_desc, &timing, &sample_buffer);
+    CFRelease(format_desc);
+
+    if (status != noErr || !sample_buffer) {
+      CVPixelBufferRelease(pixel_buffer);
+      BOOST_LOG(error) << backend_name << " dummy_img: failed to create sample buffer"sv;
+      return 1;
+    }
+
+    auto ret = process_frame(sample_buffer, img) ? 0 : 1;
+    CFRelease(sample_buffer);
+    CVPixelBufferRelease(pixel_buffer);
+
+    return ret;
+  }
 
   struct av_display_t: public display_t {
     AVVideo *av_capture {};
@@ -151,32 +257,152 @@ static void setPixelFormat(void *display, OSType pixelFormat) {
     }
   };
 
+  struct sc_display_t: public display_t {
+    SCCapture *sc_capture {};
+    CGDirectDisplayID display_id {};
+
+    ~sc_display_t() override {
+      [sc_capture stopCapture];
+      [sc_capture release];
+    }
+
+    capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) override {
+      auto signal = [sc_capture captureVideo];
+      if (!signal) {
+        BOOST_LOG(error) << "SCCapture failed to start video capture"sv;
+        return capture_e::error;
+      }
+
+      auto frame_signal = sc_capture.frameSignal;
+
+      while (true) {
+        auto frame_status = dispatch_semaphore_wait(frame_signal, dispatch_time(DISPATCH_TIME_NOW, SCKIT_SCREENSHOT_POLL_INTERVAL_NS));
+        if (dispatch_semaphore_wait(signal, DISPATCH_TIME_NOW) == 0) {
+          break;
+        }
+
+        CMSampleBufferRef sampleBuffer = nullptr;
+        if (frame_status == 0) {
+          sampleBuffer = [sc_capture copyLatestSampleBuffer];
+        } else {
+          sampleBuffer = [sc_capture copyScreenshotSampleBuffer];
+        }
+
+        if (!sampleBuffer) {
+          std::shared_ptr<img_t> probe_img;
+          if (!pull_free_image_cb(probe_img)) {
+            [sc_capture stopCapture];
+            break;
+          }
+          continue;
+        }
+
+        auto release_sample_buffer = util::fail_guard([sampleBuffer]() {
+          CFRelease(sampleBuffer);
+        });
+
+        std::shared_ptr<img_t> img_out;
+        if (!pull_free_image_cb(img_out)) {
+          [sc_capture stopCapture];
+          break;
+        }
+
+        if (!process_frame(sampleBuffer, img_out.get())) {
+          continue;
+        }
+
+        if (!push_captured_image_cb(std::move(img_out), true)) {
+          [sc_capture stopCapture];
+          break;
+        }
+      }
+
+      return capture_e::ok;
+    }
+
+    std::shared_ptr<img_t> alloc_img() override {
+      return std::make_shared<av_img_t>();
+    }
+
+    std::unique_ptr<avcodec_encode_device_t> make_avcodec_encode_device(pix_fmt_e pix_fmt) override {
+      if (pix_fmt == pix_fmt_e::yuv420p) {
+        sc_capture.pixelFormat = kCVPixelFormatType_32BGRA;
+
+        return std::make_unique<avcodec_encode_device_t>();
+      } else if (pix_fmt == pix_fmt_e::nv12 || pix_fmt == pix_fmt_e::p010) {
+        auto device = std::make_unique<nv12_zero_device>();
+
+        device->init(static_cast<void *>(sc_capture), pix_fmt, setResolution, setPixelFormat);
+
+        return device;
+      } else {
+        BOOST_LOG(error) << "Unsupported Pixel Format."sv;
+        return nullptr;
+      }
+    }
+
+    int dummy_img(img_t *img) override {
+      if (!platf::is_screen_capture_allowed()) {
+        return 1;
+      }
+
+      return make_dummy_img(img, sc_capture.frameWidth, sc_capture.frameHeight, sc_capture.pixelFormat, "SCCapture"sv);
+    }
+
+    static void setResolution(void *display, int width, int height) {
+      [static_cast<SCCapture *>(display) setFrameWidth:width frameHeight:height];
+    }
+
+    static void setPixelFormat(void *display, OSType pixelFormat) {
+      static_cast<SCCapture *>(display).pixelFormat = pixelFormat;
+    }
+  };
+
   std::shared_ptr<display_t> display(platf::mem_type_e hwdevice_type, const std::string &display_name, const video::config_t &config) {
     if (hwdevice_type != platf::mem_type_e::system && hwdevice_type != platf::mem_type_e::videotoolbox) {
       BOOST_LOG(error) << "Could not initialize display with the given hw device type."sv;
       return nullptr;
     }
 
-    auto display = std::make_shared<av_display_t>();
-
     // Default to main display
-    display->display_id = CGMainDisplayID();
+    auto display_id = CGMainDisplayID();
 
     // Print all displays available with it's name and id
     auto display_array = [AVVideo displayNames];
     BOOST_LOG(info) << "Detecting displays"sv;
     for (NSDictionary *item in display_array) {
-      NSNumber *display_id = item[@"id"];
+      NSNumber *item_display_id = item[@"id"];
       // We need show display's product name and corresponding display number given by user
       NSString *name = item[@"displayName"];
       // We are using CGGetActiveDisplayList that only returns active displays so hardcoded connected value in log to true
-      BOOST_LOG(info) << "Detected display: "sv << name.UTF8String << " (id: "sv << [NSString stringWithFormat:@"%@", display_id].UTF8String << ") connected: true"sv;
-      if (!display_name.empty() && std::atoi(display_name.c_str()) == [display_id unsignedIntValue]) {
-        display->display_id = [display_id unsignedIntValue];
+      BOOST_LOG(info) << "Detected display: "sv << name.UTF8String << " (id: "sv << [NSString stringWithFormat:@"%@", item_display_id].UTF8String << ") connected: true"sv;
+      if (!display_name.empty() && std::atoi(display_name.c_str()) == [item_display_id unsignedIntValue]) {
+        display_id = [item_display_id unsignedIntValue];
+      }
+    }
+    BOOST_LOG(info) << "Configuring selected display ("sv << display_id << ") to stream"sv;
+
+    if (@available(macOS 12.3, *)) {
+      if ([SCCapture isAvailable]) {
+        auto display = std::make_shared<sc_display_t>();
+        display->display_id = display_id;
+        display->sc_capture = [[SCCapture alloc] initWithDisplay:display_id frameRate:config.framerate];
+
+        if (display->sc_capture) {
+          display->width = display->sc_capture.frameWidth;
+          display->height = display->sc_capture.frameHeight;
+          display->env_width = display->width;
+          display->env_height = display->height;
+
+          return display;
+        }
+
+        BOOST_LOG(error) << "SCCapture setup failed, trying AVFoundation..."sv;
       }
     }
-    BOOST_LOG(info) << "Configuring selected display ("sv << display->display_id << ") to stream"sv;
 
+    auto display = std::make_shared<av_display_t>();
+    display->display_id = display_id;
     display->av_capture = [[AVVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate];
 
     if (!display->av_capture) {

src/platform/macos/nv12_zero_device.cpp

@@ -29,6 +29,10 @@ namespace platf {
   int nv12_zero_device::convert(platf::img_t &img) {
     auto *av_img = (av_img_t *) &img;
 
+    if (!av_img->pixel_buffer || !av_img->pixel_buffer->buf) {
+      return -1;
+    }
+
     // Release any existing CVPixelBuffer previously retained for encoding
     av_buffer_unref(&av_frame->buf[0]);
 

src/platform/macos/sc_capture.h

@@ -0,0 +1,47 @@
+/**
+ * @file src/platform/macos/sc_capture.h
+ * @brief Declarations for ScreenCaptureKit-based display capture on macOS.
+ */
+#pragma once
+
+#import <AppKit/AppKit.h>
+#import <CoreMedia/CoreMedia.h>
+#import <CoreVideo/CoreVideo.h>
+#import <ScreenCaptureKit/ScreenCaptureKit.h>
+
+API_AVAILABLE(macos(12.3))
+@interface SCCapture: NSObject <SCStreamDelegate, SCStreamOutput>
+
+#define kMaxDisplays 32
+
+@property (nonatomic, assign) CGDirectDisplayID displayID;
+@property (nonatomic, assign) int frameRate;
+@property (nonatomic, assign) OSType pixelFormat;
+@property (nonatomic, assign) int frameWidth;
+@property (nonatomic, assign) int frameHeight;
+
+@property (nonatomic, strong) SCStream *stream;
+@property (nonatomic, strong) SCContentFilter *contentFilter;
+@property (nonatomic, strong) SCStreamConfiguration *streamConfiguration;
+@property (nonatomic, strong) SCShareableContent *shareableContent;
+@property (nonatomic, strong) dispatch_queue_t videoQueue;
+
+@property (nonatomic, strong) dispatch_semaphore_t captureSignal;
+@property (nonatomic, strong) dispatch_semaphore_t frameSignal;
+@property (nonatomic, assign) BOOL stopping;
+@property (nonatomic, assign) CMSampleBufferRef latestSampleBuffer;
+
++ (BOOL)isAvailable;
++ (NSArray<NSDictionary *> *)displayNames;
++ (NSString *)getDisplayName:(CGDirectDisplayID)displayID;
+
+- (instancetype)initWithDisplay:(CGDirectDisplayID)displayID
+                      frameRate:(int)frameRate;
+
+- (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight;
+- (dispatch_semaphore_t)captureVideo;
+- (CMSampleBufferRef)copyLatestSampleBuffer;
+- (CMSampleBufferRef)copyScreenshotSampleBuffer;
+- (void)stopCapture;
+
+@end