From 8fda51d4fb2f66a21091db2b00323353038b4f87 Mon Sep 17 00:00:00 2001
From: Fabian Mastenbroek <mail.fabianm@gmail.com>
Date: Wed, 22 Jun 2022 10:37:19 +0200
Subject: [PATCH] mac-virtualcam: Fix IOSurface memory leak

This change fixes a memory leak in the mac-virtualcam plugin that causes
OBS to not release the CVPixelBuffers (and underlying IOSurfaces)
it emits to the virtual camera consumers.

Pull request https://github.com/obsproject/obs-studio/pull/6573 (Avoid
transcoding where possible) updated the mac-virtualcam to share the
virtual camera feed with other processes via IOSurfaces.

Although the changes work correctly, users have observed that OBS memory
usage keeps increasing when the virtual camera is active until OBS runs
out of memory or the consuming application is closed.
See the report by @SciTechNick for more information:
https://github.com/obsproject/obs-studio/pull/6573#issuecomment-1161979765

After some debugging, I have found that the plugin is leaking Mach ports
associated with IOSurfaces, preventing them from being re-used. The
previous approach using `NSMachPort` does not seem to properly release
the Mach port allocated via `CVPixelBufferGetIOSurface` and
`IOSurfaceLookupFromMachPort`. Instead, we must explicitly deallocate
the port using `mach_port_deallocate`.

I have tested the changes on a Macbook Pro (M1) running macOS Monterey with
Google Chrome, Zoom, and Cameo. OBS shows no signs of memory leakage
after multiple minutes.
---
 .../src/dal-plugin/OBSDALMachClient.mm        | 19 ++++++++++++++--
 .../src/dal-plugin/OBSDALStream.mm            |  3 +++
 .../src/obs-plugin/OBSDALMachServer.mm        | 22 ++++++++++++++-----
 3 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.mm b/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.mm
index 452ee6fbc35ec7..632e0743380dcc 100644
--- a/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.mm
+++ b/plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.mm
@@ -101,14 +101,30 @@ - (void)handlePortMessage:(NSPortMessage *)message
 		break;
 	case MachMsgIdFrame:
 		VLog(@"Received frame message");
-		if (components.count >= 4) {
+
+		if (components.count < 4)
+			return;
+
+		@autoreleasepool {
 			NSMachPort *framePort = (NSMachPort *)components[0];
+
+			if (!framePort)
+				return;
+
 			IOSurfaceRef surface = IOSurfaceLookupFromMachPort(
 				[framePort machPort]);
+			mach_port_deallocate(mach_task_self(),
+					     [framePort machPort]);
+
+			if (!surface) {
+				ELog(@"Failed to obtain IOSurface from Mach port");
+				return;
+			}
 
 			CVPixelBufferRef frame;
 			CVPixelBufferCreateWithIOSurface(kCFAllocatorDefault,
 							 surface, NULL, &frame);
+			CFRelease(surface);
 
 			uint64_t timestamp;
 			[components[1] getBytes:&timestamp
@@ -131,7 +147,6 @@ - (void)handlePortMessage:(NSPortMessage *)message
 					    fpsDenominator:fpsDenominator];
 
 			CVPixelBufferRelease(frame);
-			CFRelease(surface);
 		}
 		break;
 	case MachMsgIdStop:
diff --git a/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.mm b/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.mm
index b318f6fb73c874..571eefd70a6f2d 100644
--- a/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.mm
+++ b/plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.mm
@@ -397,7 +397,10 @@ - (void)queuePixelBuffer:(CVPixelBufferRef)frame
 	}
 
 	err = CMSimpleQueueEnqueue(self.queue, sampleBuffer);
+
 	if (err != noErr) {
+		CFRelease(sampleBuffer);
+
 		DLog(@"CMSimpleQueueEnqueue err %d", err);
 		return;
 	}
diff --git a/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.mm b/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.mm
index d2857766ee8721..efd33c1b44beae 100644
--- a/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.mm
+++ b/plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.mm
@@ -147,23 +147,33 @@ - (void)sendPixelBuffer:(CVPixelBufferRef)frame
 			dataWithBytes:&fpsDenominator
 			       length:sizeof(fpsDenominator)];
 
-		NSPort *framePort = [NSMachPort
-			portWithMachPort:IOSurfaceCreateMachPort(
-						 CVPixelBufferGetIOSurface(
-							 frame))];
+		IOSurfaceRef surface = CVPixelBufferGetIOSurface(frame);
+
+		if (!surface) {
+			blog(LOG_ERROR,
+			     "unable to access IOSurface associated with CVPixelBuffer");
+			return;
+		}
+
+		mach_port_t framePort = IOSurfaceCreateMachPort(surface);
 
 		if (!framePort) {
 			blog(LOG_ERROR,
-			     "unable to allocate mach port for pixel buffer");
+			     "unable to allocate mach port for IOSurface");
 			return;
 		}
 
 		[self sendMessageToClientsWithMsgId:MachMsgIdFrame
 					 components:@[
-						 framePort, timestampData,
+						 [NSMachPort
+							 portWithMachPort:framePort
+								  options:NSMachPortDeallocateNone],
+						 timestampData,
 						 fpsNumeratorData,
 						 fpsDenominatorData
 					 ]];
+
+		mach_port_deallocate(mach_task_self(), framePort);
 	}
 }