mac-virtualcam: Fix IOSurface memory leak

This change fixes a memory leak in the mac-virtualcam plugin that causes
OBS to not release the CVPixelBuffers (and underlying IOSurfaces)
it emits to the virtual camera consumers.

Pull request obsproject#6573 (Avoid
transcoding where possible) updated the mac-virtualcam to share the
virtual camera feed with other processes via IOSurfaces.

Although the changes work correctly, users have observed that OBS memory
usage keeps increasing when the virtual camera is active until OBS runs
out of memory or the consuming application is closed.
See the report by @SciTechNick for more information:
obsproject#6573 (comment)

After some debugging, I have found that the plugin is leaking Mach ports
associated with IOSurfaces, preventing them from being re-used. The
previous approach using `NSMachPort` does not seem to properly release
the Mach port allocated via `CVPixelBufferGetIOSurface` and
`IOSurfaceLookupFromMachPort`. Instead, we must explicitly deallocate
the port using `mach_port_deallocate`.

I have tested the changes on a Macbook Pro (M1) running macOS Monterey with
Google Chrome, Zoom, and Cameo. OBS shows no signs of memory leakage
after multiple minutes.

plugins/mac-virtualcam/src/dal-plugin/OBSDALMachClient.mm

@@ -101,14 +101,32 @@ - (void)handlePortMessage:(NSPortMessage *)message
 		break;
 	case MachMsgIdFrame:
 		VLog(@"Received frame message");
-		if (components.count >= 4) {
+
+		if (components.count < 4) {
+			break;
+		}
+
+		@autoreleasepool {
 			NSMachPort *framePort = (NSMachPort *)components[0];
+
+			if (!framePort) {
+				return;
+			}
+
 			IOSurfaceRef surface = IOSurfaceLookupFromMachPort(
 				[framePort machPort]);
+			mach_port_deallocate(mach_task_self(),
+					     [framePort machPort]);
+
+			if (!surface) {
+				ELog(@"Failed to obtain IOSurface from Mach port");
+				return;
+			}
 
 			CVPixelBufferRef frame;
 			CVPixelBufferCreateWithIOSurface(kCFAllocatorDefault,
 							 surface, NULL, &frame);
+			CFRelease(surface);
 
 			uint64_t timestamp;
 			[components[1] getBytes:&timestamp
@@ -131,7 +149,6 @@ - (void)handlePortMessage:(NSPortMessage *)message
 					    fpsDenominator:fpsDenominator];
 
 			CVPixelBufferRelease(frame);
-			CFRelease(surface);
 		}
 		break;
 	case MachMsgIdStop:

plugins/mac-virtualcam/src/dal-plugin/OBSDALStream.mm

@@ -397,7 +397,10 @@ - (void)queuePixelBuffer:(CVPixelBufferRef)frame
 	}
 
 	err = CMSimpleQueueEnqueue(self.queue, sampleBuffer);
+
 	if (err != noErr) {
+		CFRelease(sampleBuffer);
+
 		DLog(@"CMSimpleQueueEnqueue err %d", err);
 		return;
 	}

plugins/mac-virtualcam/src/obs-plugin/OBSDALMachServer.mm

@@ -147,23 +147,33 @@ - (void)sendPixelBuffer:(CVPixelBufferRef)frame
 			dataWithBytes:&fpsDenominator
 			       length:sizeof(fpsDenominator)];
 
-		NSPort *framePort = [NSMachPort
-			portWithMachPort:IOSurfaceCreateMachPort(
-						 CVPixelBufferGetIOSurface(
-							 frame))];
+		IOSurfaceRef surface = CVPixelBufferGetIOSurface(frame);
+
+		if (!surface) {
+			blog(LOG_ERROR,
+			     "unable to access IOSurface associated with CVPixelBuffer");
+			return;
+		}
+
+		mach_port_t framePort = IOSurfaceCreateMachPort(surface);
 
 		if (!framePort) {
 			blog(LOG_ERROR,
-			     "unable to allocate mach port for pixel buffer");
+			     "unable to allocate mach port for IOSurface");
 			return;
 		}
 
 		[self sendMessageToClientsWithMsgId:MachMsgIdFrame
 					 components:@[
-						 framePort, timestampData,
+						 [NSMachPort
+							 portWithMachPort:framePort
+								  options:NSMachPortDeallocateNone],
+						 timestampData,
 						 fpsNumeratorData,
 						 fpsDenominatorData
 					 ]];
+
+		mach_port_deallocate(mach_task_self(), framePort);
 	}
 }