Add support for iOS / UIKit, and clean up CoreGraphics impl

.github/workflows/ci.yml

@@ -47,7 +47,7 @@ jobs:
           - { target: x86_64-unknown-redox,     os: ubuntu-latest,   }
           - { target: x86_64-unknown-freebsd,   os: ubuntu-latest,   }
           - { target: x86_64-unknown-netbsd,    os: ubuntu-latest, options: --no-default-features, features: "x11,x11-dlopen,wayland,wayland-dlopen"  }
-          - { target: x86_64-apple-darwin,      os: macos-latest,    }
+          - { target: aarch64-apple-darwin,     os: macos-latest,    }
           - { target: wasm32-unknown-unknown,   os: ubuntu-latest,   }
         exclude:
           # Orbital doesn't follow MSRV
@@ -56,6 +56,9 @@ jobs:
         include:
           - rust_version: nightly
             platform: { target: wasm32-unknown-unknown, os: ubuntu-latest, options: "-Zbuild-std=panic_abort,std", rustflags: "-Ctarget-feature=+atomics,+bulk-memory" }
+          # Mac Catalyst is only Tier 2 since Rust 1.81
+          - rust_version: 'nightly'
+            platform: { target: aarch64-apple-ios-macabi, os: macos-latest }
 
     env:
       RUST_BACKTRACE: 1

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Unreleased
 
+- Added support for iOS, tvOS, watchOS and visionOS.
+- Redo the way surfaces work on macOS to work directly with layers, which will allow initializing directly from a `CALayer` in the future.
+
 # 0.4.5
 
 - Make the `wayland-sys` dependency optional. (#223)

Cargo.toml

@@ -45,14 +45,13 @@ x11rb = { version = "0.13.0", features = ["allow-unsafe-code", "shm"], optional
 version = "0.59.0"
 features = ["Win32_Graphics_Gdi", "Win32_UI_Shell", "Win32_UI_WindowsAndMessaging", "Win32_Foundation"]
 
-[target.'cfg(target_os = "macos")'.dependencies]
+[target.'cfg(target_vendor = "apple")'.dependencies]
 bytemuck = { version = "1.12.3", features = ["extern_crate_alloc"] }
 core-graphics = "0.24.0"
 foreign-types = "0.5.0"
-objc2 = "0.5.1"
-objc2-foundation = { version = "0.2.0", features = ["dispatch", "NSThread"] }
-objc2-app-kit = { version = "0.2.0", features = ["NSResponder", "NSView", "NSWindow"] }
-objc2-quartz-core = { version = "0.2.0", features = ["CALayer", "CATransaction"] }
+objc2 = "0.5.2"
+objc2-foundation = { version = "0.2.2", features = ["dispatch", "NSThread"] }
+objc2-quartz-core = { version = "0.2.2", features = ["CALayer", "CATransaction"] }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 js-sys = "0.3.63"

src/backend_dispatch.rs

@@ -186,8 +186,8 @@ make_dispatch! {
     Kms(Arc<backends::kms::KmsDisplayImpl<D>>, backends::kms::KmsImpl<D, W>, backends::kms::BufferImpl<'a, D, W>),
     #[cfg(target_os = "windows")]
     Win32(D, backends::win32::Win32Impl<D, W>, backends::win32::BufferImpl<'a, D, W>),
-    #[cfg(target_os = "macos")]
-    CG(D, backends::cg::CGImpl<D, W>, backends::cg::BufferImpl<'a, D, W>),
+    #[cfg(target_vendor = "apple")]
+    CoreGraphics(D, backends::cg::CGImpl<D, W>, backends::cg::BufferImpl<'a, D, W>),
     #[cfg(target_arch = "wasm32")]
     Web(backends::web::WebDisplayImpl<D>, backends::web::WebImpl<D, W>, backends::web::BufferImpl<'a, D, W>),
     #[cfg(target_os = "redox")]

src/backends/cg.rs

@@ -2,23 +2,23 @@ use crate::backend_interface::*;
 use crate::error::InitError;
 use crate::{Rect, SoftBufferError};
 use core_graphics::base::{
-    kCGBitmapByteOrder32Little, kCGImageAlphaNoneSkipFirst, kCGRenderingIntentDefault,
+    kCGBitmapByteOrder32Little, kCGImageAlphaNoneSkipFirst, kCGRenderingIntentDefault, CGFloat,
 };
 use core_graphics::color_space::CGColorSpace;
 use core_graphics::data_provider::CGDataProvider;
 use core_graphics::image::CGImage;
-use objc2::runtime::AnyObject;
+use objc2::runtime::{AnyObject, Bool};
+use objc2::{msg_send, msg_send_id};
 use raw_window_handle::{HasDisplayHandle, HasWindowHandle, RawWindowHandle};
 
 use foreign_types::ForeignType;
-use objc2::msg_send;
-use objc2::rc::Id;
-use objc2_app_kit::{NSAutoresizingMaskOptions, NSView, NSWindow};
-use objc2_foundation::{MainThreadBound, MainThreadMarker};
-use objc2_quartz_core::{kCAGravityTopLeft, CALayer, CATransaction};
+use objc2::rc::Retained;
+use objc2_foundation::{CGPoint, CGRect, CGSize, MainThreadMarker, NSObject};
+use objc2_quartz_core::{kCAGravityResize, CALayer, CATransaction};
 
 use std::marker::PhantomData;
 use std::num::NonZeroU32;
+use std::ops::Deref;
 use std::sync::Arc;
 
 struct Buffer(Vec<u32>);
@@ -30,64 +30,95 @@ impl AsRef<[u8]> for Buffer {
 }
 
 pub struct CGImpl<D, W> {
-    layer: MainThreadBound<Id<CALayer>>,
-    window: MainThreadBound<Id<NSWindow>>,
+    /// Our layer.
+    layer: SendCALayer,
+    /// The layer that our layer was created from.
+    ///
+    /// Can also be retrieved from `layer.superlayer()`.
+    root_layer: SendCALayer,
     color_space: SendCGColorSpace,
-    size: Option<(NonZeroU32, NonZeroU32)>,
     window_handle: W,
     _display: PhantomData<D>,
 }
 
-// TODO(madsmtm): Expose this in `objc2_app_kit`.
-fn set_layer(view: &NSView, layer: &CALayer) {
-    unsafe { msg_send![view, setLayer: layer] }
-}
-
 impl<D: HasDisplayHandle, W: HasWindowHandle> SurfaceInterface<D, W> for CGImpl<D, W> {
     type Context = D;
     type Buffer<'a> = BufferImpl<'a, D, W> where Self: 'a;
 
     fn new(window_src: W, _display: &D) -> Result<Self, InitError<W>> {
-        let raw = window_src.window_handle()?.as_raw();
-        let handle = match raw {
-            RawWindowHandle::AppKit(handle) => handle,
+        // `NSView`/`UIView` can only be accessed from the main thread.
+        let _mtm = MainThreadMarker::new().ok_or(SoftBufferError::PlatformError(
+            Some("can only access Core Graphics handles from the main thread".to_string()),
+            None,
+        ))?;
+
+        let root_layer = match window_src.window_handle()?.as_raw() {
+            RawWindowHandle::AppKit(handle) => {
+                // SAFETY: The pointer came from `WindowHandle`, which ensures that the
+                // `AppKitWindowHandle` contains a valid pointer to an `NSView`.
+                //
+                // We use `NSObject` here to avoid importing `objc2-app-kit`.
+                let view: &NSObject = unsafe { handle.ns_view.cast().as_ref() };
+
+                // Force the view to become layer backed
+                let _: () = unsafe { msg_send![view, setWantsLayer: Bool::YES] };
+
+                // SAFETY: `-[NSView layer]` returns an optional `CALayer`
+                let layer: Option<Retained<CALayer>> = unsafe { msg_send_id![view, layer] };
+                layer.expect("failed making the view layer-backed")
+            }
+            RawWindowHandle::UiKit(handle) => {
+                // SAFETY: The pointer came from `WindowHandle`, which ensures that the
+                // `UiKitWindowHandle` contains a valid pointer to an `UIView`.
+                //
+                // We use `NSObject` here to avoid importing `objc2-ui-kit`.
+                let view: &NSObject = unsafe { handle.ui_view.cast().as_ref() };
+
+                // SAFETY: `-[UIView layer]` returns `CALayer`
+                let layer: Retained<CALayer> = unsafe { msg_send_id![view, layer] };
+                layer
+            }
             _ => return Err(InitError::Unsupported(window_src)),
         };
 
-        // `NSView` can only be accessed from the main thread.
-        let mtm = MainThreadMarker::new().ok_or(SoftBufferError::PlatformError(
-            Some("can only access AppKit / macOS handles from the main thread".to_string()),
-            None,
-        ))?;
-        let view = handle.ns_view.as_ptr();
-        // SAFETY: The pointer came from `WindowHandle`, which ensures that
-        // the `AppKitWindowHandle` contains a valid pointer to an `NSView`.
-        // Unwrap is fine, since the pointer came from `NonNull`.
-        let view: Id<NSView> = unsafe { Id::retain(view.cast()) }.unwrap();
+        // Add a sublayer, to avoid interfering with the root layer, since setting the contents of
+        // e.g. a view-controlled layer is brittle.
         let layer = CALayer::new();
-        let subview = unsafe { NSView::initWithFrame(mtm.alloc(), view.frame()) };
-        layer.setContentsGravity(unsafe { kCAGravityTopLeft });
-        layer.setNeedsDisplayOnBoundsChange(false);
-        set_layer(&subview, &layer);
-        unsafe {
-            subview.setAutoresizingMask(NSAutoresizingMaskOptions(
-                NSAutoresizingMaskOptions::NSViewWidthSizable.0
-                    | NSAutoresizingMaskOptions::NSViewHeightSizable.0,
-            ))
-        };
+        root_layer.addSublayer(&layer);
 
-        let window = view.window().ok_or(SoftBufferError::PlatformError(
-            Some("view must be inside a window".to_string()),
-            None,
-        ))?;
+        // Set the anchor point. Used to avoid having to calculate the center point when setting
+        // `bounds` in `resize`.
+        layer.setAnchorPoint(CGPoint::new(0.0, 0.0));
+
+        // Set initial scale factor. Updated in `resize`.
+        layer.setContentsScale(root_layer.contentsScale());
+
+        // Set `bounds` and `position` so that the new layer is inside the superlayer.
+        //
+        // This differs from just setting the `bounds`, as it also takes into account any
+        // translation that the superlayer may have that we want to preserve.
+        layer.setFrame(root_layer.bounds());
 
-        unsafe { view.addSubview(&subview) };
+        // Do not use auto-resizing mask, see comments in `resize` for details.
+        // layer.setAutoresizingMask(kCALayerHeightSizable | kCALayerWidthSizable);
+
+        // Set the content gravity in a way that masks failure to redraw at the correct time.
+        //
+        // Effectively, this means that the shown surface always matches the `bounds` that the user
+        // has specified in `resize`, regardless of whether the user has rendered yet or not.
+        layer.setContentsGravity(unsafe { kCAGravityResize });
+
+        // Softbuffer uses a coordinate system with the origin in the top-left corner (doesn't
+        // really matter unless we start setting the `position` of our layer).
+        layer.setGeometryFlipped(true);
+
+        // Initialize color space here, to reduce work later on.
         let color_space = CGColorSpace::create_device_rgb();
+
         Ok(Self {
-            layer: MainThreadBound::new(layer, mtm),
-            window: MainThreadBound::new(window, mtm),
+            layer: SendCALayer(layer),
+            root_layer: SendCALayer(root_layer),
             color_space: SendCGColorSpace(color_space),
-            size: None,
             _display: PhantomData,
             window_handle: window_src,
         })
@@ -99,17 +130,70 @@ impl<D: HasDisplayHandle, W: HasWindowHandle> SurfaceInterface<D, W> for CGImpl<
     }
 
     fn resize(&mut self, width: NonZeroU32, height: NonZeroU32) -> Result<(), SoftBufferError> {
-        self.size = Some((width, height));
+        let scale_factor = self.root_layer.contentsScale();
+        let bounds = CGRect::new(
+            CGPoint::new(0.0, 0.0),
+            CGSize::new(
+                width.get() as CGFloat / scale_factor,
+                height.get() as CGFloat / scale_factor,
+            ),
+        );
+
+        // _Usually_, the programmer should be resizing the surface together in lockstep with the
+        // user action that initiated the resize, e.g. a window resize, where there would already be
+        // a transaction ongoing.
+        //
+        // With the current version of Winit, though, that isn't the case, and we end up getting the
+        // resize event emitted later, outside the callstack where the transaction was ongoing. The
+        // user could also choose to resize e.g. on a different thread, or in loads of other
+        // circumstances.
+        //
+        // This, in turn, means that the default animation with a delay of 0.25 seconds kicks in
+        // when updating these values - this is definitely not what we want, so we disable those
+        // animations here.
+        CATransaction::begin();
+        CATransaction::setDisableActions(true);
+
+        // Set the scale factor of the layer to match the root layer / super layer, in case it
+        // changed (e.g. if moved to a different monitor, or monitor settings changed).
+        self.layer.setContentsScale(scale_factor);
+
+        // Set the bounds on the layer.
+        //
+        // This is an explicit design decision: We set the bounds on the layer manually, instead of
+        // letting it be automatically updated using `autoresizingMask`.
+        //
+        // The first reason for this is that it gives the user complete control over the size of the
+        // layer (and underlying buffer, once properly implemented, see #83), which matches other
+        // platforms.
+        //
+        // The second is that it is needed to work around a bug in macOS 14 and above, where views
+        // using auto layout may end up setting fractional values as the bounds, and that in turn
+        // doesn't propagate properly through the auto-resizing mask and with contents gravity.
+        //
+        // If we were to change this so that the layer resizes automatically, we should _not_ use
+        // `layer.setAutoresizingMask(...)`, but instead register an observer on the super layer,
+        // which then propagates the bounds change to the sublayer. It is unfortunate that we cannot
+        // use the built-in functionality to do this, but not something you can avoid either way if
+        // you're doing automatic resizing, since you'd _need_ to propagate the scale factor anyhow.
+        self.layer.setBounds(bounds);
+
+        // See comment on `CATransaction::begin`.
+        CATransaction::commit();
+
         Ok(())
     }
 
     fn buffer_mut(&mut self) -> Result<BufferImpl<'_, D, W>, SoftBufferError> {
-        let (width, height) = self
-            .size
-            .expect("Must set size of surface before calling `buffer_mut()`");
+        let scale_factor = self.layer.contentsScale();
+        let bounds = self.layer.bounds();
+        // The bounds and scale factor are set in `resize`, and should result in integer values when
+        // combined like this.
+        let width = (bounds.size.width * scale_factor) as usize;
+        let height = (bounds.size.height * scale_factor) as usize;
 
         Ok(BufferImpl {
-            buffer: vec![0; width.get() as usize * height.get() as usize],
+            buffer: vec![0; width * height],
             imp: self,
         })
     }
@@ -137,41 +221,38 @@ impl<'a, D: HasDisplayHandle, W: HasWindowHandle> BufferInterface for BufferImpl
 
     fn present(self) -> Result<(), SoftBufferError> {
         let data_provider = CGDataProvider::from_buffer(Arc::new(Buffer(self.buffer)));
-        let (width, height) = self.imp.size.unwrap();
+
+        let scale_factor = self.imp.layer.contentsScale();
+        let bounds = self.imp.layer.bounds();
+        // The bounds and scale factor are set in `resize`, and should result in integer values when
+        // combined like this.
+        let width = (bounds.size.width * scale_factor) as usize;
+        let height = (bounds.size.height * scale_factor) as usize;
+
         let image = CGImage::new(
-            width.get() as usize,
-            height.get() as usize,
+            width,
+            height,
             8,
             32,
-            (width.get() * 4) as usize,
+            width * 4,
             &self.imp.color_space.0,
             kCGBitmapByteOrder32Little | kCGImageAlphaNoneSkipFirst,
             &data_provider,
             false,
             kCGRenderingIntentDefault,
         );
-
-        // TODO: Use run_on_main() instead.
-        let mtm = MainThreadMarker::new().ok_or(SoftBufferError::PlatformError(
-            Some("can only access AppKit / macOS handles from the main thread".to_string()),
-            None,
-        ))?;
+        let contents = unsafe { (image.as_ptr() as *mut AnyObject).as_ref() };
 
         // The CALayer has a default action associated with a change in the layer contents, causing
         // a quarter second fade transition to happen every time a new buffer is applied. This can
         // be mitigated by wrapping the operation in a transaction and disabling all actions.
         CATransaction::begin();
         CATransaction::setDisableActions(true);
 
-        let layer = self.imp.layer.get(mtm);
-        layer.setContentsScale(self.imp.window.get(mtm).backingScaleFactor());
-
-        unsafe {
-            layer.setContents((image.as_ptr() as *mut AnyObject).as_ref());
-        };
+        // SAFETY: The contents is `CGImage`, which is a valid class for `contents`.
+        unsafe { self.imp.layer.setContents(contents) };
 
         CATransaction::commit();
-
         Ok(())
     }
 
@@ -184,3 +265,15 @@ struct SendCGColorSpace(CGColorSpace);
 // SAFETY: `CGColorSpace` is immutable, and can freely be shared between threads.
 unsafe impl Send for SendCGColorSpace {}
 unsafe impl Sync for SendCGColorSpace {}
+
+struct SendCALayer(Retained<CALayer>);
+// CALayer is thread safe
+unsafe impl Send for SendCALayer {}
+unsafe impl Sync for SendCALayer {}
+
+impl Deref for SendCALayer {
+    type Target = CALayer;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}

src/backends/mod.rs

@@ -1,7 +1,7 @@
 use crate::{ContextInterface, InitError};
 use raw_window_handle::HasDisplayHandle;
 
-#[cfg(target_os = "macos")]
+#[cfg(target_vendor = "apple")]
 pub(crate) mod cg;
 #[cfg(kms_platform)]
 pub(crate) mod kms;

src/lib.rs

@@ -118,7 +118,7 @@ impl<D: HasDisplayHandle, W: HasWindowHandle> Surface<D, W> {
     /// ## Platform Dependent Behavior
     ///
     /// - On X11, the window must be visible.
-    /// - On macOS, Redox and Wayland, this function is unimplemented.
+    /// - On Apple platforms, Redox and Wayland, this function is unimplemented.
     /// - On Web, this will fail if the content was supplied by
     ///   a different origin depending on the sites CORS rules.
     pub fn fetch(&mut self) -> Result<Vec<u32>, SoftBufferError> {
@@ -194,7 +194,7 @@ impl<D: HasDisplayHandle, W: HasWindowHandle> HasWindowHandle for Surface<D, W>
 ///
 /// Currently [`Buffer::present`] must block copying image data on:
 /// - Web
-/// - macOS
+/// - Apple platforms
 pub struct Buffer<'a, D, W> {
     buffer_impl: BufferDispatch<'a, D, W>,
     _marker: PhantomData<(Arc<D>, Cell<()>)>,