From 290b349e2c225a5f24e3bd6c37fbbea96239ddbe Mon Sep 17 00:00:00 2001 From: Franz Busch Date: Thu, 17 Oct 2024 10:57:40 +0100 Subject: [PATCH 01/20] [CI] Fix python lint (#2925) # Motivation Fix the python lint check. # Modification This PR fixes all the warnings related to the new python lint check. # Result Only green CI. --- .github/workflows/pull_request.yml | 2 +- .github/workflows/scheduled.yml | 2 +- ...reBenchmarks.NIOAsyncChannel.init.p90.json | 3 ++ .../6.0/NIOPosixBenchmarks.TCPEcho.p90.json | 3 ++ ...sixBenchmarks.TCPEchoAsyncChannel.p90.json | 3 ++ dev/stackdiff-dtrace.py | 30 +++++++++++-------- 6 files changed, 29 insertions(+), 14 deletions(-) create mode 100644 Benchmarks/Thresholds/6.0/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json create mode 100644 Benchmarks/Thresholds/6.0/NIOPosixBenchmarks.TCPEcho.p90.json create mode 100644 Benchmarks/Thresholds/6.0/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 3491a85f64..c1ac0581f3 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -20,7 +20,7 @@ jobs: with: linux_5_9_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error" linux_5_10_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error" - linux_6_0_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error" + linux_6_0_arguments_override: "--explicit-target-dependency-import-check error" linux_nightly_6_0_arguments_override: "--explicit-target-dependency-import-check error" linux_nightly_main_arguments_override: "--explicit-target-dependency-import-check error" diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 92fcde88af..0f334ed511 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -12,7 +12,7 @@ jobs: with: linux_5_9_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error" linux_5_10_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error" - linux_6_0_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error" + linux_6_0_arguments_override: "--explicit-target-dependency-import-check error" linux_nightly_6_0_arguments_override: "--explicit-target-dependency-import-check error" linux_nightly_main_arguments_override: "--explicit-target-dependency-import-check error" diff --git a/Benchmarks/Thresholds/6.0/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json b/Benchmarks/Thresholds/6.0/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json new file mode 100644 index 0000000000..ec2d0ad0f8 --- /dev/null +++ b/Benchmarks/Thresholds/6.0/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 8000 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/6.0/NIOPosixBenchmarks.TCPEcho.p90.json b/Benchmarks/Thresholds/6.0/NIOPosixBenchmarks.TCPEcho.p90.json new file mode 100644 index 0000000000..67abcf36ac --- /dev/null +++ b/Benchmarks/Thresholds/6.0/NIOPosixBenchmarks.TCPEcho.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 548 +} diff --git a/Benchmarks/Thresholds/6.0/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json b/Benchmarks/Thresholds/6.0/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json new file mode 100644 index 0000000000..390ed2415a --- /dev/null +++ b/Benchmarks/Thresholds/6.0/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 164376 +} \ No newline at end of file diff --git a/dev/stackdiff-dtrace.py b/dev/stackdiff-dtrace.py index 0394d9dabb..6f95335f55 100755 --- a/dev/stackdiff-dtrace.py +++ b/dev/stackdiff-dtrace.py @@ -13,18 +13,19 @@ ## ##===----------------------------------------------------------------------===## -import sys -import re import collections +import re +import sys num_regex = "^ +([0-9]+)$" + def put_in_dict(path): # Our input looks something like: # # ===== # This will collect stack shots of allocations and print it when you exit dtrace. - # So go ahead, run your tests and then press Ctrl+C in this window to see the aggregated result + # So go ahead, run your tests and then press Ctrl+C in this window to see the aggregated result # noqa: E501 # ===== # DEBUG: After waiting 1 times, we quiesced to unfreeds=744 # test_1_reqs_1000_conn.total_allocations: 490000 @@ -39,11 +40,11 @@ def put_in_dict(path): # libswiftCore.dylib`swift_allocObject+0x27 # test_1_reqs_1000_conn`closure #3 in SelectableEventLoop.run()+0x166 # test_1_reqs_1000_conn`SelectableEventLoop.run()+0x234 - # test_1_reqs_1000_conn`closure #1 in static MultiThreadedEventLoopGroup.setupThreadAndEventLoop(name:selectorFactory:initializer:)+0x12e - # test_1_reqs_1000_conn`partial apply for closure #1 in static MultiThreadedEventLoopGroup.setupThreadAndEventLoop(name:selectorFactory:initializer:)+0x25 - # test_1_reqs_1000_conn`thunk for @escaping @callee_guaranteed (@guaranteed NIOThread) -> ()+0xf - # test_1_reqs_1000_conn`partial apply for thunk for @escaping @callee_guaranteed (@guaranteed NIOThread) -> ()+0x11 - # test_1_reqs_1000_conn`closure #1 in static ThreadOpsPosix.run(handle:args:detachThread:)+0x1c9 + # test_1_reqs_1000_conn`closure #1 in static MultiThreadedEventLoopGroup.setupThreadAndEventLoop(name:selectorFactory:initializer:)+0x12e # noqa: E501 + # test_1_reqs_1000_conn`partial apply for closure #1 in static MultiThreadedEventLoopGroup.setupThreadAndEventLoop(name:selectorFactory:initializer:)+0x25 # noqa: E501 + # test_1_reqs_1000_conn`thunk for @escaping @callee_guaranteed (@guaranteed NIOThread) -> ()+0xf # noqa: E501 + # test_1_reqs_1000_conn`partial apply for thunk for @escaping @callee_guaranteed (@guaranteed NIOThread) -> ()+0x11 # noqa: E501 + # test_1_reqs_1000_conn`closure #1 in static ThreadOpsPosix.run(handle:args:detachThread:)+0x1c9 # noqa: E501 # libsystem_pthread.dylib`_pthread_start+0xe0 # libsystem_pthread.dylib`thread_start+0xf # 85945 @@ -71,7 +72,7 @@ def put_in_dict(path): key = "\n".join(line.split("+")[0] for line in current_stack[:8]) # Record this stack and reset our state to build a new one. - dictionary[key].append( (int(line), "\n".join(current_stack)) ) + dictionary[key].append((int(line), "\n".join(current_stack))) current_stack = [] else: # This line doesn't contain just a number. This might be an @@ -82,9 +83,11 @@ def put_in_dict(path): return dictionary + def total_count_for_key(d, key): value = d[key] - return sum(map(lambda x : x[0], value)) + return sum(map(lambda x: x[0], value)) + def total_for_dictionary(d): total = 0 @@ -92,6 +95,7 @@ def total_for_dictionary(d): total += total_count_for_key(d, k) return total + def extract_useful_keys(d): keys = set() for k in d.keys(): @@ -99,6 +103,7 @@ def extract_useful_keys(d): keys.add(k) return keys + def print_dictionary_member(d, key): print(total_count_for_key(d, key)) print(key) @@ -106,6 +111,7 @@ def print_dictionary_member(d, key): print_dictionary_member_detail(d, key) print() + def print_dictionary_member_detail(d, key): value = d[key] for (count, stack) in value: @@ -128,6 +134,7 @@ def usage(): print(" # diff them") print(" stackdiff-dtrace.py /tmp/old /tmp/new") + if len(sys.argv) != 3: usage() sys.exit(1) @@ -173,5 +180,4 @@ def usage(): everything_before = total_for_dictionary(before_dict) everything_after = total_for_dictionary(after_dict) -print("Total of _EVERYTHING_ BEFORE: %d, AFTER: %d, DIFFERENCE: %d" % - (everything_before, everything_after, everything_after - everything_before)) +print("Total of _EVERYTHING_ BEFORE: %d, AFTER: %d, DIFFERENCE: %d" % (everything_before, everything_after, everything_after - everything_before)) # noqa: E501 From 06c16b1a26394cad90f5342e50ef92c2c512f4a1 Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Thu, 17 Oct 2024 13:03:52 +0100 Subject: [PATCH 02/20] Add future wait benchmark to catch memory leaks (#2931) ### Motivation: In the past we introduced a memory leak around the creation of and waiting on futures - we should protect against leaks in this fundamental operation. ### Modifications: Add a new benchmark which would have failed with the previous bug ### Result: Regression protection for this type of memory leak. --- .../NIOCoreBenchmarks/Benchmarks.swift | 29 +++++++++++++++++++ .../NIOCoreBenchmarks.WaitOnPromise.p90.json | 4 +++ .../NIOCoreBenchmarks.WaitOnPromise.p90.json | 4 +++ .../NIOCoreBenchmarks.WaitOnPromise.p90.json | 4 +++ 4 files changed, 41 insertions(+) create mode 100644 Benchmarks/Thresholds/5.10/NIOCoreBenchmarks.WaitOnPromise.p90.json create mode 100644 Benchmarks/Thresholds/5.8/NIOCoreBenchmarks.WaitOnPromise.p90.json create mode 100644 Benchmarks/Thresholds/5.9/NIOCoreBenchmarks.WaitOnPromise.p90.json diff --git a/Benchmarks/Benchmarks/NIOCoreBenchmarks/Benchmarks.swift b/Benchmarks/Benchmarks/NIOCoreBenchmarks/Benchmarks.swift index cea2e10af5..c821b872dd 100644 --- a/Benchmarks/Benchmarks/NIOCoreBenchmarks/Benchmarks.swift +++ b/Benchmarks/Benchmarks/NIOCoreBenchmarks/Benchmarks.swift @@ -21,6 +21,11 @@ let benchmarks = { .mallocCountTotal ] + let leakMetrics: [BenchmarkMetric] = [ + .mallocCountTotal, + .memoryLeaked, + ] + Benchmark( "NIOAsyncChannel.init", configuration: .init( @@ -46,4 +51,28 @@ let benchmarks = { blackHole(asyncChanel) } } + + Benchmark( + "WaitOnPromise", + configuration: .init( + metrics: leakMetrics, + scalingFactor: .kilo, + maxDuration: .seconds(10_000_000), + maxIterations: 10_000 // need 10k to get a signal + ) + ) { benchmark in + // Elide the cost of the 'EmbeddedEventLoop'. + let el = EmbeddedEventLoop() + + benchmark.startMeasurement() + defer { + benchmark.stopMeasurement() + } + + for _ in 0.. Date: Thu, 17 Oct 2024 13:22:10 +0100 Subject: [PATCH 03/20] Drop support for Swift 5.8 (#2924) # Motivation We only support the last three Swift released versions which are at this time 5.9, 5.10 and 6. # Modification This PR drops anything related to Swift 5.8. # Result Version support aligned. --- .github/workflows/main.yml | 2 +- .../5.8/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json | 3 --- .../Thresholds/5.8/NIOPosixBenchmarks.TCPEcho.p90.json | 3 --- CONTRIBUTING.md | 2 +- Package.swift | 2 +- README.md | 7 ++++--- Sources/NIOCore/EventLoop+SerialExecutor.swift | 2 -- Sources/NIOCore/EventLoop.swift | 4 ---- Sources/NIOCore/NIOScheduledCallback.swift | 3 +-- Sources/NIOEmbedded/AsyncTestingEventLoop.swift | 2 -- Sources/NIOEmbedded/Embedded.swift | 2 -- Sources/NIOPosix/MultiThreadedEventLoopGroup.swift | 2 -- .../NIOPosix/PosixSingletons+ConcurrencyTakeOver.swift | 2 -- Sources/NIOPosix/SelectableEventLoop.swift | 6 ------ Sources/NIOTCPEchoClient/Client.swift | 9 --------- Sources/NIOTCPEchoServer/Server.swift | 9 --------- Tests/NIOPosixTests/SerialExecutorTests.swift | 7 ------- 17 files changed, 8 insertions(+), 59 deletions(-) delete mode 100644 Benchmarks/Thresholds/5.8/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json delete mode 100644 Benchmarks/Thresholds/5.8/NIOPosixBenchmarks.TCPEcho.p90.json diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 113e35e6a2..1a971acf46 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,7 +12,7 @@ jobs: with: linux_5_9_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error" linux_5_10_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error" - linux_6_0_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error" + linux_6_0_arguments_override: "--explicit-target-dependency-import-check error" linux_nightly_6_0_arguments_override: "--explicit-target-dependency-import-check error" linux_nightly_main_arguments_override: "--explicit-target-dependency-import-check error" diff --git a/Benchmarks/Thresholds/5.8/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json b/Benchmarks/Thresholds/5.8/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json deleted file mode 100644 index ec2d0ad0f8..0000000000 --- a/Benchmarks/Thresholds/5.8/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "mallocCountTotal" : 8000 -} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.8/NIOPosixBenchmarks.TCPEcho.p90.json b/Benchmarks/Thresholds/5.8/NIOPosixBenchmarks.TCPEcho.p90.json deleted file mode 100644 index 8517c2fe45..0000000000 --- a/Benchmarks/Thresholds/5.8/NIOPosixBenchmarks.TCPEcho.p90.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "mallocCountTotal" : 556 -} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 280f2b7a4f..ec9f46f020 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -67,7 +67,7 @@ The default policy for taking contributions is “Squash and Merge” - because ### Make sure your patch works for all supported versions of swift -The CI will do this for you, but a project maintainer must kick it off for you. Currently all versions of Swift >= 5.8 are supported. +The CI will do this for you, but a project maintainer must kick it off for you. Currently all versions of Swift >= 5.9 are supported. If you wish to test this locally you have two options [act](https://github.com/nektos/act) and Docker Compose files. diff --git a/Package.swift b/Package.swift index c5edd5ebf1..99842452b9 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version:5.8 +// swift-tools-version:5.9 //===----------------------------------------------------------------------===// // // This source file is part of the SwiftNIO open source project diff --git a/README.md b/README.md index 699b607ac3..055c5359c3 100644 --- a/README.md +++ b/README.md @@ -74,9 +74,9 @@ This is the current version of SwiftNIO and will be supported for the foreseeabl ### Swift Versions We commit to support the most recently released swift version (currently 5.10) and the last two minor releases before that unless this is impossible to do in one codebase. -In addition checks are run against the latest beta release (if any) as well as the nightly swift builds and the intent is that these should pass. +In addition checks are run against the latest beta release (if any) as well as the nightly swift builds and the intent is that these should pass. -The most recent versions of SwiftNIO support Swift 5.8 and newer. The minimum Swift version supported by SwiftNIO releases are detailed below: +The most recent versions of SwiftNIO support Swift 5.9 and newer. The minimum Swift version supported by SwiftNIO releases are detailed below: SwiftNIO | Minimum Swift Version --------------------|---------------------- @@ -86,7 +86,8 @@ SwiftNIO | Minimum Swift Version `2.43.0 ..< 2.51.0` | 5.5.2 `2.51.0 ..< 2.60.0` | 5.6 `2.60.0 ..< 2.65.0` | 5.7 -`2.65.0 ...` | 5.8 +`2.65.0 ..< 2.76.0 | 5.8 +`2.76.0 ...` | 5.9 ### SwiftNIO 1 SwiftNIO 1 is considered end of life - it is strongly recommended that you move to a newer version. The Core NIO team does not actively work on this version. No new features will be added to this version but PRs which fix bugs or security vulnerabilities will be accepted until the end of May 2022. diff --git a/Sources/NIOCore/EventLoop+SerialExecutor.swift b/Sources/NIOCore/EventLoop+SerialExecutor.swift index 2fc92f1f1a..d8486a59a1 100644 --- a/Sources/NIOCore/EventLoop+SerialExecutor.swift +++ b/Sources/NIOCore/EventLoop+SerialExecutor.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if compiler(>=5.9) /// A helper protocol that can be mixed in to a NIO ``EventLoop`` to provide an /// automatic conformance to `SerialExecutor`. /// @@ -91,4 +90,3 @@ extension NIODefaultSerialEventLoopExecutor: SerialExecutor { self.loop === other.loop } } -#endif diff --git a/Sources/NIOCore/EventLoop.swift b/Sources/NIOCore/EventLoop.swift index 171a757ba9..4e76abe821 100644 --- a/Sources/NIOCore/EventLoop.swift +++ b/Sources/NIOCore/EventLoop.swift @@ -317,7 +317,6 @@ public protocol EventLoop: EventLoopGroup { /// allows `EventLoop`s to cache a pre-succeeded `Void` future to prevent superfluous allocations. func makeSucceededVoidFuture() -> EventLoopFuture - #if compiler(>=5.9) /// Returns a `SerialExecutor` corresponding to this ``EventLoop``. /// /// This executor can be used to isolate an actor to a given ``EventLoop``. Implementers are encouraged to customise @@ -330,7 +329,6 @@ public protocol EventLoop: EventLoopGroup { /// Submit a job to be executed by the `EventLoop` @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) func enqueue(_ job: consuming ExecutorJob) - #endif /// Must crash if it is not safe to call `wait()` on an `EventLoopFuture`. /// @@ -415,7 +413,6 @@ extension EventLoop { } extension EventLoop { - #if compiler(>=5.9) @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) public var executor: any SerialExecutor { NIODefaultSerialEventLoopExecutor(self) @@ -432,7 +429,6 @@ extension EventLoop { unownedJob.runSynchronously(on: self.executor.asUnownedSerialExecutor()) } } - #endif } extension EventLoopGroup { diff --git a/Sources/NIOCore/NIOScheduledCallback.swift b/Sources/NIOCore/NIOScheduledCallback.swift index 84c9f8d202..8d7a056a0a 100644 --- a/Sources/NIOCore/NIOScheduledCallback.swift +++ b/Sources/NIOCore/NIOScheduledCallback.swift @@ -89,8 +89,7 @@ public struct NIOScheduledCallback: Sendable { } extension EventLoop { - // This could be package once we drop Swift 5.8. - public func _scheduleCallback( + package func _scheduleCallback( at deadline: NIODeadline, handler: some NIOScheduledCallbackHandler ) -> NIOScheduledCallback { diff --git a/Sources/NIOEmbedded/AsyncTestingEventLoop.swift b/Sources/NIOEmbedded/AsyncTestingEventLoop.swift index 959b4bb343..52d56bf9ca 100644 --- a/Sources/NIOEmbedded/AsyncTestingEventLoop.swift +++ b/Sources/NIOEmbedded/AsyncTestingEventLoop.swift @@ -407,10 +407,8 @@ public final class NIOAsyncTestingEventLoop: EventLoop, @unchecked Sendable { } // MARK: SerialExecutor conformance -#if compiler(>=5.9) @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) extension NIOAsyncTestingEventLoop: NIOSerialEventLoopExecutor {} -#endif /// This is a thread-safe promise creation store. /// diff --git a/Sources/NIOEmbedded/Embedded.swift b/Sources/NIOEmbedded/Embedded.swift index 93e90bfe54..a6c2b33b5b 100644 --- a/Sources/NIOEmbedded/Embedded.swift +++ b/Sources/NIOEmbedded/Embedded.swift @@ -271,14 +271,12 @@ public final class EmbeddedEventLoop: EventLoop, CustomStringConvertible { precondition(scheduledTasks.isEmpty, "Embedded event loop freed with unexecuted scheduled tasks!") } - #if compiler(>=5.9) @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) public var executor: any SerialExecutor { fatalError( "EmbeddedEventLoop is not thread safe and cannot be used as a SerialExecutor. Use NIOAsyncTestingEventLoop instead." ) } - #endif } @usableFromInline diff --git a/Sources/NIOPosix/MultiThreadedEventLoopGroup.swift b/Sources/NIOPosix/MultiThreadedEventLoopGroup.swift index c188e8ef69..6e20a9a98c 100644 --- a/Sources/NIOPosix/MultiThreadedEventLoopGroup.swift +++ b/Sources/NIOPosix/MultiThreadedEventLoopGroup.swift @@ -489,7 +489,6 @@ extension MultiThreadedEventLoopGroup: CustomStringConvertible { } } -#if compiler(>=5.9) @usableFromInline struct ErasedUnownedJob { @usableFromInline @@ -507,7 +506,6 @@ struct ErasedUnownedJob { self.erasedJob as! UnownedJob } } -#endif @usableFromInline internal struct ScheduledTask { diff --git a/Sources/NIOPosix/PosixSingletons+ConcurrencyTakeOver.swift b/Sources/NIOPosix/PosixSingletons+ConcurrencyTakeOver.swift index e56242b12b..07ee8955ce 100644 --- a/Sources/NIOPosix/PosixSingletons+ConcurrencyTakeOver.swift +++ b/Sources/NIOPosix/PosixSingletons+ConcurrencyTakeOver.swift @@ -15,14 +15,12 @@ import Atomics import NIOCore -#if compiler(>=5.9) private protocol SilenceWarning { @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) func enqueue(_ job: UnownedJob) } @available(macOS 14, *) extension SelectableEventLoop: SilenceWarning {} -#endif private let _haveWeTakenOverTheConcurrencyPool = ManagedAtomic(false) extension NIOSingletons { diff --git a/Sources/NIOPosix/SelectableEventLoop.swift b/Sources/NIOPosix/SelectableEventLoop.swift index 79ba55b6c8..2ec9c71605 100644 --- a/Sources/NIOPosix/SelectableEventLoop.swift +++ b/Sources/NIOPosix/SelectableEventLoop.swift @@ -353,7 +353,6 @@ internal final class SelectableEventLoop: EventLoop { try? self._schedule0(.immediate(.function(task))) } - #if compiler(>=5.9) @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) @usableFromInline func enqueue(_ job: consuming ExecutorJob) { @@ -361,7 +360,6 @@ internal final class SelectableEventLoop: EventLoop { let erasedJob = ErasedUnownedJob(job: UnownedJob(job)) try? self._schedule0(.immediate(.unownedJob(erasedJob))) } - #endif /// Add the `ScheduledTask` to be executed. @usableFromInline @@ -901,17 +899,13 @@ extension SelectableEventLoop: CustomStringConvertible, CustomDebugStringConvert } // MARK: SerialExecutor conformance -#if compiler(>=5.9) @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) extension SelectableEventLoop: NIOSerialEventLoopExecutor {} -#endif @usableFromInline enum UnderlyingTask { case function(() -> Void) - #if compiler(>=5.9) case unownedJob(ErasedUnownedJob) - #endif case callback(any NIOScheduledCallbackHandler) } diff --git a/Sources/NIOTCPEchoClient/Client.swift b/Sources/NIOTCPEchoClient/Client.swift index e624a78667..67f917e9da 100644 --- a/Sources/NIOTCPEchoClient/Client.swift +++ b/Sources/NIOTCPEchoClient/Client.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if compiler(>=5.9) import NIOCore import NIOPosix @@ -112,11 +111,3 @@ private final class NewlineDelimiterCoder: ByteToMessageDecoder, MessageToByteEn out.writeInteger(self.newLine) } } -#else -@main -struct Client { - static func main() { - fatalError("Requires at least Swift 5.9") - } -} -#endif diff --git a/Sources/NIOTCPEchoServer/Server.swift b/Sources/NIOTCPEchoServer/Server.swift index c8f2200410..6f0d98c1b8 100644 --- a/Sources/NIOTCPEchoServer/Server.swift +++ b/Sources/NIOTCPEchoServer/Server.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if compiler(>=5.9) import NIOCore import NIOPosix @@ -122,11 +121,3 @@ private final class NewlineDelimiterCoder: ByteToMessageDecoder, MessageToByteEn out.writeInteger(self.newLine) } } -#else -@main -struct Server { - static func main() { - fatalError("Requires at least Swift 5.9") - } -} -#endif diff --git a/Tests/NIOPosixTests/SerialExecutorTests.swift b/Tests/NIOPosixTests/SerialExecutorTests.swift index 8df2ed6be0..0ec3994f9c 100644 --- a/Tests/NIOPosixTests/SerialExecutorTests.swift +++ b/Tests/NIOPosixTests/SerialExecutorTests.swift @@ -16,7 +16,6 @@ import NIOEmbedded import NIOPosix import XCTest -#if compiler(>=5.9) @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) actor EventLoopBoundActor { nonisolated let unownedExecutor: UnownedSerialExecutor @@ -45,19 +44,13 @@ actor EventLoopBoundActor { } #endif } -#endif @available(macOS 14.0, iOS 17.0, watchOS 10.0, tvOS 17.0, *) final class SerialExecutorTests: XCTestCase { private func _testBasicExecutorFitsOnEventLoop(loop1: EventLoop, loop2: EventLoop) async throws { - #if compiler(<5.9) - throw XCTSkip("Custom executors are only supported in 5.9") - #else - let testActor = EventLoopBoundActor(loop: loop1) await testActor.assertInLoop(loop1) await testActor.assertNotInLoop(loop2) - #endif } func testBasicExecutorFitsOnEventLoop_MTELG() async throws { From 19da487ef29a8227745834e946f3203dd0c08cf2 Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Thu, 17 Oct 2024 14:46:55 +0100 Subject: [PATCH 04/20] clean up 5.8 thresholds (#2932) clean up 5.8 thresholds ### Motivation: GitHub's merge retained files we don't need now we've dropped 5.8 ### Modifications: remove 5.8 thresholds ### Result: no unused files --- .../Thresholds/5.8/NIOCoreBenchmarks.WaitOnPromise.p90.json | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 Benchmarks/Thresholds/5.8/NIOCoreBenchmarks.WaitOnPromise.p90.json diff --git a/Benchmarks/Thresholds/5.8/NIOCoreBenchmarks.WaitOnPromise.p90.json b/Benchmarks/Thresholds/5.8/NIOCoreBenchmarks.WaitOnPromise.p90.json deleted file mode 100644 index eb79e76371..0000000000 --- a/Benchmarks/Thresholds/5.8/NIOCoreBenchmarks.WaitOnPromise.p90.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "mallocCountTotal" : 6000, - "memoryLeaked" : 0 -} From ff98c93fe95c65cce6d160605b8e2efcfbe65f8e Mon Sep 17 00:00:00 2001 From: Franz Busch Date: Fri, 18 Oct 2024 17:59:17 +0200 Subject: [PATCH 05/20] Fix `EventLoopFuture` and `EventLoopPromise` under strict concurrency checking (#2654) # Motivation We need to tackle the remaining strict concurrency checking related `Sendable` warnings in NIO. The first place to start is making sure that `EventLoopFuture` and `EventLoopPromise` are properly annotated. # Modification In a previous https://github.com/apple/swift-nio/pull/2496, @weissi changed the `@unchecked Sendable` conformances of `EventLoopFuture/Promise` to be conditional on the sendability of the generic `Value` type. After having looked at all the APIs on the future and promise types as well as reading the latest Concurrency evolution proposals, specifically the [Region based Isolation](https://github.com/apple/swift-evolution/blob/main/proposals/0414-region-based-isolation.md), I came to the conclusion that the previous `@unchecked Sendable` annotations were correct. The reasoning for this is: 1. An `EventLoopPromise` and `EventLoopFuture` pair are tied to a specific `EventLoop` 2. An `EventLoop` represents an isolation region and values tied to its isolation are not allowed to be shared outside of it unless they are disconnected from the region 3. The `value` used to succeed a promise often come from outside the isolation domain of the `EventLoop` hence they must be transferred into the promise. 4. The isolation region of the event loop is enforced through `@Sendable` annotations on all closures that receive the value in some kind of transformation e.g. `map()` or `whenComplete()` 5. Any method on `EventLoopFuture` that combines itself with another future must require `Sendable` of the other futures `Value` since we cannot statically enforce that futures are bound to the same event loop i.e. to the same isolation domain Due to the above rules, this PR adds back the `@unchecked Sendable` conformances to both types. Furthermore, this PR revisits every single method on `EventLoopPromise/Future` and adds missing `Sendable` and `@Sendable` annotation where necessary to uphold the above rules. A few important things to call out: - Since `transferring` is currently not available this PR requires a `Sendable` conformance for some methods on `EventLoopPromise/Future` that should rather take a `transffering` argument - To enable the common case where a value from the same event loop is used to succeed a promise I added two additional methods that take a `eventLoopBoundResult` and enforce dynamic isolation checking. We might have to do this for more methods once we adopt those changes in other targets/packages. # Result After this PR has landed our lowest level building block should be inline with what the rest of the language enforces in Concurrency. The `EventLoopFuture.swift` produces no more warnings under strict concurrency checking on the latest 5.10 snapshots. --------- Co-authored-by: George Barnett Co-authored-by: Cory Benfield --- Sources/NIOCore/AsyncAwaitSupport.swift | 11 +- Sources/NIOCore/ChannelPipeline.swift | 12 +- .../NIOCore/DispatchQueue+WithFuture.swift | 5 +- Sources/NIOCore/Docs.docc/index.md | 1 + .../Docs.docc/loops-futures-concurrency.md | 176 +++++++ Sources/NIOCore/EventLoop+Deprecated.swift | 3 +- Sources/NIOCore/EventLoop.swift | 61 +-- .../NIOCore/EventLoopFuture+Deprecated.swift | 30 +- .../EventLoopFuture+WithEventLoop.swift | 9 +- Sources/NIOCore/EventLoopFuture.swift | 472 ++++++++++++------ Sources/NIOCore/NIOScheduledCallback.swift | 9 +- .../NIOEmbedded/AsyncTestingEventLoop.swift | 6 +- Sources/NIOEmbedded/Embedded.swift | 3 +- .../AsyncChannel/AsyncChannelTests.swift | 4 +- Tests/NIOPosixTests/EventLoopFutureTest.swift | 8 +- .../NIOPosixTests/NonBlockingFileIOTest.swift | 41 +- 16 files changed, 612 insertions(+), 239 deletions(-) create mode 100644 Sources/NIOCore/Docs.docc/loops-futures-concurrency.md diff --git a/Sources/NIOCore/AsyncAwaitSupport.swift b/Sources/NIOCore/AsyncAwaitSupport.swift index 5cc6b6ace3..e8018d6e08 100644 --- a/Sources/NIOCore/AsyncAwaitSupport.swift +++ b/Sources/NIOCore/AsyncAwaitSupport.swift @@ -18,8 +18,9 @@ extension EventLoopFuture { /// This function can be used to bridge an `EventLoopFuture` into the `async` world. Ie. if you're in an `async` /// function and want to get the result of this future. @available(macOS 10.15, iOS 13, tvOS 13, watchOS 6, *) + @preconcurrency @inlinable - public func get() async throws -> Value { + public func get() async throws -> Value where Value: Sendable { try await withUnsafeThrowingContinuation { (cont: UnsafeContinuation, Error>) in self.whenComplete { result in switch result { @@ -62,8 +63,11 @@ extension EventLoopPromise { /// - returns: A `Task` which was created to `await` the `body`. @available(macOS 10.15, iOS 13, tvOS 13, watchOS 6, *) @discardableResult + @preconcurrency @inlinable - public func completeWithTask(_ body: @escaping @Sendable () async throws -> Value) -> Task { + public func completeWithTask( + _ body: @escaping @Sendable () async throws -> Value + ) -> Task where Value: Sendable { Task { do { let value = try await body() @@ -396,8 +400,9 @@ struct AsyncSequenceFromIterator: AsyncSeq @available(macOS 10.15, iOS 13, tvOS 13, watchOS 6, *) extension EventLoop { + @preconcurrency @inlinable - public func makeFutureWithTask( + public func makeFutureWithTask( _ body: @Sendable @escaping () async throws -> Return ) -> EventLoopFuture { let promise = self.makePromise(of: Return.self) diff --git a/Sources/NIOCore/ChannelPipeline.swift b/Sources/NIOCore/ChannelPipeline.swift index b3c0ef580f..b2c90c142d 100644 --- a/Sources/NIOCore/ChannelPipeline.swift +++ b/Sources/NIOCore/ChannelPipeline.swift @@ -457,10 +457,10 @@ public final class ChannelPipeline: ChannelInvoker { let promise = self.eventLoop.makePromise(of: ChannelHandlerContext.self) if self.eventLoop.inEventLoop { - promise.completeWith(self.contextSync(handler: handler)) + promise.assumeIsolated().completeWith(self.contextSync(handler: handler)) } else { self.eventLoop.execute { - promise.completeWith(self.contextSync(handler: handler)) + promise.assumeIsolated().completeWith(self.contextSync(handler: handler)) } } @@ -486,10 +486,10 @@ public final class ChannelPipeline: ChannelInvoker { let promise = self.eventLoop.makePromise(of: ChannelHandlerContext.self) if self.eventLoop.inEventLoop { - promise.completeWith(self.contextSync(name: name)) + promise.assumeIsolated().completeWith(self.contextSync(name: name)) } else { self.eventLoop.execute { - promise.completeWith(self.contextSync(name: name)) + promise.assumeIsolated().completeWith(self.contextSync(name: name)) } } @@ -519,10 +519,10 @@ public final class ChannelPipeline: ChannelInvoker { let promise = self.eventLoop.makePromise(of: ChannelHandlerContext.self) if self.eventLoop.inEventLoop { - promise.completeWith(self._contextSync(handlerType: handlerType)) + promise.assumeIsolated().completeWith(self._contextSync(handlerType: handlerType)) } else { self.eventLoop.execute { - promise.completeWith(self._contextSync(handlerType: handlerType)) + promise.assumeIsolated().completeWith(self._contextSync(handlerType: handlerType)) } } diff --git a/Sources/NIOCore/DispatchQueue+WithFuture.swift b/Sources/NIOCore/DispatchQueue+WithFuture.swift index 593380aa26..ec9bc03b49 100644 --- a/Sources/NIOCore/DispatchQueue+WithFuture.swift +++ b/Sources/NIOCore/DispatchQueue+WithFuture.swift @@ -29,9 +29,10 @@ extension DispatchQueue { /// - callbackMayBlock: The scheduled callback for the IO / task. /// - returns a new `EventLoopFuture` with value returned by the `block` parameter. @inlinable - public func asyncWithFuture( + @preconcurrency + public func asyncWithFuture( eventLoop: EventLoop, - _ callbackMayBlock: @escaping () throws -> NewValue + _ callbackMayBlock: @escaping @Sendable () throws -> NewValue ) -> EventLoopFuture { let promise = eventLoop.makePromise(of: NewValue.self) diff --git a/Sources/NIOCore/Docs.docc/index.md b/Sources/NIOCore/Docs.docc/index.md index 65e595732d..13d76894ec 100644 --- a/Sources/NIOCore/Docs.docc/index.md +++ b/Sources/NIOCore/Docs.docc/index.md @@ -15,6 +15,7 @@ More specialized modules provide concrete implementations of many of the abstrac - - +- ### Event Loops and Event Loop Groups diff --git a/Sources/NIOCore/Docs.docc/loops-futures-concurrency.md b/Sources/NIOCore/Docs.docc/loops-futures-concurrency.md new file mode 100644 index 0000000000..b5cbffa4f1 --- /dev/null +++ b/Sources/NIOCore/Docs.docc/loops-futures-concurrency.md @@ -0,0 +1,176 @@ +# EventLoops, EventLoopFutures, and Swift Concurrency + +This article aims to communicate how NIO's ``EventLoop``s and ``EventLoopFuture``s interact with the Swift 6 +concurrency model, particularly regarding data-race safety. It aims to be a reference for writing correct +concurrent code in the NIO model. + +NIO predates the Swift concurrency model. As a result, several of NIO's concepts are not perfect matches to +the concepts that Swift uses, or have overlapping responsibilities. + +## Isolation domains and executors + +First, a quick recap. The core of Swift 6's data-race safety protection is the concept of an "isolation +domain". Some valuable reading regarding the concept can be found in +[SE-0414 (Region based isolation)](https://github.com/swiftlang/swift-evolution/blob/main/proposals/0414-region-based-isolation.md) +but at a high level an isolation domain can be understood to be a collection of state and methods within which there cannot be +multiple executors executing code at the same time. + +In standard Swift Concurrency, the main boundaries of isolation domains are actors and tasks. Each actor, +including global actors, defines an isolation domain. Additionally, for functions and methods that are +not isolated to an actor, the `Task` within which that code executes defines an isolation domain. Passing +values between these isolation domains requires that these values are either `Sendable` (safe to hold in +multiple domains), or that the `sending` keyword is used to force the value to be passed from one domain +to another. + +A related concept to an "isolation domain" is an "executor". Again, useful reading can be found in +[SE-0392 (Custom actor executors)](https://github.com/swiftlang/swift-evolution/blob/main/proposals/0392-custom-actor-executors.md). +At a high level, an executor is simply an object that is capable of executing Swift `Task`s. Executors can be +concurrent, or they can be serial. Serial executors are the most common, as they can be used to back an +actor. + +## Event Loops + +NIO's core execution primitive is the ``EventLoop``. An ``EventLoop`` is fundamentally nothing more than +a Swift Concurrency Serial Executor that can also perform I/O operations directly. Indeed, NIO's +``EventLoop``s can be exposed as serial executors, using ``EventLoop/executor``. This provides a mechanism +to protect actor-isolated state using a NIO event-loop. With [the introduction of task executors](https://github.com/swiftlang/swift-evolution/blob/main/proposals/0417-task-executor-preference.md), +future versions of SwiftNIO will also be able to offer their event loops for individual `Task`s to execute +on as well. + +In a Swift 6 world, it is possible that these would be the API that NIO offered to execute tasks on the +loop. However, as NIO predates Swift 6, it also offers its own set of APIs to enqueue work. This includes +(but is not limited to): + +- ``EventLoop/execute(_:)`` +- ``EventLoop/submit(_:)`` +- ``EventLoop/scheduleTask(in:_:)`` +- ``EventLoop/scheduleRepeatedTask(initialDelay:delay:notifying:_:)`` +- ``EventLoop/scheduleCallback(at:handler:)-2xm6l`` + +The existence of these APIs requires us to also ask the question of where the submitted code executes. The +answer is that the submitted code executes on the event loop (or, in Swift Concurrency terms, on the +executor provided by the event loop). + +As the event loop only ever executes a single item of work (either an `async` function or one of the +closures above) at a time, it is a _serial_ executor. It also provides an _isolation domain_: code +submitted to a given `EventLoop` never runs in parallel with other code submitted to the same loop. + +The result here is that a all closures passed into the event loop to do work must be transferred +in: they may not be kept hold of outside of the event loop. That means they must be sent using +the `sending` keyword. + +> Note: As of the current 2.75.0 release, NIO enforces the stricter requirement that these closures + are `@Sendable`. This is not a long-term position, but reflects the need to continue + to support Swift 5 code which requires this stricter standard. In a future release of + SwiftNIO we expect to relax this constraint: if you need this relaxed constraint + then please file an issue. + +## Event loop futures + +In Swift NIO the most common mechanism to arrange a series of asynchronous work items is +_not_ to queue up a series of ``EventLoop/execute(_:)`` calls. Instead, users typically +use ``EventLoopFuture``. + +``EventLoopFuture`` has some extensive semantics documented in its API documentation. The +most important principal for this discussion is that all callbacks added to an +``EventLoopFuture`` will execute on the ``EventLoop`` to which that ``EventLoopFuture`` is +bound. By extension, then, all callbacks added to an ``EventLoopFuture`` execute on the same +executor (the ``EventLoop``) in the same isolation domain. + +The analogy to an actor here is hopefully fairly clear. Conceptually, an ``EventLoopFuture`` +could be modelled as an actor. That means all the callbacks have the same logical semantics: +the ``EventLoopFuture`` uses the isolation domain of its associated ``EventLoop``, and all +the callbacks are `sent` into the isolation domain. To that end, all the callback-taking APIs +require that the callback is sent using `sending` into the ``EventLoopFuture``. + +> Note: As of the current 2.75.0 release, NIO enforces the stricter requirement that these callbacks + are `@Sendable`. This is not a long-term position, but reflects the need to continue + to support Swift 5 code which requires this stricter standard. In a future release of + SwiftNIO we expect to relax this constraint: if you need this relaxed constraint + then please file an issue. + +Unlike ``EventLoop``s, however, ``EventLoopFuture``s also have value-receiving and value-taking +APIs. This is because ``EventLoopFuture``s pass a value along to their various callbacks, and +so need to be both given an initial value (via an ``EventLoopPromise``) and in some cases to +extract that value from the ``EventLoopFuture`` wrapper. + +This implies that ``EventLoopPromise``'s various success functions +(_and_ ``EventLoop/makeSucceededFuture(_:)``) need to take their value as `sending`. The value +is potentially sent from its current isolation domain into the ``EventLoop``, which will require +that the value is safe to move. + +> Note: As of the current 2.75.0 release, NIO enforces the stricter requirement that these values + are `Sendable`. This is not a long-term position, but reflects the need to continue + to support Swift 5 code which requires this stricter standard. In a future release of + SwiftNIO we expect to relax this constraint: if you need this relaxed constraint + then please file an issue. + +There are also a few ways to extract a value, such as ``EventLoopFuture/wait(file:line:)`` +and ``EventLoopFuture/get()``. These APIs can only safely be called when the ``EventLoopFuture`` +is carrying a `Sendable` value. This is because ``EventLoopFuture``s hold on to their value and +can give it to other closures or other callers of `get` and `wait`. Thus, `sending` is not +sufficient. + +## Combining Futures + +NIO provides a number of APIs for combining futures, such as ``EventLoopFuture/and(_:)``. +This potentially represents an issue, as two futures may not share the same isolation domain. +As a result, we can only safely call these combining functions when the ``EventLoopFuture`` +values are `Sendable`. + +> Note: We can conceptually relax this constraint somewhat by offering equivalent + functions that can only safely be called when all the combined futures share the + same bound event loop: that is, when they are all within the same isolation domain. + + This can be enforced with runtime isolation checks. If you have a need for these + functions, please reach out to the NIO team. + +## Interacting with Futures on the Event Loop + +In a number of contexts (such as in ``ChannelHandler``s), the programmer has static knowledge +that they are within an isolation domain. That isolation domain may well be shared with the +isolation domain of many futures and promises with which they interact. For example, +futures that are provided from ``ChannelHandlerContext/write(_:promise:)`` will be bound to +the event loop on which the ``ChannelHandler`` resides. + +In this context, the `sending` constraint is unnecessarily strict. The future callbacks are +guaranteed to fire on the same isolation domain as the ``ChannelHandlerContext``: no risk +of data race is present. However, Swift Concurrency cannot guarantee this at compile time, +as the specific isolation domain is determined only at runtime. + +In these contexts, today users can make their callbacks safe using ``NIOLoopBound`` and +``NIOLoopBoundBox``. These values can only be constructed on the event loop, and only allow +access to their values on the same event loop. These constraints are enforced at runtime, +so at compile time these are unconditionally `Sendable`. + +> Warning: ``NIOLoopBound`` and ``NIOLoopBoundBox`` replace compile-time isolation checks + with runtime ones. This makes it possible to introduce crashes in your code. Please + ensure that you are 100% confident that the isolation domains align. If you are not + sure that the ``EventLoopFuture`` you wish to attach a callback to is bound to your + ``EventLoop``, use ``EventLoopFuture/hop(to:)`` to move it to your isolation domain + before using these types. + +> Note: In a future NIO release we intend to improve the ergonomics of this common problem + by offering a related type that can only be created from an ``EventLoopFuture`` on a + given ``EventLoop``. This minimises the number of runtime checks, and will make it + easier and more pleasant to write this kind of code. + +## Interacting with Event Loops on the Event Loop + +As with Futures, there are occasionally times where it is necessary to schedule +``EventLoop`` operations on the ``EventLoop`` where your code is currently executing. + +Much like with ``EventLoopFuture``, you can use ``NIOLoopBound`` and ``NIOLoopBoundBox`` +to make these callbacks safe. + +> Warning: ``NIOLoopBound`` and ``NIOLoopBoundBox`` replace compile-time isolation checks + with runtime ones. This makes it possible to introduce crashes in your code. Please + ensure that you are 100% confident that the isolation domains align. If you are not + sure that the ``EventLoopFuture`` you wish to attach a callback to is bound to your + ``EventLoop``, use ``EventLoopFuture/hop(to:)`` to move it to your isolation domain + before using these types. + +> Note: In a future NIO release we intend to improve the ergonomics of this common problem + by offering a related type that can only be created from an ``EventLoopFuture`` on a + given ``EventLoop``. This minimises the number of runtime checks, and will make it + easier and more pleasant to write this kind of code. diff --git a/Sources/NIOCore/EventLoop+Deprecated.swift b/Sources/NIOCore/EventLoop+Deprecated.swift index e2321ceb74..dc3d356b5e 100644 --- a/Sources/NIOCore/EventLoop+Deprecated.swift +++ b/Sources/NIOCore/EventLoop+Deprecated.swift @@ -23,9 +23,10 @@ extension EventLoop { self.makeFailedFuture(error) } + @preconcurrency @inlinable @available(*, deprecated, message: "Please don't pass file:line:, there's no point.") - public func makeSucceededFuture( + public func makeSucceededFuture( _ value: Success, file: StaticString = #fileID, line: UInt = #line diff --git a/Sources/NIOCore/EventLoop.swift b/Sources/NIOCore/EventLoop.swift index 4e76abe821..5732a989be 100644 --- a/Sources/NIOCore/EventLoop.swift +++ b/Sources/NIOCore/EventLoop.swift @@ -60,7 +60,7 @@ public struct Scheduled { } } -extension Scheduled: Sendable where T: Sendable {} +extension Scheduled: Sendable {} /// Returned once a task was scheduled to be repeatedly executed on the `EventLoop`. /// @@ -368,20 +368,22 @@ public protocol EventLoop: EventLoopGroup { /// /// - NOTE: Event loops that provide a custom scheduled callback implementation **must** also implement /// `cancelScheduledCallback`. Failure to do so will result in a runtime error. + @preconcurrency @discardableResult func scheduleCallback( at deadline: NIODeadline, - handler: some NIOScheduledCallbackHandler + handler: some (NIOScheduledCallbackHandler & Sendable) ) throws -> NIOScheduledCallback /// Schedule a callback after given time. /// /// - NOTE: Event loops that provide a custom scheduled callback implementation **must** also implement /// `cancelScheduledCallback`. Failure to do so will result in a runtime error. + @preconcurrency @discardableResult func scheduleCallback( in amount: TimeAmount, - handler: some NIOScheduledCallbackHandler + handler: some (NIOScheduledCallbackHandler & Sendable) ) throws -> NIOScheduledCallback /// Cancel a scheduled callback. @@ -749,13 +751,7 @@ extension EventLoop { /// - returns: An `EventLoopFuture` containing the result of `task`'s execution. @inlinable @preconcurrency - public func submit(_ task: @escaping @Sendable () throws -> T) -> EventLoopFuture { - _submit(task) - } - @usableFromInline typealias SubmitCallback = @Sendable () throws -> T - - @inlinable - func _submit(_ task: @escaping SubmitCallback) -> EventLoopFuture { + public func submit(_ task: @escaping @Sendable () throws -> T) -> EventLoopFuture { let promise: EventLoopPromise = makePromise(file: #fileID, line: #line) self.execute { @@ -779,18 +775,15 @@ extension EventLoop { /// - returns: An `EventLoopFuture` identical to the `EventLoopFuture` returned from `task`. @inlinable @preconcurrency - public func flatSubmit(_ task: @escaping @Sendable () -> EventLoopFuture) -> EventLoopFuture { - self._flatSubmit(task) - } - @usableFromInline typealias FlatSubmitCallback = @Sendable () -> EventLoopFuture - - @inlinable - func _flatSubmit(_ task: @escaping FlatSubmitCallback) -> EventLoopFuture { + public func flatSubmit(_ task: @escaping @Sendable () -> EventLoopFuture) -> EventLoopFuture { self.submit(task).flatMap { $0 } } /// Schedule a `task` that is executed by this `EventLoop` at the given time. /// + /// - Note: The `T` must be `Sendable` since the isolation domains of the event loop future returned from `task` and + /// this event loop might differ. + /// /// - parameters: /// - task: The asynchronous task to run. As with everything that runs on the `EventLoop`, it must not block. /// - returns: A `Scheduled` object which may be used to cancel the task if it has not yet run, or to wait @@ -800,23 +793,11 @@ extension EventLoop { @discardableResult @inlinable @preconcurrency - public func flatScheduleTask( + public func flatScheduleTask( deadline: NIODeadline, file: StaticString = #fileID, line: UInt = #line, _ task: @escaping @Sendable () throws -> EventLoopFuture - ) -> Scheduled { - self._flatScheduleTask(deadline: deadline, file: file, line: line, task) - } - @usableFromInline typealias FlatScheduleTaskDeadlineCallback = () throws -> EventLoopFuture - - @discardableResult - @inlinable - func _flatScheduleTask( - deadline: NIODeadline, - file: StaticString, - line: UInt, - _ task: @escaping FlatScheduleTaskDelayCallback ) -> Scheduled { let promise: EventLoopPromise = self.makePromise(file: file, line: line) let scheduled = self.scheduleTask(deadline: deadline, task) @@ -827,6 +808,9 @@ extension EventLoop { /// Schedule a `task` that is executed by this `EventLoop` after the given amount of time. /// + /// - Note: The `T` must be `Sendable` since the isolation domains of the event loop future returned from `task` and + /// this event loop might differ. + /// /// - parameters: /// - task: The asynchronous task to run. As everything that runs on the `EventLoop`, it must not block. /// - returns: A `Scheduled` object which may be used to cancel the task if it has not yet run, or to wait @@ -836,7 +820,7 @@ extension EventLoop { @discardableResult @inlinable @preconcurrency - public func flatScheduleTask( + public func flatScheduleTask( in delay: TimeAmount, file: StaticString = #fileID, line: UInt = #line, @@ -848,7 +832,7 @@ extension EventLoop { @usableFromInline typealias FlatScheduleTaskDelayCallback = @Sendable () throws -> EventLoopFuture @inlinable - func _flatScheduleTask( + func _flatScheduleTask( in delay: TimeAmount, file: StaticString, line: UInt, @@ -886,8 +870,9 @@ extension EventLoop { /// - parameters: /// - result: the value that is used by the `EventLoopFuture`. /// - returns: a succeeded `EventLoopFuture`. + @preconcurrency @inlinable - public func makeSucceededFuture(_ value: Success) -> EventLoopFuture { + public func makeSucceededFuture(_ value: Success) -> EventLoopFuture { if Success.self == Void.self { // The as! will always succeed because we previously checked that Success.self == Void.self. return self.makeSucceededVoidFuture() as! EventLoopFuture @@ -901,8 +886,9 @@ extension EventLoop { /// - Parameters: /// - result: The value that is used by the `EventLoopFuture` /// - Returns: A completed `EventLoopFuture`. + @preconcurrency @inlinable - public func makeCompletedFuture(_ result: Result) -> EventLoopFuture { + public func makeCompletedFuture(_ result: Result) -> EventLoopFuture { switch result { case .success(let value): return self.makeSucceededFuture(value) @@ -916,8 +902,11 @@ extension EventLoop { /// - Parameters: /// - body: The function that is used to complete the `EventLoopFuture` /// - Returns: A completed `EventLoopFuture`. + @preconcurrency @inlinable - public func makeCompletedFuture(withResultOf body: () throws -> Success) -> EventLoopFuture { + public func makeCompletedFuture( + withResultOf body: () throws -> Success + ) -> EventLoopFuture { let trans = Result(catching: body) return self.makeCompletedFuture(trans) } @@ -999,7 +988,7 @@ extension EventLoop { notifying promise: EventLoopPromise?, _ task: @escaping ScheduleRepeatedTaskCallback ) -> RepeatedTask { - let futureTask: (RepeatedTask) -> EventLoopFuture = { repeatedTask in + let futureTask: @Sendable (RepeatedTask) -> EventLoopFuture = { repeatedTask in do { try task(repeatedTask) return self.makeSucceededFuture(()) diff --git a/Sources/NIOCore/EventLoopFuture+Deprecated.swift b/Sources/NIOCore/EventLoopFuture+Deprecated.swift index 6883c9ffd9..2e48e7c798 100644 --- a/Sources/NIOCore/EventLoopFuture+Deprecated.swift +++ b/Sources/NIOCore/EventLoopFuture+Deprecated.swift @@ -13,22 +13,24 @@ //===----------------------------------------------------------------------===// extension EventLoopFuture { + @preconcurrency @inlinable @available(*, deprecated, message: "Please don't pass file:line:, there's no point.") - public func flatMap( + public func flatMap( file: StaticString = #fileID, line: UInt = #line, - _ callback: @escaping (Value) -> EventLoopFuture + _ callback: @escaping @Sendable (Value) -> EventLoopFuture ) -> EventLoopFuture { self.flatMap(callback) } + @preconcurrency @inlinable @available(*, deprecated, message: "Please don't pass file:line:, there's no point.") - public func flatMapThrowing( + public func flatMapThrowing( file: StaticString = #fileID, line: UInt = #line, - _ callback: @escaping (Value) throws -> NewValue + _ callback: @escaping @Sendable (Value) throws -> NewValue ) -> EventLoopFuture { self.flatMapThrowing(callback) } @@ -38,7 +40,7 @@ extension EventLoopFuture { public func flatMapErrorThrowing( file: StaticString = #fileID, line: UInt = #line, - _ callback: @escaping (Error) throws -> Value + _ callback: @escaping @Sendable (Error) throws -> Value ) -> EventLoopFuture { self.flatMapErrorThrowing(callback) } @@ -48,7 +50,7 @@ extension EventLoopFuture { public func map( file: StaticString = #fileID, line: UInt = #line, - _ callback: @escaping (Value) -> (NewValue) + _ callback: @escaping @Sendable (Value) -> (NewValue) ) -> EventLoopFuture { self.map(callback) } @@ -58,34 +60,37 @@ extension EventLoopFuture { public func flatMapError( file: StaticString = #fileID, line: UInt = #line, - _ callback: @escaping (Error) -> EventLoopFuture - ) -> EventLoopFuture { + _ callback: @escaping @Sendable (Error) -> EventLoopFuture + ) -> EventLoopFuture where Value: Sendable { self.flatMapError(callback) } + @preconcurrency @inlinable @available(*, deprecated, message: "Please don't pass file:line:, there's no point.") public func flatMapResult( file: StaticString = #fileID, line: UInt = #line, - _ body: @escaping (Value) -> Result + _ body: @escaping @Sendable (Value) -> Result ) -> EventLoopFuture { self.flatMapResult(body) } + @preconcurrency @inlinable @available(*, deprecated, message: "Please don't pass file:line:, there's no point.") public func recover( file: StaticString = #fileID, line: UInt = #line, - _ callback: @escaping (Error) -> Value + _ callback: @escaping @Sendable (Error) -> Value ) -> EventLoopFuture { self.recover(callback) } + @preconcurrency @inlinable @available(*, deprecated, message: "Please don't pass file:line:, there's no point.") - public func and( + public func and( _ other: EventLoopFuture, file: StaticString = #fileID, line: UInt = #line @@ -93,9 +98,10 @@ extension EventLoopFuture { self.and(other) } + @preconcurrency @inlinable @available(*, deprecated, message: "Please don't pass file:line:, there's no point.") - public func and( + public func and( value: OtherValue, file: StaticString = #fileID, line: UInt = #line diff --git a/Sources/NIOCore/EventLoopFuture+WithEventLoop.swift b/Sources/NIOCore/EventLoopFuture+WithEventLoop.swift index bf76a0e97e..48e2b526f6 100644 --- a/Sources/NIOCore/EventLoopFuture+WithEventLoop.swift +++ b/Sources/NIOCore/EventLoopFuture+WithEventLoop.swift @@ -41,7 +41,7 @@ extension EventLoopFuture { /// - returns: A future that will receive the eventual value. @inlinable @preconcurrency - public func flatMapWithEventLoop( + public func flatMapWithEventLoop( _ callback: @escaping @Sendable (Value, EventLoop) -> EventLoopFuture ) -> EventLoopFuture { let next = EventLoopPromise.makeUnleakablePromise(eventLoop: self.eventLoop) @@ -79,7 +79,7 @@ extension EventLoopFuture { @preconcurrency public func flatMapErrorWithEventLoop( _ callback: @escaping @Sendable (Error, EventLoop) -> EventLoopFuture - ) -> EventLoopFuture { + ) -> EventLoopFuture where Value: Sendable { let next = EventLoopPromise.makeUnleakablePromise(eventLoop: self.eventLoop) self._whenComplete { [eventLoop = self.eventLoop] in switch self._value! { @@ -118,10 +118,11 @@ extension EventLoopFuture { /// - returns: A new `EventLoopFuture` with the folded value whose callbacks run on `self.eventLoop`. @inlinable @preconcurrency - public func foldWithEventLoop( + public func foldWithEventLoop( _ futures: [EventLoopFuture], with combiningFunction: @escaping @Sendable (Value, OtherValue, EventLoop) -> EventLoopFuture - ) -> EventLoopFuture { + ) -> EventLoopFuture where Value: Sendable { + @Sendable func fold0(eventLoop: EventLoop) -> EventLoopFuture { let body = futures.reduce(self) { (f1: EventLoopFuture, f2: EventLoopFuture) -> EventLoopFuture in diff --git a/Sources/NIOCore/EventLoopFuture.swift b/Sources/NIOCore/EventLoopFuture.swift index a6aee1b2c1..d72fe23c33 100644 --- a/Sources/NIOCore/EventLoopFuture.swift +++ b/Sources/NIOCore/EventLoopFuture.swift @@ -26,7 +26,7 @@ import Dispatch /// In particular, note that _run() here continues to obtain and execute lists of callbacks until it completes. /// This eliminates recursion when processing `flatMap()` chains. @usableFromInline -internal struct CallbackList { +internal struct CallbackList: Sendable { @usableFromInline internal typealias Element = @Sendable () -> CallbackList @usableFromInline @@ -183,8 +183,9 @@ public struct EventLoopPromise { /// /// - parameters: /// - value: The successful result of the operation. + @preconcurrency @inlinable - public func succeed(_ value: Value) { + public func succeed(_ value: Value) where Value: Sendable { self._resolve(value: .success(value)) } @@ -194,7 +195,13 @@ public struct EventLoopPromise { /// - error: The error from the operation. @inlinable public func fail(_ error: Error) { - self._resolve(value: .failure(error)) + if self.futureResult.eventLoop.inEventLoop { + self.futureResult._setError(error)._run() + } else { + self.futureResult.eventLoop.execute { + self.futureResult._setError(error)._run() + } + } } /// Complete the promise with the passed in `EventLoopFuture`. @@ -202,11 +209,15 @@ public struct EventLoopPromise { /// This method is equivalent to invoking `future.cascade(to: promise)`, /// but sometimes may read better than its cascade counterpart. /// + /// - Note: The `Value` must be `Sendable` since the isolation domains of the passed future and this promise might differ i.e. + /// they might be bound to different event loops. + /// /// - parameters: /// - future: The future whose value will be used to succeed or fail this promise. /// - seealso: `EventLoopFuture.cascade(to:)` + @preconcurrency @inlinable - public func completeWith(_ future: EventLoopFuture) { + public func completeWith(_ future: EventLoopFuture) where Value: Sendable { future.cascade(to: self) } @@ -224,8 +235,9 @@ public struct EventLoopPromise { /// /// - parameters: /// - result: The result which will be used to succeed or fail this promise. + @preconcurrency @inlinable - public func completeWith(_ result: Result) { + public func completeWith(_ result: Result) where Value: Sendable { self._resolve(value: result) } @@ -238,7 +250,7 @@ public struct EventLoopPromise { /// - parameters: /// - value: The value to fire the future with. @inlinable - internal func _resolve(value: Result) { + internal func _resolve(value: Result) where Value: Sendable { if self.futureResult.eventLoop.inEventLoop { self._setValue(value: value)._run() } else { @@ -411,7 +423,7 @@ public final class EventLoopFuture { /// A EventLoopFuture that has already succeeded @inlinable - internal init(eventLoop: EventLoop, value: Value) { + internal init(eventLoop: EventLoop, value: Value) where Value: Sendable { self.eventLoop = eventLoop self._value = .success(value) self._callbacks = .init() @@ -471,13 +483,16 @@ extension EventLoopFuture { /// /// Note: In a sense, the `EventLoopFuture` is returned before it's created. /// + /// - Note: The `NewValue` must be `Sendable` since the isolation domains of this future and the future returned from the callback + /// might differ i.e. they might be bound to different event loops. + /// /// - parameters: /// - callback: Function that will receive the value of this `EventLoopFuture` and return /// a new `EventLoopFuture`. /// - returns: A future that will receive the eventual value. @inlinable @preconcurrency - public func flatMap( + public func flatMap( _ callback: @escaping @Sendable (Value) -> EventLoopFuture ) -> EventLoopFuture { self._flatMap(callback) @@ -485,7 +500,7 @@ extension EventLoopFuture { @usableFromInline typealias FlatMapCallback = @Sendable (Value) -> EventLoopFuture @inlinable - func _flatMap(_ callback: @escaping FlatMapCallback) -> EventLoopFuture { + func _flatMap(_ callback: @escaping FlatMapCallback) -> EventLoopFuture { let next = EventLoopPromise.makeUnleakablePromise(eventLoop: self.eventLoop) self._whenComplete { switch self._value! { @@ -516,6 +531,9 @@ extension EventLoopFuture { /// /// If your callback function throws, the returned `EventLoopFuture` will error. /// + /// - Note: The `NewValue` must be `Sendable` since the isolation domains of this future and the future returned from the callback + /// might differ i.e. they might be bound to different event loops. + /// /// - parameters: /// - callback: Function that will receive the value of this `EventLoopFuture` and return /// a new value lifted into a new `EventLoopFuture`. @@ -566,8 +584,9 @@ extension EventLoopFuture { /// - returns: A future that will receive the eventual value or a rethrown error. @inlinable @preconcurrency - public func flatMapErrorThrowing(_ callback: @escaping @Sendable (Error) throws -> Value) -> EventLoopFuture - { + public func flatMapErrorThrowing( + _ callback: @escaping @Sendable (Error) throws -> Value + ) -> EventLoopFuture { self._flatMapErrorThrowing(callback) } @usableFromInline typealias FlatMapErrorThrowingCallback = @Sendable (Error) throws -> Value @@ -619,13 +638,17 @@ extension EventLoopFuture { /// - returns: A future that will receive the eventual value. @inlinable @preconcurrency - public func map(_ callback: @escaping @Sendable (Value) -> (NewValue)) -> EventLoopFuture { + public func map( + _ callback: @escaping @Sendable (Value) -> (NewValue) + ) -> EventLoopFuture { self._map(callback) } @usableFromInline typealias MapCallback = @Sendable (Value) -> (NewValue) @inlinable - func _map(_ callback: @escaping MapCallback) -> EventLoopFuture { + func _map( + _ callback: @escaping @Sendable (Value) -> (NewValue) + ) -> EventLoopFuture { if NewValue.self == Value.self && NewValue.self == Void.self { self.whenSuccess(callback as! @Sendable (Value) -> Void) return self as! EventLoopFuture @@ -645,6 +668,9 @@ extension EventLoopFuture { /// /// If the callback cannot recover it should return a failed `EventLoopFuture`. /// + /// - Note: The `Value` must be `Sendable` since the isolation domains of this future and the future returned from the callback + /// might differ i.e. they might be bound to different event loops. + /// /// - parameters: /// - callback: Function that will receive the error value of this `EventLoopFuture` and return /// a new value lifted into a new `EventLoopFuture`. @@ -653,13 +679,7 @@ extension EventLoopFuture { @preconcurrency public func flatMapError( _ callback: @escaping @Sendable (Error) -> EventLoopFuture - ) -> EventLoopFuture { - self._flatMapError(callback) - } - @usableFromInline typealias FlatMapErrorCallback = @Sendable (Error) -> EventLoopFuture - - @inlinable - func _flatMapError(_ callback: @escaping FlatMapErrorCallback) -> EventLoopFuture { + ) -> EventLoopFuture where Value: Sendable { let next = EventLoopPromise.makeUnleakablePromise(eventLoop: self.eventLoop) self._whenComplete { switch self._value! { @@ -740,12 +760,6 @@ extension EventLoopFuture { @inlinable @preconcurrency public func recover(_ callback: @escaping @Sendable (Error) -> Value) -> EventLoopFuture { - self._recover(callback) - } - @usableFromInline typealias RecoverCallback = @Sendable (Error) -> Value - - @inlinable - func _recover(_ callback: @escaping RecoverCallback) -> EventLoopFuture { let next = EventLoopPromise.makeUnleakablePromise(eventLoop: self.eventLoop) self._whenComplete { switch self._value! { @@ -758,10 +772,9 @@ extension EventLoopFuture { return next.futureResult } - @usableFromInline typealias AddCallbackCallback = @Sendable () -> CallbackList /// Add a callback. If there's already a value, invoke it and return the resulting list of new callback functions. @inlinable - internal func _addCallback(_ callback: @escaping AddCallbackCallback) -> CallbackList { + internal func _addCallback(_ callback: @escaping @Sendable () -> CallbackList) -> CallbackList { self.eventLoop.assertInEventLoop() if self._value == nil { self._callbacks.append(callback) @@ -777,11 +790,10 @@ extension EventLoopFuture { internal func _whenComplete(_ callback: @escaping @Sendable () -> CallbackList) { self._internalWhenComplete(callback) } - @usableFromInline typealias InternalWhenCompleteCallback = @Sendable () -> CallbackList /// Add a callback. If there's already a value, run as much of the chain as we can. @inlinable - internal func _internalWhenComplete(_ callback: @escaping InternalWhenCompleteCallback) { + internal func _internalWhenComplete(_ callback: @escaping @Sendable () -> CallbackList) { if self.eventLoop.inEventLoop { self._addCallback(callback)._run() } else { @@ -804,12 +816,6 @@ extension EventLoopFuture { @inlinable @preconcurrency public func whenSuccess(_ callback: @escaping @Sendable (Value) -> Void) { - self._whenSuccess(callback) - } - @usableFromInline typealias WhenSuccessCallback = @Sendable (Value) -> Void - - @inlinable - func _whenSuccess(_ callback: @escaping WhenSuccessCallback) { self._whenComplete { if case .success(let t) = self._value! { callback(t) @@ -831,12 +837,6 @@ extension EventLoopFuture { @inlinable @preconcurrency public func whenFailure(_ callback: @escaping @Sendable (Error) -> Void) { - self._whenFailure(callback) - } - @usableFromInline typealias WhenFailureCallback = @Sendable (Error) -> Void - - @inlinable - func _whenFailure(_ callback: @escaping WhenFailureCallback) { self._whenComplete { if case .failure(let e) = self._value! { callback(e) @@ -853,11 +853,6 @@ extension EventLoopFuture { @inlinable @preconcurrency public func whenComplete(_ callback: @escaping @Sendable (Result) -> Void) { - self._publicWhenComplete(callback) - } - @usableFromInline typealias WhenCompleteCallback = @Sendable (Result) -> Void - @inlinable - func _publicWhenComplete(_ callback: @escaping WhenCompleteCallback) { self._whenComplete { callback(self._value!) return CallbackList() @@ -876,6 +871,21 @@ extension EventLoopFuture { } return CallbackList() } + + /// Internal: Set the value and return a list of callbacks that should be invoked as a result. + /// + /// We need a seperate method for setting the error to avoid Sendable checking of `Value` + @inlinable + internal func _setError(_ error: Error) -> CallbackList { + self.eventLoop.assertInEventLoop() + if self._value == nil { + self._value = .failure(error) + let callbacks = self._callbacks + self._callbacks = CallbackList() + return callbacks + } + return CallbackList() + } } // MARK: and @@ -885,8 +895,14 @@ extension EventLoopFuture { /// provided `EventLoopFuture` both succeed. It then provides the pair /// of results. If either one fails, the combined `EventLoopFuture` will fail with /// the first error encountered. + /// + /// - Note: The `NewValue` must be `Sendable` since the isolation domains of this future and the other future might differ i.e. + /// they might be bound to different event loops. + @preconcurrency @inlinable - public func and(_ other: EventLoopFuture) -> EventLoopFuture<(Value, OtherValue)> { + public func and( + _ other: EventLoopFuture + ) -> EventLoopFuture<(Value, OtherValue)> { let promise = EventLoopPromise<(Value, OtherValue)>.makeUnleakablePromise(eventLoop: self.eventLoop) let box: UnsafeMutableTransferBox<(t: Value?, u: OtherValue?)> = .init((nil, nil)) @@ -926,8 +942,11 @@ extension EventLoopFuture { /// Return a new EventLoopFuture that contains this "and" another value. /// This is just syntactic sugar for `future.and(loop.makeSucceedFuture(value))`. + @preconcurrency @inlinable - public func and(value: OtherValue) -> EventLoopFuture<(Value, OtherValue)> { + public func and( + value: OtherValue // TODO: This should be transferring + ) -> EventLoopFuture<(Value, OtherValue)> { self.and(EventLoopFuture(eventLoop: self.eventLoop, value: value)) } } @@ -954,10 +973,14 @@ extension EventLoopFuture { /// }.cascade(to: userPromise) /// ``` /// + /// - Note: The `Value` must be `Sendable` since the isolation domains of this future and the promise might differ i.e. + /// they might be bound to different event loops. + /// /// - Parameter to: The `EventLoopPromise` to fulfill with the results of this future. /// - SeeAlso: `EventLoopPromise.completeWith(_:)` + @preconcurrency @inlinable - public func cascade(to promise: EventLoopPromise?) { + public func cascade(to promise: EventLoopPromise?) where Value: Sendable { guard let promise = promise else { return } self.whenComplete { result in switch result { @@ -978,9 +1001,13 @@ extension EventLoopFuture { /// doWorkReturningInt().map({ $0 >= 0 }).cascade(to: boolPromise) /// ``` /// + /// - Note: The `Value` must be `Sendable` since the isolation domains of this future and the promise might differ i.e. + /// they might be bound to different event loops. + /// /// - Parameter to: The `EventLoopPromise` to fulfill when a successful result is available. + @preconcurrency @inlinable - public func cascadeSuccess(to promise: EventLoopPromise?) { + public func cascadeSuccess(to promise: EventLoopPromise?) where Value: Sendable { guard let promise = promise else { return } self.whenSuccess { promise.succeed($0) } } @@ -990,6 +1017,7 @@ extension EventLoopFuture { /// This is an alternative variant of `cascade` that allows you to potentially return early failures in /// error cases, while passing the user `EventLoopPromise` onwards. /// + /// /// - Parameter to: The `EventLoopPromise` that should fail with the error of this `EventLoopFuture`. @inlinable public func cascadeFailure(to promise: EventLoopPromise?) { @@ -1012,16 +1040,19 @@ extension EventLoopFuture { /// /// This is also forbidden in async contexts: prefer ``EventLoopFuture/get()``. /// + /// - Note: The `Value` must be `Sendable` since it is shared outside of the isolation domain of the event loop. + /// /// - returns: The value of the `EventLoopFuture` when it completes. /// - throws: The error value of the `EventLoopFuture` if it errors. @available(*, noasync, message: "wait() can block indefinitely, prefer get()", renamed: "get()") + @preconcurrency @inlinable - public func wait(file: StaticString = #file, line: UInt = #line) throws -> Value { + public func wait(file: StaticString = #file, line: UInt = #line) throws -> Value where Value: Sendable { try self._wait(file: file, line: line) } @inlinable - func _wait(file: StaticString, line: UInt) throws -> Value { + func _wait(file: StaticString, line: UInt) throws -> Value where Value: Sendable { self.eventLoop._preconditionSafeToWait(file: file, line: line) let v: UnsafeMutableTransferBox?> = .init(nil) @@ -1059,25 +1090,20 @@ extension EventLoopFuture { /// `EventLoopFuture` objects will no longer be waited for. This function therefore fails fast: once /// a failure is encountered, it will immediately fail the overall EventLoopFuture. /// + /// - Note: The `Value` and `NewValue` must be `Sendable` since the isolation domains of this future and the other futures might differ i.e. + /// they might be bound to different event loops. + /// /// - parameters: /// - futures: An array of `EventLoopFuture` to wait for. /// - with: A function that will be used to fold the values of two `EventLoopFuture`s and return a new value wrapped in an `EventLoopFuture`. /// - returns: A new `EventLoopFuture` with the folded value whose callbacks run on `self.eventLoop`. @inlinable @preconcurrency - public func fold( + public func fold( _ futures: [EventLoopFuture], with combiningFunction: @escaping @Sendable (Value, OtherValue) -> EventLoopFuture - ) -> EventLoopFuture { - self._fold(futures, with: combiningFunction) - } - @usableFromInline typealias FoldCallback = @Sendable (Value, OtherValue) -> EventLoopFuture - - @inlinable - func _fold( - _ futures: [EventLoopFuture], - with combiningFunction: @escaping FoldCallback - ) -> EventLoopFuture { + ) -> EventLoopFuture where Value: Sendable { + @Sendable func fold0() -> EventLoopFuture { let body = futures.reduce(self) { (f1: EventLoopFuture, f2: EventLoopFuture) -> EventLoopFuture in @@ -1120,6 +1146,9 @@ extension EventLoopFuture { /// `EventLoopFuture` objects will no longer be waited for. This function therefore fails fast: once /// a failure is encountered, it will immediately fail the overall `EventLoopFuture`. /// + /// - Note: The `Value` and `InputValue` must be `Sendable` since the isolation domains of this future and the other futures might differ i.e. + /// they might be bound to different event loops. + /// /// - parameters: /// - initialResult: An initial result to begin the reduction. /// - futures: An array of `EventLoopFuture` to wait for. @@ -1128,23 +1157,23 @@ extension EventLoopFuture { /// - returns: A new `EventLoopFuture` with the reduced value. @preconcurrency @inlinable - public static func reduce( + public static func reduce( _ initialResult: Value, _ futures: [EventLoopFuture], on eventLoop: EventLoop, _ nextPartialResult: @escaping @Sendable (Value, InputValue) -> Value - ) -> EventLoopFuture { + ) -> EventLoopFuture where Value: Sendable { Self._reduce(initialResult, futures, on: eventLoop, nextPartialResult) } @usableFromInline typealias ReduceCallback = @Sendable (Value, InputValue) -> Value @inlinable - static func _reduce( + static func _reduce( _ initialResult: Value, _ futures: [EventLoopFuture], on eventLoop: EventLoop, _ nextPartialResult: @escaping ReduceCallback - ) -> EventLoopFuture { + ) -> EventLoopFuture where Value: Sendable { let f0 = eventLoop.makeSucceededFuture(initialResult) let body = f0.fold(futures) { (t: Value, u: InputValue) -> EventLoopFuture in @@ -1165,6 +1194,9 @@ extension EventLoopFuture { /// `EventLoopFuture` objects will no longer be waited for. This function therefore fails fast: once /// a failure is encountered, it will immediately fail the overall `EventLoopFuture`. /// + /// - Note: The `Value` and `InputValue` must be `Sendable` since the isolation domains of this future and the other futures might differ i.e. + /// they might be bound to different event loops. + /// /// - parameters: /// - initialResult: An initial result to begin the reduction. /// - futures: An array of `EventLoopFuture` to wait for. @@ -1173,36 +1205,27 @@ extension EventLoopFuture { /// - returns: A new `EventLoopFuture` with the combined value. @inlinable @preconcurrency - public static func reduce( + public static func reduce( into initialResult: Value, _ futures: [EventLoopFuture], on eventLoop: EventLoop, _ updateAccumulatingResult: @escaping @Sendable (inout Value, InputValue) -> Void - ) -> EventLoopFuture { - Self._reduce(into: initialResult, futures, on: eventLoop, updateAccumulatingResult) - } - @usableFromInline typealias ReduceIntoCallback = @Sendable (inout Value, InputValue) -> Void - - @inlinable - static func _reduce( - into initialResult: Value, - _ futures: [EventLoopFuture], - on eventLoop: EventLoop, - _ updateAccumulatingResult: @escaping ReduceIntoCallback - ) -> EventLoopFuture { + ) -> EventLoopFuture where Value: Sendable { let p0 = eventLoop.makePromise(of: Value.self) - var value: Value = initialResult + let value = NIOLoopBoundBox(_value: initialResult, uncheckedEventLoop: eventLoop) let f0 = eventLoop.makeSucceededFuture(()) let future = f0.fold(futures) { (_: (), newValue: InputValue) -> EventLoopFuture in eventLoop.assertInEventLoop() - updateAccumulatingResult(&value, newValue) + var v = value.value + updateAccumulatingResult(&v, newValue) + value.value = v return eventLoop.makeSucceededFuture(()) } future.whenSuccess { eventLoop.assertInEventLoop() - p0.succeed(value) + p0.succeed(value.value) } future.whenFailure { (error) in eventLoop.assertInEventLoop() @@ -1245,14 +1268,17 @@ extension EventLoopFuture { /// - futures: An array of homogenous `EventLoopFutures`s to wait for. /// - promise: The `EventLoopPromise` to complete with the result of this call. @inlinable - public static func andAllSucceed(_ futures: [EventLoopFuture], promise: EventLoopPromise) { + public static func andAllSucceed( + _ futures: [EventLoopFuture], + promise: EventLoopPromise + ) { let eventLoop = promise.futureResult.eventLoop if eventLoop.inEventLoop { - self._reduceSuccesses0(promise, futures, eventLoop, onValue: { _, _ in }) + self._reduceSuccesses0(promise, futures, eventLoop) } else { eventLoop.execute { - self._reduceSuccesses0(promise, futures, eventLoop, onValue: { _, _ in }) + self._reduceSuccesses0(promise, futures, eventLoop) } } } @@ -1261,14 +1287,19 @@ extension EventLoopFuture { /// The new `EventLoopFuture` will contain all of the values fulfilled by the futures. /// /// The returned `EventLoopFuture` will fail as soon as any of the futures fails. + /// + /// - Note: The `Value` must be `Sendable` since the isolation domains of the futures might differ i.e. + /// they might be bound to different event loops. + /// /// - Parameters: /// - futures: An array of homogenous `EventLoopFuture`s to wait on for fulfilled values. /// - on: The `EventLoop` on which the new `EventLoopFuture` callbacks will fire. /// - Returns: A new `EventLoopFuture` with all of the values fulfilled by the provided futures. + @preconcurrency public static func whenAllSucceed( _ futures: [EventLoopFuture], on eventLoop: EventLoop - ) -> EventLoopFuture<[Value]> { + ) -> EventLoopFuture<[Value]> where Value: Sendable { let promise = eventLoop.makePromise(of: [Value].self) EventLoopFuture.whenAllSucceed(futures, promise: promise) return promise.futureResult @@ -1279,10 +1310,17 @@ extension EventLoopFuture { /// /// If the _results of all futures should be collected use `andAllComplete` instead. /// + /// - Note: The `Value` must be `Sendable` since the isolation domains of the futures might differ i.e. + /// they might be bound to different event loops. + /// /// - Parameters: /// - futures: An array of homogenous `EventLoopFutures`s to wait for. /// - promise: The `EventLoopPromise` to complete with the result of this call. - public static func whenAllSucceed(_ futures: [EventLoopFuture], promise: EventLoopPromise<[Value]>) { + @preconcurrency + public static func whenAllSucceed( + _ futures: [EventLoopFuture], + promise: EventLoopPromise<[Value]> + ) where Value: Sendable { let eventLoop = promise.futureResult.eventLoop let reduced = eventLoop.makePromise(of: Void.self) @@ -1311,7 +1349,6 @@ extension EventLoopFuture { } } - @usableFromInline typealias ReduceSuccessCallback = @Sendable (Int, InputValue) -> Void /// Loops through the futures array and attaches callbacks to execute `onValue` on the provided `EventLoop` when /// they succeed. The `onValue` will receive the index of the future that fulfilled the provided `Result`. /// @@ -1322,25 +1359,26 @@ extension EventLoopFuture { _ promise: EventLoopPromise, _ futures: [EventLoopFuture], _ eventLoop: EventLoop, - onValue: @escaping ReduceSuccessCallback - ) { + onValue: @escaping @Sendable (Int, InputValue) -> Void + ) where InputValue: Sendable { eventLoop.assertInEventLoop() - var remainingCount = futures.count - - if remainingCount == 0 { + if futures.count == 0 { promise.succeed(()) return } + let remainingCount = NIOLoopBoundBox(_value: futures.count, uncheckedEventLoop: eventLoop) + // Sends the result to `onValue` in case of success and succeeds/fails the input promise, if appropriate. + @Sendable func processResult(_ index: Int, _ result: Result) { switch result { case .success(let result): onValue(index, result) - remainingCount -= 1 + remainingCount.value -= 1 - if remainingCount == 0 { + if remainingCount.value == 0 { promise.succeed(()) } case .failure(let error): @@ -1365,6 +1403,72 @@ extension EventLoopFuture { } } } + + /// Loops through the futures array and attaches callbacks to execute `onValue` on the provided `EventLoop` when + /// they succeed. The `onValue` will receive the index of the future that fulfilled the provided `Result`. + /// + /// Once all the futures have succeed, the provided promise will succeed. + /// Once any future fails, the provided promise will fail. + @inlinable + internal static func _reduceSuccesses0( + _ promise: EventLoopPromise, + _ futures: [EventLoopFuture], + _ eventLoop: EventLoop + ) { + eventLoop.assertInEventLoop() + + if futures.count == 0 { + promise.succeed(()) + return + } + + let remainingCount = NIOLoopBoundBox(_value: futures.count, uncheckedEventLoop: eventLoop) + + // Sends the result to `onValue` in case of success and succeeds/fails the input promise, if appropriate. + @Sendable + func processResult(_ index: Int, _ result: Result) { + switch result { + case .success: + remainingCount.value -= 1 + + if remainingCount.value == 0 { + promise.succeed(()) + } + case .failure(let error): + promise.fail(error) + } + } + // loop through the futures to chain callbacks to execute on the initiating event loop and grab their index + // in the "futures" to pass their result to the caller + for (index, future) in futures.enumerated() { + if future.eventLoop.inEventLoop, + let result = future._value + { + // Fast-track already-fulfilled results without the overhead of calling `whenComplete`. This can yield a + // ~20% performance improvement in the case of large arrays where all elements are already fulfilled. + switch result { + case .success: + processResult(index, .success(())) + case .failure(let error): + processResult(index, .failure(error)) + return + } + } else { + // We have to map to `Void` here to avoid sharing the potentially non-Sendable + // value across event loops. + future.whenComplete { result in + let voidResult = result.map { _ in } + if eventLoop.inEventLoop { + processResult(index, voidResult) + } else { + eventLoop.execute { + processResult(index, voidResult) + } + } + } + } + } + } } // MARK: "fail slow" reduce @@ -1400,14 +1504,17 @@ extension EventLoopFuture { /// - futures: An array of homogenous `EventLoopFuture`s to wait for. /// - promise: The `EventLoopPromise` to succeed when all futures have completed. @inlinable - public static func andAllComplete(_ futures: [EventLoopFuture], promise: EventLoopPromise) { + public static func andAllComplete( + _ futures: [EventLoopFuture], + promise: EventLoopPromise + ) { let eventLoop = promise.futureResult.eventLoop if eventLoop.inEventLoop { - self._reduceCompletions0(promise, futures, eventLoop, onResult: { _, _ in }) + self._reduceCompletions0(promise, futures, eventLoop) } else { eventLoop.execute { - self._reduceCompletions0(promise, futures, eventLoop, onResult: { _, _ in }) + self._reduceCompletions0(promise, futures, eventLoop) } } } @@ -1417,17 +1524,21 @@ extension EventLoopFuture { /// /// The returned `EventLoopFuture` always succeeds, regardless of any failures from the waiting futures. /// + /// - Note: The `Value` must be `Sendable` since the isolation domains of the futures might differ i.e. + /// they might be bound to different event loops. + /// /// If it is desired to flatten them into a single `EventLoopFuture` that fails on the first `EventLoopFuture` failure, /// use one of the `reduce` methods instead. /// - Parameters: /// - futures: An array of homogenous `EventLoopFuture`s to gather results from. /// - on: The `EventLoop` on which the new `EventLoopFuture` callbacks will fire. /// - Returns: A new `EventLoopFuture` with all the results of the provided futures. + @preconcurrency @inlinable public static func whenAllComplete( _ futures: [EventLoopFuture], on eventLoop: EventLoop - ) -> EventLoopFuture<[Result]> { + ) -> EventLoopFuture<[Result]> where Value: Sendable { let promise = eventLoop.makePromise(of: [Result].self) EventLoopFuture.whenAllComplete(futures, promise: promise) return promise.futureResult @@ -1437,14 +1548,18 @@ extension EventLoopFuture { /// /// The promise will always be succeeded, regardless of the outcome of the futures. /// + /// - Note: The `Value` must be `Sendable` since the isolation domains of the futures might differ i.e. + /// they might be bound to different event loops. + /// /// - Parameters: /// - futures: An array of homogenous `EventLoopFuture`s to gather results from. /// - promise: The `EventLoopPromise` to complete with the result of the futures. + @preconcurrency @inlinable public static func whenAllComplete( _ futures: [EventLoopFuture], promise: EventLoopPromise<[Result]> - ) { + ) where Value: Sendable { let eventLoop = promise.futureResult.eventLoop let reduced = eventLoop.makePromise(of: Void.self) @@ -1481,34 +1596,33 @@ extension EventLoopFuture { } } - @usableFromInline typealias ReduceCompletions = @Sendable (Int, Result) -> Void - /// Loops through the futures array and attaches callbacks to execute `onResult` on the provided `EventLoop` when /// they complete. The `onResult` will receive the index of the future that fulfilled the provided `Result`. /// /// Once all the futures have completed, the provided promise will succeed. @inlinable - internal static func _reduceCompletions0( + internal static func _reduceCompletions0( _ promise: EventLoopPromise, _ futures: [EventLoopFuture], _ eventLoop: EventLoop, - onResult: @escaping ReduceCompletions + onResult: @escaping @Sendable (Int, Result) -> Void ) { eventLoop.assertInEventLoop() - var remainingCount = futures.count - - if remainingCount == 0 { + if futures.count == 0 { promise.succeed(()) return } + let remainingCount = NIOLoopBoundBox(_value: futures.count, uncheckedEventLoop: eventLoop) + // Sends the result to `onResult` in case of success and succeeds the input promise, if appropriate. + @Sendable func processResult(_ index: Int, _ result: Result) { onResult(index, result) - remainingCount -= 1 + remainingCount.value -= 1 - if remainingCount == 0 { + if remainingCount.value == 0 { promise.succeed(()) } } @@ -1527,6 +1641,65 @@ extension EventLoopFuture { } } } + + /// Loops through the futures array and attaches callbacks to execute `onResult` on the provided `EventLoop` when + /// they complete. The `onResult` will receive the index of the future that fulfilled the provided `Result`. + /// + /// Once all the futures have completed, the provided promise will succeed. + @inlinable + internal static func _reduceCompletions0( + _ promise: EventLoopPromise, + _ futures: [EventLoopFuture], + _ eventLoop: EventLoop + ) { + eventLoop.assertInEventLoop() + + if futures.count == 0 { + promise.succeed(()) + return + } + + let remainingCount = NIOLoopBoundBox(_value: futures.count, uncheckedEventLoop: eventLoop) + + // Sends the result to `onResult` in case of success and succeeds the input promise, if appropriate. + @Sendable + func processResult(_ index: Int, _ result: Result) { + remainingCount.value -= 1 + + if remainingCount.value == 0 { + promise.succeed(()) + } + } + // loop through the futures to chain callbacks to execute on the initiating event loop and grab their index + // in the "futures" to pass their result to the caller + for (index, future) in futures.enumerated() { + if future.eventLoop.inEventLoop, + let result = future._value + { + // Fast-track already-fulfilled results without the overhead of calling `whenComplete`. This can yield a + // ~30% performance improvement in the case of large arrays where all elements are already fulfilled. + switch result { + case .success: + processResult(index, .success(())) + case .failure(let error): + processResult(index, .failure(error)) + } + } else { + // We have to map to `Void` here to avoid sharing the potentially non-Sendable + // value across event loops. + future.whenComplete { result in + let voidResult = result.map { _ in } + if eventLoop.inEventLoop { + processResult(index, voidResult) + } else { + eventLoop.execute { + processResult(index, voidResult) + } + } + } + } + } + } } // MARK: hop @@ -1540,11 +1713,14 @@ extension EventLoopFuture { /// succinctly. It also contains an optimisation for the case when the loop you're hopping *from* is the same as /// the one you're hopping *to*, allowing you to avoid doing allocations in that case. /// + /// - Note: The `Value` must be `Sendable` since it is shared with the isolation domain of the target event loop. + /// /// - parameters: /// - to: The `EventLoop` that the returned `EventLoopFuture` will run on. /// - returns: An `EventLoopFuture` whose callbacks run on `target` instead of the original loop. + @preconcurrency @inlinable - public func hop(to target: EventLoop) -> EventLoopFuture { + public func hop(to target: EventLoop) -> EventLoopFuture where Value: Sendable { if target === self.eventLoop { // We're already on that event loop, nothing to do here. Save an allocation. return self @@ -1567,12 +1743,6 @@ extension EventLoopFuture { @inlinable @preconcurrency public func always(_ callback: @escaping @Sendable (Result) -> Void) -> EventLoopFuture { - self._always(callback) - } - @usableFromInline typealias AlwaysCallback = @Sendable (Result) -> Void - - @inlinable - func _always(_ callback: @escaping AlwaysCallback) -> EventLoopFuture { self.whenComplete { result in callback(result) } return self } @@ -1620,9 +1790,11 @@ extension EventLoopFuture { /// - parameters: /// - orReplace: the value of the returned `EventLoopFuture` when then resolved future's value is `Optional.some()`. /// - returns: an new `EventLoopFuture` with new type parameter `NewValue` and the value passed in the `orReplace` parameter. + @preconcurrency @inlinable - public func unwrap(orReplace replacement: NewValue) -> EventLoopFuture - where Value == NewValue? { + public func unwrap( + orReplace replacement: NewValue + ) -> EventLoopFuture where Value == NewValue? { self.map { (value) -> NewValue in guard let value = value else { return replacement @@ -1677,25 +1849,18 @@ extension EventLoopFuture { /// blockingTask(value) /// } /// + /// - Note: The `Value` and `NewValue` must be `Sendable` since it is shared between the isolation region queue and the event loop. + /// /// - parameters: /// - onto: the `DispatchQueue` on which the blocking IO / task specified by `callbackMayBlock` is scheduled. /// - callbackMayBlock: Function that will receive the value of this `EventLoopFuture` and return /// a new `EventLoopFuture`. @inlinable @preconcurrency - public func flatMapBlocking( + public func flatMapBlocking( onto queue: DispatchQueue, _ callbackMayBlock: @escaping @Sendable (Value) throws -> NewValue - ) -> EventLoopFuture { - self._flatMapBlocking(onto: queue, callbackMayBlock) - } - @usableFromInline typealias FlatMapBlockingCallback = @Sendable (Value) throws -> NewValue - - @inlinable - func _flatMapBlocking( - onto queue: DispatchQueue, - _ callbackMayBlock: @escaping FlatMapBlockingCallback - ) -> EventLoopFuture { + ) -> EventLoopFuture where Value: Sendable { self.flatMap { result in queue.asyncWithFuture(eventLoop: self.eventLoop) { try callbackMayBlock(result) } } @@ -1709,11 +1874,17 @@ extension EventLoopFuture { /// If you find yourself passing the results from this `EventLoopFuture` to a new `EventLoopPromise` /// in the body of this function, consider using `cascade` instead. /// + /// - Note: The `NewValue` must be `Sendable` since it is shared between the isolation region queue and the event loop. + /// /// - parameters: /// - onto: the `DispatchQueue` on which the blocking IO / task specified by `callbackMayBlock` is scheduled. /// - callbackMayBlock: The callback that is called with the successful result of the `EventLoopFuture`. + @preconcurrency @inlinable - public func whenSuccessBlocking(onto queue: DispatchQueue, _ callbackMayBlock: @escaping (Value) -> Void) { + public func whenSuccessBlocking( + onto queue: DispatchQueue, + _ callbackMayBlock: @escaping @Sendable (Value) -> Void + ) where Value: Sendable { self.whenSuccess { value in queue.async { callbackMayBlock(value) } } @@ -1732,8 +1903,10 @@ extension EventLoopFuture { /// - callbackMayBlock: The callback that is called with the failed result of the `EventLoopFuture`. @inlinable @preconcurrency - public func whenFailureBlocking(onto queue: DispatchQueue, _ callbackMayBlock: @escaping @Sendable (Error) -> Void) - { + public func whenFailureBlocking( + onto queue: DispatchQueue, + _ callbackMayBlock: @escaping @Sendable (Error) -> Void + ) { self._whenFailureBlocking(onto: queue, callbackMayBlock) } @usableFromInline typealias WhenFailureBlockingCallback = @Sendable (Error) -> Void @@ -1748,6 +1921,8 @@ extension EventLoopFuture { /// Adds an observer callback to this `EventLoopFuture` that is called when the /// `EventLoopFuture` has any result. The observer callback is permitted to block. /// + /// - Note: The `NewValue` must be `Sendable` since it is shared between the isolation region queue and the event loop. + /// /// - parameters: /// - onto: the `DispatchQueue` on which the blocking IO / task specified by `callbackMayBlock` is scheduled. /// - callbackMayBlock: The callback that is called when the `EventLoopFuture` is fulfilled. @@ -1756,13 +1931,7 @@ extension EventLoopFuture { public func whenCompleteBlocking( onto queue: DispatchQueue, _ callbackMayBlock: @escaping @Sendable (Result) -> Void - ) { - self._whenCompleteBlocking(onto: queue, callbackMayBlock) - } - @usableFromInline typealias WhenCompleteBlocking = @Sendable (Result) -> Void - - @inlinable - func _whenCompleteBlocking(onto queue: DispatchQueue, _ callbackMayBlock: @escaping WhenCompleteBlocking) { + ) where Value: Sendable { self.whenComplete { value in queue.async { callbackMayBlock(value) } } @@ -1875,13 +2044,19 @@ public struct _NIOEventLoopFutureIdentifier: Hashable, Sendable { } } -// EventLoopPromise is a reference type, but by its very nature is Sendable (if its Value is). -extension EventLoopPromise: Sendable where Value: Sendable {} +// The future is unchecked Sendable following the below isolation rules this is safe +// +// 1. Receiving the value of the future is always done on the EventLoop of the future, hence +// the value is never transferred out of the event loops isolation domain. It only gets transferred +// by certain methods such as `hop()` and those methods are annotated with requiring the Value to be +// Sendable +// 2. The promise is `Sendable` but fulfilling the promise with a value requires the user to +// transfer the value to the promise. This ensures that the value is now isolated to the event loops +// isolation domain. Note: Sendable values can always be transferred + +extension EventLoopPromise: Sendable {} -// EventLoopFuture is a reference type, but it is Sendable (if its Value is). However, we enforce -// that by way of the guarantees of the EventLoop protocol, so the compiler cannot -// check it. -extension EventLoopFuture: @unchecked Sendable where Value: Sendable {} +extension EventLoopFuture: @unchecked Sendable {} extension EventLoopPromise where Value == Void { // Deliver a successful result to the associated `EventLoopFuture` object. @@ -1899,7 +2074,8 @@ extension Optional { /// to `promise`. /// /// - Parameter promise: The promise to set or cascade to. - public mutating func setOrCascade(to promise: EventLoopPromise?) + @preconcurrency + public mutating func setOrCascade(to promise: EventLoopPromise?) where Wrapped == EventLoopPromise { guard let promise = promise else { return } diff --git a/Sources/NIOCore/NIOScheduledCallback.swift b/Sources/NIOCore/NIOScheduledCallback.swift index 8d7a056a0a..b2af415aad 100644 --- a/Sources/NIOCore/NIOScheduledCallback.swift +++ b/Sources/NIOCore/NIOScheduledCallback.swift @@ -89,9 +89,10 @@ public struct NIOScheduledCallback: Sendable { } extension EventLoop { + @preconcurrency package func _scheduleCallback( at deadline: NIODeadline, - handler: some NIOScheduledCallbackHandler + handler: some (NIOScheduledCallbackHandler & Sendable) ) -> NIOScheduledCallback { let task = self.scheduleTask(deadline: deadline) { handler.handleScheduledCallback(eventLoop: self) } task.futureResult.whenFailure { error in @@ -130,20 +131,22 @@ extension EventLoop { /// /// The implementation of this default conformance has been further factored out so we can use it in /// `NIOAsyncTestingEventLoop`, where the use of `wait()` is _less bad_. + @preconcurrency @discardableResult public func scheduleCallback( at deadline: NIODeadline, - handler: some NIOScheduledCallbackHandler + handler: some (NIOScheduledCallbackHandler & Sendable) ) -> NIOScheduledCallback { self._scheduleCallback(at: deadline, handler: handler) } /// Default implementation of `scheduleCallback(in amount:handler:)`: calls `scheduleCallback(at deadline:handler:)`. + @preconcurrency @discardableResult @inlinable public func scheduleCallback( in amount: TimeAmount, - handler: some NIOScheduledCallbackHandler + handler: some (NIOScheduledCallbackHandler & Sendable) ) throws -> NIOScheduledCallback { try self.scheduleCallback(at: .now() + amount, handler: handler) } diff --git a/Sources/NIOEmbedded/AsyncTestingEventLoop.swift b/Sources/NIOEmbedded/AsyncTestingEventLoop.swift index 52d56bf9ca..14e5242df0 100644 --- a/Sources/NIOEmbedded/AsyncTestingEventLoop.swift +++ b/Sources/NIOEmbedded/AsyncTestingEventLoop.swift @@ -192,9 +192,10 @@ public final class NIOAsyncTestingEventLoop: EventLoop, @unchecked Sendable { self.scheduleTask(deadline: self.now + `in`, task) } + @preconcurrency public func scheduleCallback( at deadline: NIODeadline, - handler: some NIOScheduledCallbackHandler + handler: some (NIOScheduledCallbackHandler & Sendable) ) throws -> NIOScheduledCallback { /// The default implementation of `scheduledCallback(at:handler)` makes two calls to the event loop because it /// needs to hook the future of the backing scheduled task, which can lead to lost cancellation callbacks when @@ -213,10 +214,11 @@ public final class NIOAsyncTestingEventLoop: EventLoop, @unchecked Sendable { } } + @preconcurrency @discardableResult public func scheduleCallback( in amount: TimeAmount, - handler: some NIOScheduledCallbackHandler + handler: some (NIOScheduledCallbackHandler & Sendable) ) throws -> NIOScheduledCallback { /// Even though this type does not implement a custom `scheduleCallback(at:handler)`, it uses a manual clock so /// it cannot rely on the default implementation of `scheduleCallback(in:handler:)`, which computes the deadline diff --git a/Sources/NIOEmbedded/Embedded.swift b/Sources/NIOEmbedded/Embedded.swift index a6c2b33b5b..3105f3af85 100644 --- a/Sources/NIOEmbedded/Embedded.swift +++ b/Sources/NIOEmbedded/Embedded.swift @@ -160,10 +160,11 @@ public final class EmbeddedEventLoop: EventLoop, CustomStringConvertible { scheduleTask(deadline: self._now + `in`, task) } + @preconcurrency @discardableResult public func scheduleCallback( in amount: TimeAmount, - handler: some NIOScheduledCallbackHandler + handler: some (NIOScheduledCallbackHandler & Sendable) ) -> NIOScheduledCallback { /// Even though this type does not implement a custom `scheduleCallback(at:handler)`, it uses a manual clock so /// it cannot rely on the default implementation of `scheduleCallback(in:handler:)`, which computes the deadline diff --git a/Tests/NIOCoreTests/AsyncChannel/AsyncChannelTests.swift b/Tests/NIOCoreTests/AsyncChannel/AsyncChannelTests.swift index 7b38e49301..4e4c6a34e0 100644 --- a/Tests/NIOCoreTests/AsyncChannel/AsyncChannelTests.swift +++ b/Tests/NIOCoreTests/AsyncChannel/AsyncChannelTests.swift @@ -253,7 +253,9 @@ final class AsyncChannelTests: XCTestCase { let strongSentinel: Sentinel? = Sentinel() sentinel = strongSentinel! try await XCTAsyncAssertNotNil( - await channel.pipeline.handler(type: NIOAsyncChannelHandler.self).get() + await channel.pipeline.handler(type: NIOAsyncChannelHandler.self).map { _ in + true + }.get() ) try await channel.writeInbound(strongSentinel!) _ = try await channel.readInbound(as: Sentinel.self) diff --git a/Tests/NIOPosixTests/EventLoopFutureTest.swift b/Tests/NIOPosixTests/EventLoopFutureTest.swift index fac8f2ab6d..d0f7bb3335 100644 --- a/Tests/NIOPosixTests/EventLoopFutureTest.swift +++ b/Tests/NIOPosixTests/EventLoopFutureTest.swift @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// import Dispatch +import NIOConcurrencyHelpers import NIOEmbedded import NIOPosix import XCTest @@ -1408,18 +1409,19 @@ class EventLoopFutureTest: XCTestCase { func testWhenSuccessBlocking() { let eventLoop = EmbeddedEventLoop() let sem = DispatchSemaphore(value: 0) - var nonBlockingRan = false + let nonBlockingRan = NIOLockedValueBox(false) let p = eventLoop.makePromise(of: String.self) p.futureResult.whenSuccessBlocking(onto: DispatchQueue.global()) { sem.wait() // Block in callback XCTAssertEqual($0, "hello") - XCTAssertTrue(nonBlockingRan) + nonBlockingRan.withLockedValue { XCTAssertTrue($0) } + } p.succeed("hello") let p2 = eventLoop.makePromise(of: Bool.self) p2.futureResult.whenSuccess { _ in - nonBlockingRan = true + nonBlockingRan.withLockedValue { $0 = true } } p2.succeed(true) diff --git a/Tests/NIOPosixTests/NonBlockingFileIOTest.swift b/Tests/NIOPosixTests/NonBlockingFileIOTest.swift index 3fbdc71e30..405f7e84b7 100644 --- a/Tests/NIOPosixTests/NonBlockingFileIOTest.swift +++ b/Tests/NIOPosixTests/NonBlockingFileIOTest.swift @@ -634,34 +634,41 @@ class NonBlockingFileIOTest: XCTestCase { func testFileOpenWorks() throws { let content = "123" try withTemporaryFile(content: content) { (fileHandle, path) -> Void in - let (fh, fr) = try self.fileIO.openFile(path: path, eventLoop: self.eventLoop).wait() - try fh.withUnsafeFileDescriptor { fd in - XCTAssertGreaterThanOrEqual(fd, 0) - } - XCTAssertTrue(fh.isOpen) - XCTAssertEqual(0, fr.readerIndex) - XCTAssertEqual(3, fr.endIndex) - try fh.close() + try self.fileIO.openFile(path: path, eventLoop: self.eventLoop).flatMapThrowing { vals in + let (fh, fr) = vals + try fh.withUnsafeFileDescriptor { fd in + XCTAssertGreaterThanOrEqual(fd, 0) + } + XCTAssertTrue(fh.isOpen) + XCTAssertEqual(0, fr.readerIndex) + XCTAssertEqual(3, fr.endIndex) + try fh.close() + }.wait() } } func testFileOpenWorksWithEmptyFile() throws { let content = "" try withTemporaryFile(content: content) { (fileHandle, path) -> Void in - let (fh, fr) = try self.fileIO.openFile(path: path, eventLoop: self.eventLoop).wait() - try fh.withUnsafeFileDescriptor { fd in - XCTAssertGreaterThanOrEqual(fd, 0) - } - XCTAssertTrue(fh.isOpen) - XCTAssertEqual(0, fr.readerIndex) - XCTAssertEqual(0, fr.endIndex) - try fh.close() + try self.fileIO.openFile(path: path, eventLoop: self.eventLoop).flatMapThrowing { vals in + let (fh, fr) = vals + try fh.withUnsafeFileDescriptor { fd in + XCTAssertGreaterThanOrEqual(fd, 0) + } + XCTAssertTrue(fh.isOpen) + XCTAssertEqual(0, fr.readerIndex) + XCTAssertEqual(0, fr.endIndex) + try fh.close() + }.wait() } } func testFileOpenFails() throws { do { - _ = try self.fileIO.openFile(path: "/dev/null/this/does/not/exist", eventLoop: self.eventLoop).wait() + try self.fileIO.openFile( + path: "/dev/null/this/does/not/exist", + eventLoop: self.eventLoop + ).map { _ in }.wait() XCTFail("should've thrown") } catch let e as IOError where e.errnoCode == ENOTDIR { // OK From 8b66b22895115954dca2f1a6c5ab67172b6a6a1c Mon Sep 17 00:00:00 2001 From: Jeff Date: Sat, 19 Oct 2024 02:25:51 -0700 Subject: [PATCH 06/20] Fix Windows build break. (#2935) Use `withLockPrimitive` to access the underlying lock. ### Motivation: The current method of accessing the lock doesn't build. This fixes bug #2934. ### Modifications: Access the lock via `withLockPrimitive`. ### Result: Now it builds! :) --- Sources/NIOConcurrencyHelpers/lock.swift | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Sources/NIOConcurrencyHelpers/lock.swift b/Sources/NIOConcurrencyHelpers/lock.swift index 42e2a3533c..3b985afaa6 100644 --- a/Sources/NIOConcurrencyHelpers/lock.swift +++ b/Sources/NIOConcurrencyHelpers/lock.swift @@ -229,12 +229,10 @@ public final class ConditionLock { } let dwWaitStart = timeGetTime() - if !SleepConditionVariableSRW( - self.cond, - self.mutex._storage.mutex, - dwMilliseconds, - 0 - ) { + let result = self.mutex.withLockPrimitive { mutex in + SleepConditionVariableSRW(self.cond, mutex, dwMilliseconds, 0) + } + if !result { let dwError = GetLastError() if dwError == ERROR_TIMEOUT { self.unlock() @@ -242,7 +240,6 @@ public final class ConditionLock { } fatalError("SleepConditionVariableSRW: \(dwError)") } - // NOTE: this may be a spurious wakeup, adjust the timeout accordingly dwMilliseconds = dwMilliseconds - (timeGetTime() - dwWaitStart) } From cc1c57c2912a058870f4f47d84a001aeb7be39ac Mon Sep 17 00:00:00 2001 From: Clinton Nkwocha <32041805+clintonpi@users.noreply.github.com> Date: Mon, 21 Oct 2024 09:41:10 +0100 Subject: [PATCH 07/20] Provide APIs to read file into more data types (#2923) Motivation: As requested in issues [#2875](https://github.com/apple/swift-nio/issues/2875) and [#2876](https://github.com/apple/swift-nio/issues/2876), it would be convenient to be able to read the contents of a file into more data types such as `Array`, `ArraySlice` & Foundation's `Data`. Modifications: - Extend `Array`, `ArraySlice` & `Data` to be initialisable with the contents of a file. Result: The contents of a file can be read into more data types. --------- Co-authored-by: George Barnett --- Package.swift | 3 +- Sources/NIOFileSystem/Array+FileSystem.swift | 54 ++++++++++++++++++ .../NIOFileSystem/ArraySlice+FileSystem.swift | 56 ++++++++++++++++++ .../Data+FileSystem.swift | 57 +++++++++++++++++++ .../FileSystemFoundationCompatTests.swift | 44 ++++++++++++++ .../FileSystemTests.swift | 24 ++++++++ 6 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 Sources/NIOFileSystem/Array+FileSystem.swift create mode 100644 Sources/NIOFileSystem/ArraySlice+FileSystem.swift create mode 100644 Sources/NIOFileSystemFoundationCompat/Data+FileSystem.swift diff --git a/Package.swift b/Package.swift index 99842452b9..bb2eaeddcb 100644 --- a/Package.swift +++ b/Package.swift @@ -259,7 +259,8 @@ let package = Package( .target( name: "_NIOFileSystemFoundationCompat", dependencies: [ - "_NIOFileSystem" + "_NIOFileSystem", + "NIOFoundationCompat", ], path: "Sources/NIOFileSystemFoundationCompat" ), diff --git a/Sources/NIOFileSystem/Array+FileSystem.swift b/Sources/NIOFileSystem/Array+FileSystem.swift new file mode 100644 index 0000000000..b8707fbbd5 --- /dev/null +++ b/Sources/NIOFileSystem/Array+FileSystem.swift @@ -0,0 +1,54 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) +import NIOCore + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension Array where Element == UInt8 { + /// Reads the contents of the file at the path. + /// + /// - Parameters: + /// - path: The path of the file to read. + /// - maximumSizeAllowed: The maximum size of file which can be read, in bytes, as a ``ByteCount``. + /// - fileSystem: The ``FileSystemProtocol`` instance to use to read the file. + public init( + contentsOf path: FilePath, + maximumSizeAllowed: ByteCount, + fileSystem: some FileSystemProtocol + ) async throws { + let byteBuffer = try await fileSystem.withFileHandle(forReadingAt: path) { handle in + try await handle.readToEnd(maximumSizeAllowed: maximumSizeAllowed) + } + + self = Self(buffer: byteBuffer) + } + + /// Reads the contents of the file at the path using ``FileSystem``. + /// + /// - Parameters: + /// - path: The path of the file to read. + /// - maximumSizeAllowed: The maximum size of file which can be read, as a ``ByteCount``. + public init( + contentsOf path: FilePath, + maximumSizeAllowed: ByteCount + ) async throws { + self = try await Self( + contentsOf: path, + maximumSizeAllowed: maximumSizeAllowed, + fileSystem: .shared + ) + } +} +#endif diff --git a/Sources/NIOFileSystem/ArraySlice+FileSystem.swift b/Sources/NIOFileSystem/ArraySlice+FileSystem.swift new file mode 100644 index 0000000000..37e324acfc --- /dev/null +++ b/Sources/NIOFileSystem/ArraySlice+FileSystem.swift @@ -0,0 +1,56 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) +import NIOCore + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension ArraySlice where Element == UInt8 { + /// Reads the contents of the file at the path. + /// + /// - Parameters: + /// - path: The path of the file to read. + /// - maximumSizeAllowed: The maximum size of file which can be read, in bytes, as a ``ByteCount``. + /// - fileSystem: The ``FileSystemProtocol`` instance to use to read the file. + public init( + contentsOf path: FilePath, + maximumSizeAllowed: ByteCount, + fileSystem: some FileSystemProtocol + ) async throws { + let bytes = try await Array( + contentsOf: path, + maximumSizeAllowed: maximumSizeAllowed, + fileSystem: fileSystem + ) + + self = Self(bytes) + } + + /// Reads the contents of the file at the path using ``FileSystem``. + /// + /// - Parameters: + /// - path: The path of the file to read. + /// - maximumSizeAllowed: The maximum size of file which can be read, as a ``ByteCount``. + public init( + contentsOf path: FilePath, + maximumSizeAllowed: ByteCount + ) async throws { + self = try await Self( + contentsOf: path, + maximumSizeAllowed: maximumSizeAllowed, + fileSystem: .shared + ) + } +} +#endif diff --git a/Sources/NIOFileSystemFoundationCompat/Data+FileSystem.swift b/Sources/NIOFileSystemFoundationCompat/Data+FileSystem.swift new file mode 100644 index 0000000000..7e0652f24d --- /dev/null +++ b/Sources/NIOFileSystemFoundationCompat/Data+FileSystem.swift @@ -0,0 +1,57 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftNIO open source project +// +// Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftNIO project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) +import _NIOFileSystem +import NIOCore +import NIOFoundationCompat +import struct Foundation.Data + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension Data { + /// Reads the contents of the file at the path. + /// + /// - Parameters: + /// - path: The path of the file to read. + /// - maximumSizeAllowed: The maximum size of file which can be read, in bytes, as a ``ByteCount``. + /// - fileSystem: The ``FileSystemProtocol`` instance to use to read the file. + public init( + contentsOf path: FilePath, + maximumSizeAllowed: ByteCount, + fileSystem: some FileSystemProtocol + ) async throws { + let byteBuffer = try await fileSystem.withFileHandle(forReadingAt: path) { handle in + try await handle.readToEnd(maximumSizeAllowed: maximumSizeAllowed) + } + + self = Data(buffer: byteBuffer) + } + + /// Reads the contents of the file at the path using ``FileSystem``. + /// + /// - Parameters: + /// - path: The path of the file to read. + /// - maximumSizeAllowed: The maximum size of file which can be read, as a ``ByteCount``. + public init( + contentsOf path: FilePath, + maximumSizeAllowed: ByteCount + ) async throws { + self = try await Self( + contentsOf: path, + maximumSizeAllowed: maximumSizeAllowed, + fileSystem: .shared + ) + } +} +#endif diff --git a/Tests/NIOFileSystemFoundationCompatTests/FileSystemFoundationCompatTests.swift b/Tests/NIOFileSystemFoundationCompatTests/FileSystemFoundationCompatTests.swift index 2f6d0e68cf..717b9762dd 100644 --- a/Tests/NIOFileSystemFoundationCompatTests/FileSystemFoundationCompatTests.swift +++ b/Tests/NIOFileSystemFoundationCompatTests/FileSystemFoundationCompatTests.swift @@ -17,6 +17,37 @@ import _NIOFileSystem import _NIOFileSystemFoundationCompat import XCTest +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension FileSystem { + func temporaryFilePath( + _ function: String = #function, + inTemporaryDirectory: Bool = true + ) async throws -> FilePath { + if inTemporaryDirectory { + let directory = try await self.temporaryDirectory + return self.temporaryFilePath(function, inDirectory: directory) + } else { + return self.temporaryFilePath(function, inDirectory: nil) + } + } + + func temporaryFilePath( + _ function: String = #function, + inDirectory directory: FilePath? + ) -> FilePath { + let index = function.firstIndex(of: "(")! + let functionName = function.prefix(upTo: index) + let random = UInt32.random(in: .min ... .max) + let fileName = "\(functionName)-\(random)" + + if let directory = directory { + return directory.appending(fileName) + } else { + return FilePath(fileName) + } + } +} + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) final class FileSystemBytesConformanceTests: XCTestCase { func testTimepecToDate() async throws { @@ -33,5 +64,18 @@ final class FileSystemBytesConformanceTests: XCTestCase { Date(timeIntervalSince1970: 1.000000001) ) } + + func testReadFileIntoData() async throws { + let fs = FileSystem.shared + let path = try await fs.temporaryFilePath() + + try await fs.withFileHandle(forReadingAndWritingAt: path) { fileHandle in + _ = try await fileHandle.write(contentsOf: [0, 1, 2], toAbsoluteOffset: 0) + } + + let contents = try await Data(contentsOf: path, maximumSizeAllowed: .bytes(1024)) + + XCTAssertEqual(contents, Data([0, 1, 2])) + } } #endif diff --git a/Tests/NIOFileSystemIntegrationTests/FileSystemTests.swift b/Tests/NIOFileSystemIntegrationTests/FileSystemTests.swift index 409ef26cef..a7c2786a27 100644 --- a/Tests/NIOFileSystemIntegrationTests/FileSystemTests.swift +++ b/Tests/NIOFileSystemIntegrationTests/FileSystemTests.swift @@ -1828,6 +1828,30 @@ extension FileSystemTests { ) } } + + func testReadIntoArray() async throws { + let path = try await self.fs.temporaryFilePath() + + try await self.fs.withFileHandle(forReadingAndWritingAt: path) { fileHandle in + _ = try await fileHandle.write(contentsOf: [0, 1, 2], toAbsoluteOffset: 0) + } + + let contents = try await Array(contentsOf: path, maximumSizeAllowed: .bytes(1024)) + + XCTAssertEqual(contents, [0, 1, 2]) + } + + func testReadIntoArraySlice() async throws { + let path = try await self.fs.temporaryFilePath() + + try await self.fs.withFileHandle(forReadingAndWritingAt: path) { fileHandle in + _ = try await fileHandle.write(contentsOf: [0, 1, 2], toAbsoluteOffset: 0) + } + + let contents = try await ArraySlice(contentsOf: path, maximumSizeAllowed: .bytes(1024)) + + XCTAssertEqual(contents, [0, 1, 2]) + } } #if !canImport(Darwin) && swift(<5.9.2) From 49cd78bb767b1773688d33f6acb91f507ad109b8 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 21 Oct 2024 15:52:31 +0100 Subject: [PATCH 08/20] Fix withConnectedSocket in async mode (#2937) Motivation: The async flavour of withConnectedSocket accidentally type-erased the channel, which caused it to be unable to be used. Modifications: Un-erase the type of the channel. Add a test. Result: withConnectedSocket works again Resolves #2936 --- Sources/NIOPosix/Bootstrap.swift | 2 +- .../AsyncChannelBootstrapTests.swift | 79 +++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/Sources/NIOPosix/Bootstrap.swift b/Sources/NIOPosix/Bootstrap.swift index 3d1e3b9d93..f37abb37c5 100644 --- a/Sources/NIOPosix/Bootstrap.swift +++ b/Sources/NIOPosix/Bootstrap.swift @@ -1398,7 +1398,7 @@ extension ClientBootstrap { private func initializeAndRegisterChannel( channel: SocketChannel, channelInitializer: @escaping @Sendable (Channel) -> EventLoopFuture, - registration: @escaping @Sendable (Channel) -> EventLoopFuture, + registration: @escaping @Sendable (SocketChannel) -> EventLoopFuture, postRegisterTransformation: @escaping @Sendable (ChannelInitializerResult, EventLoop) -> EventLoopFuture< PostRegistrationTransformationResult > diff --git a/Tests/NIOPosixTests/AsyncChannelBootstrapTests.swift b/Tests/NIOPosixTests/AsyncChannelBootstrapTests.swift index 98d8943598..62dfbbc291 100644 --- a/Tests/NIOPosixTests/AsyncChannelBootstrapTests.swift +++ b/Tests/NIOPosixTests/AsyncChannelBootstrapTests.swift @@ -609,6 +609,69 @@ final class AsyncChannelBootstrapTests: XCTestCase { } } + func testServerClientBootstrap_withAsyncChannel_clientConnectedSocket() async throws { + let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 3) + defer { + try! eventLoopGroup.syncShutdownGracefully() + } + + let channel = try await ServerBootstrap(group: eventLoopGroup) + .serverChannelOption(.socketOption(.so_reuseaddr), value: 1) + .childChannelOption(.autoRead, value: true) + .bind( + host: "127.0.0.1", + port: 0 + ) { channel in + channel.eventLoop.makeCompletedFuture { () -> NIOAsyncChannel in + try channel.pipeline.syncOperations.addHandler(ByteToMessageHandler(LineDelimiterCoder())) + try channel.pipeline.syncOperations.addHandler(MessageToByteHandler(LineDelimiterCoder())) + try channel.pipeline.syncOperations.addHandler(ByteBufferToStringHandler()) + return try NIOAsyncChannel( + wrappingChannelSynchronously: channel, + configuration: .init( + inboundType: String.self, + outboundType: String.self + ) + ) + } + } + + try await withThrowingTaskGroup(of: Void.self) { group in + let (stream, continuation) = AsyncStream.makeStream() + var iterator = stream.makeAsyncIterator() + + group.addTask { + try await withThrowingTaskGroup(of: Void.self) { _ in + try await channel.executeThenClose { inbound in + for try await childChannel in inbound { + try await childChannel.executeThenClose { childChannelInbound, _ in + for try await value in childChannelInbound { + continuation.yield(.string(value)) + } + } + } + } + } + } + + let s = try Socket(protocolFamily: .inet, type: .stream) + XCTAssert(try s.connect(to: channel.channel.localAddress!)) + let fd = try s.takeDescriptorOwnership() + + let stringChannel = try await self.makeClientChannel( + eventLoopGroup: eventLoopGroup, + fileDescriptor: fd + ) + try await stringChannel.executeThenClose { _, outbound in + try await outbound.write("hello") + } + + await XCTAsyncAssertEqual(await iterator.next(), .string("hello")) + + group.cancelAll() + } + } + // MARK: Datagram Bootstrap func testDatagramBootstrap_withAsyncChannel_andHostPort() async throws { @@ -1280,6 +1343,22 @@ final class AsyncChannelBootstrapTests: XCTestCase { } } + private func makeClientChannel( + eventLoopGroup: EventLoopGroup, + fileDescriptor: CInt + ) async throws -> NIOAsyncChannel { + try await ClientBootstrap(group: eventLoopGroup) + .withConnectedSocket(fileDescriptor) { channel in + channel.eventLoop.makeCompletedFuture { + try channel.pipeline.syncOperations.addHandler(AddressedEnvelopingHandler()) + try channel.pipeline.syncOperations.addHandler(ByteToMessageHandler(LineDelimiterCoder())) + try channel.pipeline.syncOperations.addHandler(MessageToByteHandler(LineDelimiterCoder())) + try channel.pipeline.syncOperations.addHandler(ByteBufferToStringHandler()) + return try NIOAsyncChannel(wrappingChannelSynchronously: channel) + } + } + } + private func makeClientChannelWithProtocolNegotiation( eventLoopGroup: EventLoopGroup, port: Int, From 8666af5d5f54ddcd9e63f2b3352a4a89d955f49c Mon Sep 17 00:00:00 2001 From: Jeff Date: Tue, 22 Oct 2024 00:09:28 -0700 Subject: [PATCH 09/20] Fix Windows build for NIOCore. (#2938) Fix build failures in `NIOCore` on Windows. ### Motivation: I'd like it to build on Windows. ### Modifications: * Add missing `WinSDK` imports. * Add missing `#elseif os(Windows)` branch for UDP. ### Result: ``` c:\users\jeff\src\swift-nio> swift build -Xcc -DBYTE_ORDER=LITTLE_ENDIAN --target NIOCore Building for debugging... [1/1] Write auxiliary file c:\users\jeff\src\swift-nio\.build\x86_64-unknown-windows-msvc\debug\swift-version--15FA6356563A8EA6.txt Build of target: 'NIOCore' complete! (1.14s) ``` --- Sources/NIOCore/BSDSocketAPI.swift | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Sources/NIOCore/BSDSocketAPI.swift b/Sources/NIOCore/BSDSocketAPI.swift index 0c23ba71b1..3899c8ec96 100644 --- a/Sources/NIOCore/BSDSocketAPI.swift +++ b/Sources/NIOCore/BSDSocketAPI.swift @@ -18,6 +18,7 @@ import ucrt import let WinSDK.IPPROTO_IP import let WinSDK.IPPROTO_IPV6 import let WinSDK.IPPROTO_TCP +import let WinSDK.IPPROTO_UDP import let WinSDK.IP_ADD_MEMBERSHIP import let WinSDK.IP_DROP_MEMBERSHIP @@ -40,12 +41,14 @@ import let WinSDK.PF_INET import let WinSDK.PF_INET6 import let WinSDK.PF_UNIX +import let WinSDK.SO_BROADCAST import let WinSDK.SO_ERROR import let WinSDK.SO_KEEPALIVE import let WinSDK.SO_LINGER import let WinSDK.SO_RCVBUF import let WinSDK.SO_RCVTIMEO import let WinSDK.SO_REUSEADDR +import let WinSDK.SO_SNDBUF import let WinSDK.SOL_SOCKET @@ -282,6 +285,9 @@ extension NIOBSDSocket.OptionLevel { #if os(Linux) || os(Android) public static let udp: NIOBSDSocket.OptionLevel = NIOBSDSocket.OptionLevel(rawValue: CInt(IPPROTO_UDP)) + #elseif os(Windows) + public static let udp: NIOBSDSocket.OptionLevel = + NIOBSDSocket.OptionLevel(rawValue: IPPROTO_UDP.rawValue) #else public static let udp: NIOBSDSocket.OptionLevel = NIOBSDSocket.OptionLevel(rawValue: IPPROTO_UDP) From ff6dea9d11484ccdb314464f5e24a1d9ef41b3c5 Mon Sep 17 00:00:00 2001 From: Franz Busch Date: Tue, 22 Oct 2024 10:53:24 +0200 Subject: [PATCH 10/20] [CI] Add Windows matrix build (#2929) # Motivation In the long term we want to add support for Windows in our Swift packages. To do this we need a CI setup to ensure that our packages are building and the tests are passing. # Modification This PR adds a windows job to our swift matrix that installs Swift on the runner and then uses Swift PM to build and test the package. Windows jobs are disabled by default since most of our packages require some work to add Windows support. # Result We have a Windows CI pipeline that allows us to verify that our packages are working. --- .github/workflows/swift_matrix.yml | 87 ++++++++++++++++++++++++++++++ .github/workflows/unit_tests.yml | 31 +++++++++++ scripts/check-matrix-job.ps1 | 78 +++++++++++++++++++++++++++ 3 files changed, 196 insertions(+) create mode 100644 scripts/check-matrix-job.ps1 diff --git a/.github/workflows/swift_matrix.yml b/.github/workflows/swift_matrix.yml index e093aa3133..47ad6e94ff 100644 --- a/.github/workflows/swift_matrix.yml +++ b/.github/workflows/swift_matrix.yml @@ -67,6 +67,32 @@ on: type: string description: "The command of the nightly main Swift version linux matrix job to execute." + matrix_windows_command: + type: string + description: "The command of the current Swift version windows matrix job to execute." + default: "" + matrix_windows_6_0_enabled: + type: boolean + description: "Boolean to enable the 6.0 Swift version matrix job. Defaults to true." + default: false + matrix_windows_6_0_command_override: + type: string + description: "The command of the 6.0 Swift version windows matrix job to execute." + matrix_windows_nightly_6_0_enabled: + type: boolean + description: "Boolean to enable the nightly 6.0 Swift version matrix job. Defaults to true." + default: false + matrix_windows_nightly_6_0_command_override: + type: string + description: "The command of the nightly 6.0 Swift version windows matrix job to execute." + matrix_windows_nightly_main_enabled: + type: boolean + description: "Boolean to enable the nightly main Swift version matrix job. Defaults to true." + default: false + matrix_windows_nightly_main_command_override: + type: string + description: "The command of the nightly main Swift version windows matrix job to execute." + # We are cancelling previously triggered workflow runs concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.name }} @@ -121,3 +147,64 @@ jobs: run: | apt-get -qq update && apt-get -qq -y install curl curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-matrix-job.sh | bash + + windows: + name: Windows (${{ matrix.swift.swift_version }}) + runs-on: windows-2022 + if: ${{ inputs.matrix_windows_6_0_enabled }} + strategy: + fail-fast: false + matrix: + # We are specifying only the major and minor of the docker images to automatically pick up the latest patch release + swift: + - image: swift:6.0-windowsservercore-ltsc2022 + swift_version: "6.0" + enabled: ${{ inputs.matrix_windows_6_0_enabled }} + steps: + - name: Pull Docker image + if: ${{ matrix.swift.enabled }} + run: docker pull ${{ matrix.swift.image }} + - name: Checkout repository + if: ${{ matrix.swift.enabled }} + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Donwload matrix script + if: ${{ matrix.swift.enabled }} + run: curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-matrix-job.ps1 -o __check-matrix-job.ps1 + - name: Run matrix job + if: ${{ matrix.swift.enabled }} + run: | + docker run --env SWIFT_VERSION="${{ matrix.swift.swift_version }}" --env COMMAND="${{ inputs.matrix_windows_command }}" --env COMMAND_OVERRIDE_6_0="${{ inputs.matrix_windows_6_0_command_override }}" -v ${{ github.workspace }}:C:\source ${{ matrix.swift.image }} cmd /s /c "swift --version & cd C:\source\ & powershell -File __check-matrix-job.ps1" + + windows-nightly: + name: Windows (${{ matrix.swift.swift_version }}) + runs-on: windows-2019 + if: ${{ inputs.matrix_windows_nightly_6_0_enabled }} && ${{ inputs.matrix_windows_nightly_main_enabled }} + strategy: + fail-fast: false + matrix: + # We are specifying only the major and minor of the docker images to automatically pick up the latest patch release + swift: + - image: swiftlang/swift:nightly-6.0-windowsservercore-1809 + swift_version: "nightly-6.0" + enabled: ${{ inputs.matrix_windows_nightly_6_0_enabled }} + - image: swiftlang/swift:nightly-main-windowsservercore-1809 + swift_version: "nightly-main" + enabled: ${{ inputs.matrix_windows_nightly_main_enabled }} + steps: + - name: Pull Docker image + if: ${{ matrix.swift.enabled }} + run: docker pull ${{ matrix.swift.image }} + - name: Checkout repository + if: ${{ matrix.swift.enabled }} + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Donwload matrix script + if: ${{ matrix.swift.enabled }} + run: curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-matrix-job.ps1 -o __check-matrix-job.ps1 + - name: Run matrix job + if: ${{ matrix.swift.enabled }} + run: | + docker run --env SWIFT_VERSION="${{ matrix.swift.swift_version }}" --env COMMAND="${{ inputs.matrix_windows_command }}" --env COMMAND_OVERRIDE_NIGHTLY_6_0="${{ inputs.matrix_windows_nightly_6_0_command_override }}" --env COMMAND_OVERRIDE_NIGHTLY_MAIN="${{ inputs.matrix_windows_nightly_main_command_override }}" -v ${{ github.workspace }}:C:\source ${{ matrix.swift.image }} cmd /s /c "swift --version & cd C:\source\ & powershell -File __check-matrix-job.ps1" diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 997c6389d4..091a2b7655 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -44,6 +44,31 @@ on: description: "The arguments passed to swift test in the Linux nightly main Swift version matrix job." default: "" + windows_6_0_enabled: + type: boolean + description: "Boolean to enable the Windows 6.0 Swift version matrix job. Defaults to true." + default: false + windows_6_0_arguments_override: + type: string + description: "The arguments passed to swift test in the Windows 6.0 Swift version matrix job." + default: "" + windows_nightly_6_0_enabled: + type: boolean + description: "Boolean to enable the Windows nightly 6.0 Swift version matrix job. Defaults to true." + default: false + windows_nightly_6_0_arguments_override: + type: string + description: "The arguments passed to swift test in the Windows nightly 6.0 Swift version matrix job." + default: "" + windows_nightly_main_enabled: + type: boolean + description: "Boolean to enable the Windows nightly main Swift version matrix job. Defaults to true." + default: false + windows_nightly_main_arguments_override: + type: string + description: "The arguments passed to swift test in the Windows nightly main Swift version matrix job." + default: "" + jobs: unit-tests: name: Unit tests @@ -62,3 +87,9 @@ jobs: matrix_linux_nightly_6_0_command_override: "swift test ${{ inputs.linux_nightly_6_0_arguments_override }}" matrix_linux_nightly_main_enabled: ${{ inputs.linux_nightly_main_enabled }} matrix_linux_nightly_main_command_override: "swift test ${{ inputs.linux_nightly_main_arguments_override }}" + matrix_windows_6_0_enabled: ${{ inputs.windows_6_0_enabled }} + matrix_windows_6_0_command_override: "swift test ${{ inputs.windows_6_0_arguments_override }}" + matrix_windows_nightly_6_0_enabled: ${{ inputs.windows_nightly_6_0_enabled }} + matrix_windows_nightly_6_0_command_override: "swift test ${{ inputs.windows_nightly_6_0_arguments_override }}" + matrix_windows_nightly_main_enabled: ${{ inputs.windows_nightly_main_enabled }} + matrix_windows_nightly_main_command_override: "swift test ${{ inputs.windows_nightly_main_arguments_override }}" diff --git a/scripts/check-matrix-job.ps1 b/scripts/check-matrix-job.ps1 new file mode 100644 index 0000000000..c48aeb5505 --- /dev/null +++ b/scripts/check-matrix-job.ps1 @@ -0,0 +1,78 @@ +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +# Set strict mode to catch errors +Set-StrictMode -Version Latest + +function Log { + param ( + [string]$Message + ) + Write-Host ("** " + $Message) -ForegroundColor Yellow +} + +function Error { + param ( + [string]$Message + ) + Write-Host ("** ERROR: " + $Message) -ForegroundColor Red +} + +function Fatal { + param ( + [string]$Message + ) + Error $Message + exit 1 +} + +# Check if SWIFT_VERSION is set +if (-not $env:SWIFT_VERSION) { + Fatal "SWIFT_VERSION unset" +} + +# Check if COMMAND is set +if (-not $env:COMMAND) { + Fatal "COMMAND unset" +} + +$swift_version = $env:SWIFT_VERSION +$command = $env:COMMAND +$command_5_9 = $env:COMMAND_OVERRIDE_5_9 +$command_5_10 = $env:COMMAND_OVERRIDE_5_10 +$command_6_0 = $env:COMMAND_OVERRIDE_6_0 +$command_nightly_6_0 = $env:COMMAND_OVERRIDE_NIGHTLY_6_0 +$command_nightly_main = $env:COMMAND_OVERRIDE_NIGHTLY_MAIN + +if ($swift_version -eq "5.9" -and $command_5_9) { + Log "Running 5.9 command override" + Invoke-Expression $command_5_9 +} elseif ($swift_version -eq "5.10" -and $command_5_10) { + Log "Running 5.10 command override" + Invoke-Expression $command_5_10 +} elseif ($swift_version -eq "6.0" -and $command_6_0) { + Log "Running 6.0 command override" + Invoke-Expression $command_6_0 +} elseif ($swift_version -eq "nightly-6.0" -and $command_nightly_6_0) { + Log "Running nightly 6.0 command override" + Invoke-Expression $command_nightly_6_0 +} elseif ($swift_version -eq "nightly-main" -and $command_nightly_main) { + Log "Running nightly main command override" + Invoke-Expression $command_nightly_main +} else { + Log "Running default command" + Invoke-Expression $command +} + +Exit $LASTEXITCODE From ecbb5ea78a1f1db29a8c19dd0a133fb6acade55e Mon Sep 17 00:00:00 2001 From: Franz Busch Date: Tue, 22 Oct 2024 11:36:48 +0200 Subject: [PATCH 11/20] [CI] Small adjustments to the Windows actions (#2939) # Motivation There were two small problems: 1. The CI for the nightlies wasn't properly skipped 2. The name for the matrix wasn't applied in the case where it was skipped # Modification This PR hopefully fixes both issues. --- .github/workflows/swift_matrix.yml | 4 ++-- .github/workflows/unit_tests.yml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/swift_matrix.yml b/.github/workflows/swift_matrix.yml index 47ad6e94ff..72f918ff65 100644 --- a/.github/workflows/swift_matrix.yml +++ b/.github/workflows/swift_matrix.yml @@ -151,7 +151,6 @@ jobs: windows: name: Windows (${{ matrix.swift.swift_version }}) runs-on: windows-2022 - if: ${{ inputs.matrix_windows_6_0_enabled }} strategy: fail-fast: false matrix: @@ -160,6 +159,7 @@ jobs: - image: swift:6.0-windowsservercore-ltsc2022 swift_version: "6.0" enabled: ${{ inputs.matrix_windows_6_0_enabled }} + if: ${{ inputs.matrix_windows_6_0_enabled }} steps: - name: Pull Docker image if: ${{ matrix.swift.enabled }} @@ -180,7 +180,6 @@ jobs: windows-nightly: name: Windows (${{ matrix.swift.swift_version }}) runs-on: windows-2019 - if: ${{ inputs.matrix_windows_nightly_6_0_enabled }} && ${{ inputs.matrix_windows_nightly_main_enabled }} strategy: fail-fast: false matrix: @@ -192,6 +191,7 @@ jobs: - image: swiftlang/swift:nightly-main-windowsservercore-1809 swift_version: "nightly-main" enabled: ${{ inputs.matrix_windows_nightly_main_enabled }} + if: ${{ inputs.matrix_windows_nightly_6_0_enabled }} || ${{ inputs.matrix_windows_nightly_main_enabled }} steps: - name: Pull Docker image if: ${{ matrix.swift.enabled }} diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 091a2b7655..8a70518bb6 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -87,6 +87,7 @@ jobs: matrix_linux_nightly_6_0_command_override: "swift test ${{ inputs.linux_nightly_6_0_arguments_override }}" matrix_linux_nightly_main_enabled: ${{ inputs.linux_nightly_main_enabled }} matrix_linux_nightly_main_command_override: "swift test ${{ inputs.linux_nightly_main_arguments_override }}" + matrix_windows_command: "swift test" matrix_windows_6_0_enabled: ${{ inputs.windows_6_0_enabled }} matrix_windows_6_0_command_override: "swift test ${{ inputs.windows_6_0_arguments_override }}" matrix_windows_nightly_6_0_enabled: ${{ inputs.windows_nightly_6_0_enabled }} From 6d24ab0222c596e8ad0eeffc99b7d883bffed6a8 Mon Sep 17 00:00:00 2001 From: Franz Busch Date: Wed, 23 Oct 2024 09:38:21 +0200 Subject: [PATCH 12/20] [CI] Enable benchmarks & cxx Windows CI (#2940) This PR makes it possible to enable benchmarks and cxx interop checks on Windows. By default they are turned off. This PR also fixes the job of the Windows 6 pipeline which was broken since single configuration matrix jobs seem to behave differently. --- .github/workflows/benchmarks.yml | 16 ++++++++++++++++ .github/workflows/cxx_interop.yml | 16 ++++++++++++++++ .github/workflows/swift_matrix.yml | 4 ++-- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 502d9ddd51..2b2e4eb38b 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -31,6 +31,19 @@ on: description: "Boolean to enable the Linux nightly main Swift version matrix job. Defaults to true." default: true + windows_6_0_enabled: + type: boolean + description: "Boolean to enable the Windows 6.0 Swift version matrix job. Defaults to true." + default: false + windows_nightly_6_0_enabled: + type: boolean + description: "Boolean to enable the Windows nightly 6.0 Swift version matrix job. Defaults to true." + default: false + windows_nightly_main_enabled: + type: boolean + description: "Boolean to enable the Windows nightly main Swift version matrix job. Defaults to true." + default: false + jobs: benchmarks: name: Benchmarks @@ -44,3 +57,6 @@ jobs: matrix_linux_6_0_enabled: ${{ inputs.linux_6_0_enabled }} matrix_linux_nightly_6_0_enabled: ${{ inputs.linux_nightly_6_0_enabled }} matrix_linux_nightly_main_enabled: ${{ inputs.linux_nightly_main_enabled }} + matrix_windows_6_0_enabled: ${{ inputs.windows_6_0_enabled }} + matrix_windows_nightly_6_0_enabled: ${{ inputs.windows_nightly_6_0_enabled }} + matrix_windows_nightly_main_enabled: ${{ inputs.windows_nightly_main_enabled }} diff --git a/.github/workflows/cxx_interop.yml b/.github/workflows/cxx_interop.yml index e0f9e790a8..3e66426076 100644 --- a/.github/workflows/cxx_interop.yml +++ b/.github/workflows/cxx_interop.yml @@ -24,6 +24,19 @@ on: description: "Boolean to enable the Linux nightly main Swift version matrix job. Defaults to true." default: true + windows_6_0_enabled: + type: boolean + description: "Boolean to enable the Windows 6.0 Swift version matrix job. Defaults to true." + default: false + windows_nightly_6_0_enabled: + type: boolean + description: "Boolean to enable the Windows nightly 6.0 Swift version matrix job. Defaults to true." + default: false + windows_nightly_main_enabled: + type: boolean + description: "Boolean to enable the Windows nightly main Swift version matrix job. Defaults to true." + default: false + jobs: cxx-interop: name: Cxx interop @@ -37,3 +50,6 @@ jobs: matrix_linux_6_0_enabled: ${{ inputs.linux_6_0_enabled }} matrix_linux_nightly_6_0_enabled: ${{ inputs.linux_nightly_6_0_enabled }} matrix_linux_nightly_main_enabled: ${{ inputs.linux_nightly_main_enabled }} + matrix_windows_6_0_enabled: ${{ inputs.windows_6_0_enabled }} + matrix_windows_nightly_6_0_enabled: ${{ inputs.windows_nightly_6_0_enabled }} + matrix_windows_nightly_main_enabled: ${{ inputs.windows_nightly_main_enabled }} diff --git a/.github/workflows/swift_matrix.yml b/.github/workflows/swift_matrix.yml index 72f918ff65..5fa052d46f 100644 --- a/.github/workflows/swift_matrix.yml +++ b/.github/workflows/swift_matrix.yml @@ -149,7 +149,7 @@ jobs: curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-matrix-job.sh | bash windows: - name: Windows (${{ matrix.swift.swift_version }}) + name: Windows (6.0) runs-on: windows-2022 strategy: fail-fast: false @@ -191,7 +191,7 @@ jobs: - image: swiftlang/swift:nightly-main-windowsservercore-1809 swift_version: "nightly-main" enabled: ${{ inputs.matrix_windows_nightly_main_enabled }} - if: ${{ inputs.matrix_windows_nightly_6_0_enabled }} || ${{ inputs.matrix_windows_nightly_main_enabled }} + if: (${{ inputs.matrix_windows_nightly_6_0_enabled }} || ${{ inputs.matrix_windows_nightly_main_enabled }}) steps: - name: Pull Docker image if: ${{ matrix.swift.enabled }} From be823e6d1e0d306709ac8f0f92566241d1fcd910 Mon Sep 17 00:00:00 2001 From: Franz Busch Date: Wed, 23 Oct 2024 10:39:34 +0200 Subject: [PATCH 13/20] [CI] Fix the Windows 6.0 check name (#2941) # Motivation Currently, the Windows 6.0 check name includes the matrix properties. # Modification This PR makes it dynamic with the swift version again. # Result Aligned CI check names across platforms. --- .github/workflows/swift_matrix.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/swift_matrix.yml b/.github/workflows/swift_matrix.yml index 5fa052d46f..a5d0b798b6 100644 --- a/.github/workflows/swift_matrix.yml +++ b/.github/workflows/swift_matrix.yml @@ -149,7 +149,7 @@ jobs: curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-matrix-job.sh | bash windows: - name: Windows (6.0) + name: Windows (${{ matrix.swift.swift_version }}) runs-on: windows-2022 strategy: fail-fast: false @@ -159,7 +159,6 @@ jobs: - image: swift:6.0-windowsservercore-ltsc2022 swift_version: "6.0" enabled: ${{ inputs.matrix_windows_6_0_enabled }} - if: ${{ inputs.matrix_windows_6_0_enabled }} steps: - name: Pull Docker image if: ${{ matrix.swift.enabled }} @@ -191,7 +190,6 @@ jobs: - image: swiftlang/swift:nightly-main-windowsservercore-1809 swift_version: "nightly-main" enabled: ${{ inputs.matrix_windows_nightly_main_enabled }} - if: (${{ inputs.matrix_windows_nightly_6_0_enabled }} || ${{ inputs.matrix_windows_nightly_main_enabled }}) steps: - name: Pull Docker image if: ${{ matrix.swift.enabled }} From f6230d37089fbb52612caf338dc6671e027817b0 Mon Sep 17 00:00:00 2001 From: Johannes Weiss Date: Wed, 23 Oct 2024 17:14:35 +0100 Subject: [PATCH 14/20] NIOPosix on Darwin: inherit main thread QoS (#2944) ### Motivation: On Darwin, QoS (quality of service) of threads plays an important role, especially on Apple Silicon machines with P-cores and E-cores. If you spawn raw threads (like NIOPosix) and use a mechanism that doesn't support QoS propagation (like reading/writing to networks -- like NIOPosix does), it's recommended to default to the main thread's QoS. Otherwise you'll always be at the default QoS for "legacy" threads which means bad latencies, especially on Apple Silicon machines. In a follow-up PR #2943 we're adding better configurability for thread configuration. ### Modifications: Default to main thread QoS on Darwin. ### Result: Better latencies for applications with higher QoS classes. --- Sources/NIOPosix/ThreadPosix.swift | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Sources/NIOPosix/ThreadPosix.swift b/Sources/NIOPosix/ThreadPosix.swift index aacaba5b47..f876c42c8a 100644 --- a/Sources/NIOPosix/ThreadPosix.swift +++ b/Sources/NIOPosix/ThreadPosix.swift @@ -43,7 +43,12 @@ private func sysPthread_create( args: UnsafeMutableRawPointer? ) -> CInt { #if canImport(Darwin) - return pthread_create(handle, nil, destructor, args) + var attr: pthread_attr_t = .init() + pthread_attr_init(&attr) + pthread_attr_set_qos_class_np(&attr, qos_class_main(), 0) + let thread = pthread_create(handle, &attr, destructor, args) + pthread_attr_destroy(&attr) + return thread #else #if canImport(Musl) var handleLinux: OpaquePointer? = nil From c4a6cdea3aa2e3d683939d91fbdb5d01f8f209e8 Mon Sep 17 00:00:00 2001 From: Johannes Weiss Date: Thu, 24 Oct 2024 08:50:47 +0100 Subject: [PATCH 15/20] workaround Xcode 15.4 bug with swift build --arch x86_64 --arch arm64 (#2945) ### Motivation: `swift build --arch x64_64 --arch arm64` on at least Xcode 15.4 fails with ``` /Users/johannes/devel/my-proj/.build/checkouts/swift-nio/Sources/NIOCore/NIOScheduledCallback.swift:93:18: error: '_scheduleCallback' has a package access level but no -package-name was specified: /Users/johannes/devel/tfai-swift/.build/checkouts/swift-nio/Sources/NIOCore/NIOScheduledCallback.swift package func _scheduleCallback( ^ ``` That's clearly an Xcode bug but that needs a workaround. ### Modifications: - Don't use the `package` modifier for now ### Result: SwiftNIO builds again. --- Sources/NIOCore/NIOScheduledCallback.swift | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Sources/NIOCore/NIOScheduledCallback.swift b/Sources/NIOCore/NIOScheduledCallback.swift index b2af415aad..c1405cd93e 100644 --- a/Sources/NIOCore/NIOScheduledCallback.swift +++ b/Sources/NIOCore/NIOScheduledCallback.swift @@ -90,7 +90,11 @@ public struct NIOScheduledCallback: Sendable { extension EventLoop { @preconcurrency - package func _scheduleCallback( + /// This method is not part of the public API + /// + /// Should use `package` not `public` but then it won't compile in + /// Xcode 15.4 if you run `swift build --arch x86_64 --arch arm64`. + public func _scheduleCallback( at deadline: NIODeadline, handler: some (NIOScheduledCallbackHandler & Sendable) ) -> NIOScheduledCallback { From 062b28eeb0b9ee282cbca0577cbb03a998752a56 Mon Sep 17 00:00:00 2001 From: Peter Adams <63288215+PeterAdams-A@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:38:16 +0100 Subject: [PATCH 16/20] Checkout any submodules when running Actions (#2946) Motivation: Some repos using this as their source of GitHub Actions logic have submodules. Modifications: Change workflows to checkout submodules. Result: All CI will work cleanly --- .github/workflows/swift_6_language_mode.yml | 1 + .github/workflows/swift_matrix.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/swift_6_language_mode.yml b/.github/workflows/swift_6_language_mode.yml index 15ecb2c6f3..29a11a5806 100644 --- a/.github/workflows/swift_6_language_mode.yml +++ b/.github/workflows/swift_6_language_mode.yml @@ -19,6 +19,7 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false + submodules: true - name: Set the language mode run: swift package tools-version --set 6.0 - name: Build with Swift 6 language mode diff --git a/.github/workflows/swift_matrix.yml b/.github/workflows/swift_matrix.yml index a5d0b798b6..4c4734cbb6 100644 --- a/.github/workflows/swift_matrix.yml +++ b/.github/workflows/swift_matrix.yml @@ -130,6 +130,7 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false + submodules: true - name: Mark the workspace as safe if: ${{ matrix.swift.enabled }} # https://github.com/actions/checkout/issues/766 @@ -199,6 +200,7 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false + submodules: true - name: Donwload matrix script if: ${{ matrix.swift.enabled }} run: curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-matrix-job.ps1 -o __check-matrix-job.ps1 From 914081701062b11e3bb9e21accc379822621995e Mon Sep 17 00:00:00 2001 From: Ali Ali <47680736+ali-ahsan-ali@users.noreply.github.com> Date: Thu, 24 Oct 2024 21:26:32 +1100 Subject: [PATCH 17/20] Issue-2900 - Add VisionOS support (#2947) Add Vision support ### Motivation: https://github.com/apple/swift-nio/issues/2900 - close this ### Modifications: - Bump SystemPackage to 1.4.0 and remove @preconcurrency - remove `#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android)` to allow for visionOS support - Remove conditional import for swift system ### Result: Vision OS support --- Package.swift | 8 ++------ Sources/NIOFileSystem/BufferedReader.swift | 3 --- Sources/NIOFileSystem/BufferedWriter.swift | 3 --- Sources/NIOFileSystem/ByteBuffer+FileSystem.swift | 3 --- Sources/NIOFileSystem/ByteCount.swift | 4 ---- Sources/NIOFileSystem/Convenience.swift | 3 --- Sources/NIOFileSystem/DirectoryEntries.swift | 5 +---- Sources/NIOFileSystem/DirectoryEntry.swift | 5 +---- Sources/NIOFileSystem/Exports.swift | 4 ---- Sources/NIOFileSystem/FileChunks.swift | 4 +--- Sources/NIOFileSystem/FileHandle.swift | 4 ---- Sources/NIOFileSystem/FileHandleProtocol.swift | 2 -- Sources/NIOFileSystem/FileInfo.swift | 3 --- Sources/NIOFileSystem/FileSystem.swift | 5 +---- Sources/NIOFileSystem/FileSystemError+Syscall.swift | 3 --- Sources/NIOFileSystem/FileSystemError.swift | 3 --- Sources/NIOFileSystem/FileSystemProtocol.swift | 3 --- Sources/NIOFileSystem/FileType.swift | 3 --- Sources/NIOFileSystem/Internal/BufferedOrAnyStream.swift | 3 --- Sources/NIOFileSystem/Internal/BufferedStream.swift | 3 --- Sources/NIOFileSystem/Internal/Cancellation.swift | 4 ---- .../Internal/Concurrency Primitives/UnsafeTransfer.swift | 2 -- Sources/NIOFileSystem/Internal/ParallelDirCopy.swift | 2 -- .../Internal/String+UnsafeUnititializedCapacity.swift | 4 ---- .../NIOFileSystem/Internal/System Calls/CInterop.swift | 2 -- Sources/NIOFileSystem/Internal/System Calls/Errno.swift | 2 -- .../Internal/System Calls/FileDescriptor+Syscalls.swift | 2 -- Sources/NIOFileSystem/Internal/System Calls/Mocking.swift | 2 -- Sources/NIOFileSystem/Internal/System Calls/Syscall.swift | 2 -- .../NIOFileSystem/Internal/System Calls/Syscalls.swift | 2 -- Sources/NIOFileSystem/Internal/SystemFileHandle.swift | 5 +---- Sources/NIOFileSystem/Internal/Utilities.swift | 3 --- Sources/NIOFileSystem/OpenOptions.swift | 3 --- Sources/NIOFileSystemFoundationCompat/Date+FileInfo.swift | 2 -- .../FileSystemFoundationCompatTests.swift | 4 +--- .../BufferedReaderTests.swift | 4 +--- .../BufferedWriterTests.swift | 4 +--- .../NIOFileSystemIntegrationTests/ConvenienceTests.swift | 4 +--- Tests/NIOFileSystemIntegrationTests/FileHandleTests.swift | 6 ++---- .../FileSystemTests+SPI.swift | 4 +--- Tests/NIOFileSystemIntegrationTests/FileSystemTests.swift | 8 +++----- .../NIOFileSystemIntegrationTests/XCTestExtensions.swift | 4 +--- Tests/NIOFileSystemTests/ByteCountTests.swift | 4 +--- Tests/NIOFileSystemTests/DirectoryEntriesTests.swift | 4 +--- Tests/NIOFileSystemTests/FileChunksTests.swift | 4 +--- Tests/NIOFileSystemTests/FileHandleTests.swift | 4 +--- Tests/NIOFileSystemTests/FileInfoTests.swift | 4 +--- Tests/NIOFileSystemTests/FileOpenOptionsTests.swift | 4 +--- Tests/NIOFileSystemTests/FileSystemErrorTests.swift | 4 +--- Tests/NIOFileSystemTests/FileTypeTests.swift | 4 +--- Tests/NIOFileSystemTests/Internal/CancellationTests.swift | 4 +--- .../Concurrency Primitives/BufferedStreamTests.swift | 2 -- .../Internal/MockingInfrastructure.swift | 4 +--- Tests/NIOFileSystemTests/Internal/SyscallTests.swift | 4 +--- Tests/NIOFileSystemTests/XCTestExtensions.swift | 4 +--- 55 files changed, 30 insertions(+), 169 deletions(-) diff --git a/Package.swift b/Package.swift index bb2eaeddcb..3b380614a1 100644 --- a/Package.swift +++ b/Package.swift @@ -21,11 +21,7 @@ import class Foundation.ProcessInfo let swiftAtomics: PackageDescription.Target.Dependency = .product(name: "Atomics", package: "swift-atomics") let swiftCollections: PackageDescription.Target.Dependency = .product(name: "DequeModule", package: "swift-collections") -let swiftSystem: PackageDescription.Target.Dependency = .product( - name: "SystemPackage", - package: "swift-system", - condition: .when(platforms: [.macOS, .iOS, .tvOS, .watchOS, .linux, .android]) -) +let swiftSystem: PackageDescription.Target.Dependency = .product(name: "SystemPackage", package: "swift-system") // These platforms require a depdency on `NIOPosix` from `NIOHTTP1` to maintain backward // compatibility with previous NIO versions. @@ -558,7 +554,7 @@ if Context.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil { package.dependencies += [ .package(url: "https://github.com/apple/swift-atomics.git", from: "1.1.0"), .package(url: "https://github.com/apple/swift-collections.git", from: "1.0.2"), - .package(url: "https://github.com/apple/swift-system.git", from: "1.2.0"), + .package(url: "https://github.com/apple/swift-system.git", from: "1.4.0"), ] } else { package.dependencies += [ diff --git a/Sources/NIOFileSystem/BufferedReader.swift b/Sources/NIOFileSystem/BufferedReader.swift index d66aa5d95e..fabbd1238e 100644 --- a/Sources/NIOFileSystem/BufferedReader.swift +++ b/Sources/NIOFileSystem/BufferedReader.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import DequeModule import NIOCore @@ -240,5 +239,3 @@ extension ReadableFileHandleProtocol { BufferedReader(wrapping: self, initialOffset: initialOffset, capacity: Int(capacity.bytes)) } } - -#endif diff --git a/Sources/NIOFileSystem/BufferedWriter.swift b/Sources/NIOFileSystem/BufferedWriter.swift index 3f02f22181..9763525cf0 100644 --- a/Sources/NIOFileSystem/BufferedWriter.swift +++ b/Sources/NIOFileSystem/BufferedWriter.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import NIOCore /// A writer which buffers bytes in memory before writing them to the file system. @@ -244,5 +243,3 @@ extension WritableFileHandleProtocol { } } } - -#endif diff --git a/Sources/NIOFileSystem/ByteBuffer+FileSystem.swift b/Sources/NIOFileSystem/ByteBuffer+FileSystem.swift index 4c3c394c46..6c5be612e4 100644 --- a/Sources/NIOFileSystem/ByteBuffer+FileSystem.swift +++ b/Sources/NIOFileSystem/ByteBuffer+FileSystem.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import NIOCore @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) @@ -96,5 +95,3 @@ extension ByteBuffer { ) } } - -#endif diff --git a/Sources/NIOFileSystem/ByteCount.swift b/Sources/NIOFileSystem/ByteCount.swift index 276b1f8eee..d881a3ed0c 100644 --- a/Sources/NIOFileSystem/ByteCount.swift +++ b/Sources/NIOFileSystem/ByteCount.swift @@ -12,8 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) - /// Represents the number of bytes. public struct ByteCount: Hashable, Sendable { /// The number of bytes @@ -117,5 +115,3 @@ extension ByteCount: Comparable { lhs.bytes < rhs.bytes } } - -#endif diff --git a/Sources/NIOFileSystem/Convenience.swift b/Sources/NIOFileSystem/Convenience.swift index e07f368b95..54f0a56441 100644 --- a/Sources/NIOFileSystem/Convenience.swift +++ b/Sources/NIOFileSystem/Convenience.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import SystemPackage extension String { @@ -201,5 +200,3 @@ extension AsyncSequence where Self.Element == UInt8, Self: Sendable { ) } } - -#endif diff --git a/Sources/NIOFileSystem/DirectoryEntries.swift b/Sources/NIOFileSystem/DirectoryEntries.swift index 105b2b7b85..3830d94abd 100644 --- a/Sources/NIOFileSystem/DirectoryEntries.swift +++ b/Sources/NIOFileSystem/DirectoryEntries.swift @@ -12,13 +12,12 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import CNIODarwin import CNIOLinux import NIOConcurrencyHelpers import NIOCore import NIOPosix -@preconcurrency import SystemPackage +import SystemPackage /// An `AsyncSequence` of entries in a directory. @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) @@ -737,5 +736,3 @@ extension UnsafeMutablePointer { FilePath(platformString: self.pointee.fts_path!) } } - -#endif diff --git a/Sources/NIOFileSystem/DirectoryEntry.swift b/Sources/NIOFileSystem/DirectoryEntry.swift index 2bb502110f..f8e6692617 100644 --- a/Sources/NIOFileSystem/DirectoryEntry.swift +++ b/Sources/NIOFileSystem/DirectoryEntry.swift @@ -12,8 +12,7 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) -@preconcurrency import SystemPackage +import SystemPackage /// Information about an item within a directory. public struct DirectoryEntry: Sendable, Hashable, Equatable { @@ -46,5 +45,3 @@ public struct DirectoryEntry: Sendable, Hashable, Equatable { self.type = type } } - -#endif diff --git a/Sources/NIOFileSystem/Exports.swift b/Sources/NIOFileSystem/Exports.swift index f7f0c244cc..2b73a58783 100644 --- a/Sources/NIOFileSystem/Exports.swift +++ b/Sources/NIOFileSystem/Exports.swift @@ -12,8 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) - // These types are used in our public API; expose them to make // life easier for users. @_exported import enum SystemPackage.CInterop @@ -21,5 +19,3 @@ @_exported import struct SystemPackage.FileDescriptor @_exported import struct SystemPackage.FilePath @_exported import struct SystemPackage.FilePermissions - -#endif diff --git a/Sources/NIOFileSystem/FileChunks.swift b/Sources/NIOFileSystem/FileChunks.swift index 337dcbe42f..00338236b0 100644 --- a/Sources/NIOFileSystem/FileChunks.swift +++ b/Sources/NIOFileSystem/FileChunks.swift @@ -12,11 +12,10 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import NIOConcurrencyHelpers import NIOCore import NIOPosix -@preconcurrency import SystemPackage +import SystemPackage /// An `AsyncSequence` of ordered chunks read from a file. @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) @@ -544,4 +543,3 @@ extension ProducerState.Producing { return .moreToRead } } -#endif diff --git a/Sources/NIOFileSystem/FileHandle.swift b/Sources/NIOFileSystem/FileHandle.swift index 2f6f0ed47f..287c7408b5 100644 --- a/Sources/NIOFileSystem/FileHandle.swift +++ b/Sources/NIOFileSystem/FileHandle.swift @@ -12,8 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) - import NIOCore /// Provides a ``FileHandle``. @@ -357,5 +355,3 @@ public struct DirectoryFileHandle: DirectoryFileHandleProtocol, _HasFileHandle { return DirectoryFileHandle(wrapping: systemFileHandle) } } - -#endif diff --git a/Sources/NIOFileSystem/FileHandleProtocol.swift b/Sources/NIOFileSystem/FileHandleProtocol.swift index b99136eb20..3251de175f 100644 --- a/Sources/NIOFileSystem/FileHandleProtocol.swift +++ b/Sources/NIOFileSystem/FileHandleProtocol.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import NIOCore import SystemPackage @@ -775,4 +774,3 @@ extension DirectoryFileHandleProtocol { } } } -#endif diff --git a/Sources/NIOFileSystem/FileInfo.swift b/Sources/NIOFileSystem/FileInfo.swift index 9c33fcc302..0524c6128b 100644 --- a/Sources/NIOFileSystem/FileInfo.swift +++ b/Sources/NIOFileSystem/FileInfo.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import SystemPackage #if canImport(Darwin) @@ -277,5 +276,3 @@ extension FilePermissions { self = .init(rawValue: rawValue & ~S_IFMT) } } - -#endif diff --git a/Sources/NIOFileSystem/FileSystem.swift b/Sources/NIOFileSystem/FileSystem.swift index 291102d014..64de1b8669 100644 --- a/Sources/NIOFileSystem/FileSystem.swift +++ b/Sources/NIOFileSystem/FileSystem.swift @@ -12,12 +12,10 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) - import Atomics import NIOCore import NIOPosix -@preconcurrency import SystemPackage +import SystemPackage #if canImport(Darwin) import Darwin @@ -1539,4 +1537,3 @@ extension FileSystem { } } } -#endif diff --git a/Sources/NIOFileSystem/FileSystemError+Syscall.swift b/Sources/NIOFileSystem/FileSystemError+Syscall.swift index 633b6dc7e1..264c714718 100644 --- a/Sources/NIOFileSystem/FileSystemError+Syscall.swift +++ b/Sources/NIOFileSystem/FileSystemError+Syscall.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import SystemPackage #if canImport(Darwin) @@ -1157,5 +1156,3 @@ extension FileSystemError { ) } } - -#endif diff --git a/Sources/NIOFileSystem/FileSystemError.swift b/Sources/NIOFileSystem/FileSystemError.swift index 801fcd699c..8a29ccb36c 100644 --- a/Sources/NIOFileSystem/FileSystemError.swift +++ b/Sources/NIOFileSystem/FileSystemError.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import SystemPackage /// An error thrown as a result of interaction with the file system. @@ -279,5 +278,3 @@ extension FileSystemError { } } } - -#endif diff --git a/Sources/NIOFileSystem/FileSystemProtocol.swift b/Sources/NIOFileSystem/FileSystemProtocol.swift index 7163a88469..e420a4d10a 100644 --- a/Sources/NIOFileSystem/FileSystemProtocol.swift +++ b/Sources/NIOFileSystem/FileSystemProtocol.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import SystemPackage /// The interface for interacting with a file system. @@ -663,5 +662,3 @@ extension FileSystemProtocol { } } } - -#endif diff --git a/Sources/NIOFileSystem/FileType.swift b/Sources/NIOFileSystem/FileType.swift index 10932a6c89..dd38518ade 100644 --- a/Sources/NIOFileSystem/FileType.swift +++ b/Sources/NIOFileSystem/FileType.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import SystemPackage #if canImport(Darwin) @@ -170,5 +169,3 @@ extension FileType { } } } - -#endif diff --git a/Sources/NIOFileSystem/Internal/BufferedOrAnyStream.swift b/Sources/NIOFileSystem/Internal/BufferedOrAnyStream.swift index 2c88405c02..d28b756b96 100644 --- a/Sources/NIOFileSystem/Internal/BufferedOrAnyStream.swift +++ b/Sources/NIOFileSystem/Internal/BufferedOrAnyStream.swift @@ -14,7 +14,6 @@ import NIOCore -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) /// Wraps a ``NIOThrowingAsyncSequenceProducer`` or ``AnyAsyncSequence``. @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) internal enum BufferedOrAnyStream { @@ -95,5 +94,3 @@ internal struct AnyAsyncSequence: AsyncSequence { } } } - -#endif diff --git a/Sources/NIOFileSystem/Internal/BufferedStream.swift b/Sources/NIOFileSystem/Internal/BufferedStream.swift index 6653e4f8c0..d68ed6b61d 100644 --- a/Sources/NIOFileSystem/Internal/BufferedStream.swift +++ b/Sources/NIOFileSystem/Internal/BufferedStream.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import DequeModule import NIOConcurrencyHelpers @@ -1732,5 +1731,3 @@ extension BufferedStream { } } } - -#endif diff --git a/Sources/NIOFileSystem/Internal/Cancellation.swift b/Sources/NIOFileSystem/Internal/Cancellation.swift index 307e78c6be..467ea3c5b3 100644 --- a/Sources/NIOFileSystem/Internal/Cancellation.swift +++ b/Sources/NIOFileSystem/Internal/Cancellation.swift @@ -12,8 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) - /// Executes the closure and masks cancellation. @_spi(Testing) @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) @@ -54,5 +52,3 @@ public func withUncancellableTearDown( try tearDownResult.get() return try result.get() } - -#endif diff --git a/Sources/NIOFileSystem/Internal/Concurrency Primitives/UnsafeTransfer.swift b/Sources/NIOFileSystem/Internal/Concurrency Primitives/UnsafeTransfer.swift index f6cf49d439..c39dd0b5e7 100644 --- a/Sources/NIOFileSystem/Internal/Concurrency Primitives/UnsafeTransfer.swift +++ b/Sources/NIOFileSystem/Internal/Concurrency Primitives/UnsafeTransfer.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) @usableFromInline struct UnsafeTransfer: @unchecked Sendable { @usableFromInline @@ -23,4 +22,3 @@ struct UnsafeTransfer: @unchecked Sendable { self.wrappedValue = wrappedValue } } -#endif diff --git a/Sources/NIOFileSystem/Internal/ParallelDirCopy.swift b/Sources/NIOFileSystem/Internal/ParallelDirCopy.swift index d2010b6523..8f1d6b3813 100644 --- a/Sources/NIOFileSystem/Internal/ParallelDirCopy.swift +++ b/Sources/NIOFileSystem/Internal/ParallelDirCopy.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import NIOCore @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) @@ -161,4 +160,3 @@ private struct DirCopyDelegate: NIOAsyncSequenceProducerDelegate, Sendable { @inlinable func didTerminate() {} } -#endif diff --git a/Sources/NIOFileSystem/Internal/String+UnsafeUnititializedCapacity.swift b/Sources/NIOFileSystem/Internal/String+UnsafeUnititializedCapacity.swift index 5a9643843e..54c20a92af 100644 --- a/Sources/NIOFileSystem/Internal/String+UnsafeUnititializedCapacity.swift +++ b/Sources/NIOFileSystem/Internal/String+UnsafeUnititializedCapacity.swift @@ -12,8 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) - extension String { @inlinable init( @@ -61,5 +59,3 @@ extension String { } } } - -#endif diff --git a/Sources/NIOFileSystem/Internal/System Calls/CInterop.swift b/Sources/NIOFileSystem/Internal/System Calls/CInterop.swift index 2962128aac..57408615f8 100644 --- a/Sources/NIOFileSystem/Internal/System Calls/CInterop.swift +++ b/Sources/NIOFileSystem/Internal/System Calls/CInterop.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import SystemPackage #if canImport(Darwin) @@ -82,4 +81,3 @@ extension CInterop { typealias FTSPointer = UnsafeMutablePointer typealias FTSEntPointer = UnsafeMutablePointer } -#endif diff --git a/Sources/NIOFileSystem/Internal/System Calls/Errno.swift b/Sources/NIOFileSystem/Internal/System Calls/Errno.swift index 171b958708..1b011a0cb8 100644 --- a/Sources/NIOFileSystem/Internal/System Calls/Errno.swift +++ b/Sources/NIOFileSystem/Internal/System Calls/Errno.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import SystemPackage #if canImport(Darwin) @@ -147,4 +146,3 @@ public func valueOrErrno( } } } -#endif diff --git a/Sources/NIOFileSystem/Internal/System Calls/FileDescriptor+Syscalls.swift b/Sources/NIOFileSystem/Internal/System Calls/FileDescriptor+Syscalls.swift index 9212839c29..8b16de0f6c 100644 --- a/Sources/NIOFileSystem/Internal/System Calls/FileDescriptor+Syscalls.swift +++ b/Sources/NIOFileSystem/Internal/System Calls/FileDescriptor+Syscalls.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import NIOCore import SystemPackage @@ -325,4 +324,3 @@ extension FileDescriptor { } } #endif -#endif diff --git a/Sources/NIOFileSystem/Internal/System Calls/Mocking.swift b/Sources/NIOFileSystem/Internal/System Calls/Mocking.swift index 1978d24f35..b2a9288a36 100644 --- a/Sources/NIOFileSystem/Internal/System Calls/Mocking.swift +++ b/Sources/NIOFileSystem/Internal/System Calls/Mocking.swift @@ -17,7 +17,6 @@ // Licensed under Apache License v2.0 with Runtime Library Exception// // See https://swift.org/LICENSE.txt for license information -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import SystemPackage #if canImport(Darwin) @@ -370,4 +369,3 @@ internal func setTLS(_ key: _PlatformTLSKey, _ p: UnsafeMutableRawPointer?) { internal func getTLS(_ key: _PlatformTLSKey) -> UnsafeMutableRawPointer? { pthread_getspecific(key) } -#endif diff --git a/Sources/NIOFileSystem/Internal/System Calls/Syscall.swift b/Sources/NIOFileSystem/Internal/System Calls/Syscall.swift index 0a80b9f1dd..de53259404 100644 --- a/Sources/NIOFileSystem/Internal/System Calls/Syscall.swift +++ b/Sources/NIOFileSystem/Internal/System Calls/Syscall.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import SystemPackage #if canImport(Darwin) @@ -443,4 +442,3 @@ public enum Libc { } } } -#endif diff --git a/Sources/NIOFileSystem/Internal/System Calls/Syscalls.swift b/Sources/NIOFileSystem/Internal/System Calls/Syscalls.swift index 1e9c166132..dcff578018 100644 --- a/Sources/NIOFileSystem/Internal/System Calls/Syscalls.swift +++ b/Sources/NIOFileSystem/Internal/System Calls/Syscalls.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import SystemPackage #if canImport(Darwin) @@ -482,4 +481,3 @@ internal func libc_fts_close( ) -> CInt { fts_close(fts) } -#endif diff --git a/Sources/NIOFileSystem/Internal/SystemFileHandle.swift b/Sources/NIOFileSystem/Internal/SystemFileHandle.swift index 4140657377..7de45d39c5 100644 --- a/Sources/NIOFileSystem/Internal/SystemFileHandle.swift +++ b/Sources/NIOFileSystem/Internal/SystemFileHandle.swift @@ -12,11 +12,10 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import NIOConcurrencyHelpers import NIOCore import NIOPosix -@preconcurrency import SystemPackage +import SystemPackage #if canImport(Darwin) import Darwin @@ -1536,5 +1535,3 @@ extension SystemFileHandle { } } } - -#endif diff --git a/Sources/NIOFileSystem/Internal/Utilities.swift b/Sources/NIOFileSystem/Internal/Utilities.swift index 1f90b45d0f..9d6b944582 100644 --- a/Sources/NIOFileSystem/Internal/Utilities.swift +++ b/Sources/NIOFileSystem/Internal/Utilities.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import SystemPackage @usableFromInline @@ -44,5 +43,3 @@ extension Array where Element == UInt8 { return alphaNumericValues }() } - -#endif diff --git a/Sources/NIOFileSystem/OpenOptions.swift b/Sources/NIOFileSystem/OpenOptions.swift index 454826897a..5907243063 100644 --- a/Sources/NIOFileSystem/OpenOptions.swift +++ b/Sources/NIOFileSystem/OpenOptions.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import SystemPackage /// Options for opening file handles. @@ -296,5 +295,3 @@ extension FilePermissions { .otherReadExecute, ] } - -#endif diff --git a/Sources/NIOFileSystemFoundationCompat/Date+FileInfo.swift b/Sources/NIOFileSystemFoundationCompat/Date+FileInfo.swift index f65db1f773..9acb95a351 100644 --- a/Sources/NIOFileSystemFoundationCompat/Date+FileInfo.swift +++ b/Sources/NIOFileSystemFoundationCompat/Date+FileInfo.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import _NIOFileSystem import struct Foundation.Date @@ -30,4 +29,3 @@ extension FileInfo.Timespec { Date(timespec: self) } } -#endif diff --git a/Tests/NIOFileSystemFoundationCompatTests/FileSystemFoundationCompatTests.swift b/Tests/NIOFileSystemFoundationCompatTests/FileSystemFoundationCompatTests.swift index 717b9762dd..70cfc65bc1 100644 --- a/Tests/NIOFileSystemFoundationCompatTests/FileSystemFoundationCompatTests.swift +++ b/Tests/NIOFileSystemFoundationCompatTests/FileSystemFoundationCompatTests.swift @@ -12,10 +12,9 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) +import XCTest import _NIOFileSystem import _NIOFileSystemFoundationCompat -import XCTest @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension FileSystem { @@ -78,4 +77,3 @@ final class FileSystemBytesConformanceTests: XCTestCase { XCTAssertEqual(contents, Data([0, 1, 2])) } } -#endif diff --git a/Tests/NIOFileSystemIntegrationTests/BufferedReaderTests.swift b/Tests/NIOFileSystemIntegrationTests/BufferedReaderTests.swift index e3cef90f63..839821ee3b 100644 --- a/Tests/NIOFileSystemIntegrationTests/BufferedReaderTests.swift +++ b/Tests/NIOFileSystemIntegrationTests/BufferedReaderTests.swift @@ -12,10 +12,9 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import NIOCore -import _NIOFileSystem import XCTest +import _NIOFileSystem @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) final class BufferedReaderTests: XCTestCase { @@ -280,4 +279,3 @@ final class BufferedReaderTests: XCTestCase { } } } -#endif diff --git a/Tests/NIOFileSystemIntegrationTests/BufferedWriterTests.swift b/Tests/NIOFileSystemIntegrationTests/BufferedWriterTests.swift index 47b6a5e98e..31353108b4 100644 --- a/Tests/NIOFileSystemIntegrationTests/BufferedWriterTests.swift +++ b/Tests/NIOFileSystemIntegrationTests/BufferedWriterTests.swift @@ -12,10 +12,9 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import NIOCore -@_spi(Testing) import _NIOFileSystem import XCTest +@_spi(Testing) import _NIOFileSystem @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) final class BufferedWriterTests: XCTestCase { @@ -254,4 +253,3 @@ final class BufferedWriterTests: XCTestCase { } } } -#endif diff --git a/Tests/NIOFileSystemIntegrationTests/ConvenienceTests.swift b/Tests/NIOFileSystemIntegrationTests/ConvenienceTests.swift index ddc0fca15a..85a09505df 100644 --- a/Tests/NIOFileSystemIntegrationTests/ConvenienceTests.swift +++ b/Tests/NIOFileSystemIntegrationTests/ConvenienceTests.swift @@ -12,10 +12,9 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import NIOCore -import _NIOFileSystem import XCTest +import _NIOFileSystem @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) final class ConvenienceTests: XCTestCase { @@ -72,4 +71,3 @@ final class ConvenienceTests: XCTestCase { XCTAssertEqual(bytes, ByteBuffer(bytes: Array(0..<64))) } } -#endif diff --git a/Tests/NIOFileSystemIntegrationTests/FileHandleTests.swift b/Tests/NIOFileSystemIntegrationTests/FileHandleTests.swift index 17217b4008..2e828ff9d5 100644 --- a/Tests/NIOFileSystemIntegrationTests/FileHandleTests.swift +++ b/Tests/NIOFileSystemIntegrationTests/FileHandleTests.swift @@ -12,12 +12,11 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import NIOCore -import NIOPosix -@_spi(Testing) import _NIOFileSystem import NIOFoundationCompat +import NIOPosix import XCTest +@_spi(Testing) import _NIOFileSystem @available(macOS 12.0, iOS 15.0, watchOS 8.0, tvOS 15.0, *) final class FileHandleTests: XCTestCase { @@ -1314,4 +1313,3 @@ private func assertThrowsErrorClosed( XCTAssertEqual(error.code, .closed) } } -#endif diff --git a/Tests/NIOFileSystemIntegrationTests/FileSystemTests+SPI.swift b/Tests/NIOFileSystemIntegrationTests/FileSystemTests+SPI.swift index b112a5c199..c42251bbaf 100644 --- a/Tests/NIOFileSystemIntegrationTests/FileSystemTests+SPI.swift +++ b/Tests/NIOFileSystemIntegrationTests/FileSystemTests+SPI.swift @@ -12,10 +12,9 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) -@_spi(Testing) import _NIOFileSystem import SystemPackage import XCTest +@_spi(Testing) import _NIOFileSystem @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension FileSystemTests { @@ -26,4 +25,3 @@ extension FileSystemTests { XCTAssertEqual(removed, 0) } } -#endif diff --git a/Tests/NIOFileSystemIntegrationTests/FileSystemTests.swift b/Tests/NIOFileSystemIntegrationTests/FileSystemTests.swift index a7c2786a27..5de5846603 100644 --- a/Tests/NIOFileSystemIntegrationTests/FileSystemTests.swift +++ b/Tests/NIOFileSystemIntegrationTests/FileSystemTests.swift @@ -12,12 +12,11 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) +import NIOConcurrencyHelpers import NIOCore -@_spi(Testing) @testable import _NIOFileSystem -@preconcurrency import SystemPackage +import SystemPackage import XCTest -import NIOConcurrencyHelpers +@_spi(Testing) @testable import _NIOFileSystem extension FilePath { static let testData = FilePath(#filePath) @@ -1865,4 +1864,3 @@ extension XCTestCase { } } #endif -#endif diff --git a/Tests/NIOFileSystemIntegrationTests/XCTestExtensions.swift b/Tests/NIOFileSystemIntegrationTests/XCTestExtensions.swift index 2c46e5b844..cf401f93db 100644 --- a/Tests/NIOFileSystemIntegrationTests/XCTestExtensions.swift +++ b/Tests/NIOFileSystemIntegrationTests/XCTestExtensions.swift @@ -12,9 +12,8 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) -import _NIOFileSystem import XCTest +import _NIOFileSystem @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) func XCTAssertThrowsErrorAsync( @@ -81,4 +80,3 @@ func XCTAssertNoThrowAsync( XCTFail("Expression did throw: \(error)", file: file, line: line) } } -#endif diff --git a/Tests/NIOFileSystemTests/ByteCountTests.swift b/Tests/NIOFileSystemTests/ByteCountTests.swift index f1a67387da..7d42253dab 100644 --- a/Tests/NIOFileSystemTests/ByteCountTests.swift +++ b/Tests/NIOFileSystemTests/ByteCountTests.swift @@ -12,9 +12,8 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) -import _NIOFileSystem import XCTest +import _NIOFileSystem class ByteCountTests: XCTestCase { func testByteCountBytes() { @@ -90,4 +89,3 @@ class ByteCountTests: XCTestCase { XCTAssertGreaterThan(byteCount2, byteCount1) } } -#endif diff --git a/Tests/NIOFileSystemTests/DirectoryEntriesTests.swift b/Tests/NIOFileSystemTests/DirectoryEntriesTests.swift index c882e0d297..6e8df1b44e 100644 --- a/Tests/NIOFileSystemTests/DirectoryEntriesTests.swift +++ b/Tests/NIOFileSystemTests/DirectoryEntriesTests.swift @@ -12,9 +12,8 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) -import _NIOFileSystem import XCTest +import _NIOFileSystem @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) final class DirectoryEntriesTests: XCTestCase { @@ -71,4 +70,3 @@ final class DirectoryEntriesTests: XCTestCase { XCTAssertNil(end) } } -#endif diff --git a/Tests/NIOFileSystemTests/FileChunksTests.swift b/Tests/NIOFileSystemTests/FileChunksTests.swift index fd8dc310f3..31841d3a96 100644 --- a/Tests/NIOFileSystemTests/FileChunksTests.swift +++ b/Tests/NIOFileSystemTests/FileChunksTests.swift @@ -12,10 +12,9 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import NIOCore -import _NIOFileSystem import XCTest +import _NIOFileSystem @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) final class FileChunksTests: XCTestCase { @@ -39,4 +38,3 @@ final class FileChunksTests: XCTestCase { XCTAssertNil(end) } } -#endif diff --git a/Tests/NIOFileSystemTests/FileHandleTests.swift b/Tests/NIOFileSystemTests/FileHandleTests.swift index 22ba37a32c..0ae1a7b901 100644 --- a/Tests/NIOFileSystemTests/FileHandleTests.swift +++ b/Tests/NIOFileSystemTests/FileHandleTests.swift @@ -12,10 +12,9 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) -@_spi(Testing) import _NIOFileSystem import NIOPosix import XCTest +@_spi(Testing) import _NIOFileSystem #if ENABLE_MOCKING @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) @@ -271,4 +270,3 @@ extension MockingDriver { } } #endif -#endif diff --git a/Tests/NIOFileSystemTests/FileInfoTests.swift b/Tests/NIOFileSystemTests/FileInfoTests.swift index e69e772084..2e679d9a90 100644 --- a/Tests/NIOFileSystemTests/FileInfoTests.swift +++ b/Tests/NIOFileSystemTests/FileInfoTests.swift @@ -12,9 +12,8 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) -import _NIOFileSystem import XCTest +import _NIOFileSystem #if canImport(Darwin) import Darwin @@ -159,4 +158,3 @@ final class FileInfoTests: XCTestCase { #endif } } -#endif diff --git a/Tests/NIOFileSystemTests/FileOpenOptionsTests.swift b/Tests/NIOFileSystemTests/FileOpenOptionsTests.swift index 6424a560c7..a36652c31b 100644 --- a/Tests/NIOFileSystemTests/FileOpenOptionsTests.swift +++ b/Tests/NIOFileSystemTests/FileOpenOptionsTests.swift @@ -12,9 +12,8 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) -@_spi(Testing) import _NIOFileSystem import XCTest +@_spi(Testing) import _NIOFileSystem final class FileOpenOptionsTests: XCTestCase { private let expectedDefaults: FilePermissions = [ @@ -99,4 +98,3 @@ final class FileOpenOptionsTests: XCTestCase { XCTAssertEqual(FileDescriptor.OpenOptions(options), [.create, .exclusiveCreate, .noFollow]) } } -#endif diff --git a/Tests/NIOFileSystemTests/FileSystemErrorTests.swift b/Tests/NIOFileSystemTests/FileSystemErrorTests.swift index 83e504b277..ddc25b4e64 100644 --- a/Tests/NIOFileSystemTests/FileSystemErrorTests.swift +++ b/Tests/NIOFileSystemTests/FileSystemErrorTests.swift @@ -12,9 +12,8 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) -@_spi(Testing) import _NIOFileSystem import XCTest +@_spi(Testing) import _NIOFileSystem final class FileSystemErrorTests: XCTestCase { func testFileSystemErrorCustomStringConvertible() throws { @@ -623,4 +622,3 @@ private func assertCauseIsSyscall( extension FileSystemError.SourceLocation { fileprivate static let fixed = Self(function: "fn", file: "file", line: 1) } -#endif diff --git a/Tests/NIOFileSystemTests/FileTypeTests.swift b/Tests/NIOFileSystemTests/FileTypeTests.swift index 4b436d87ff..0655d32a59 100644 --- a/Tests/NIOFileSystemTests/FileTypeTests.swift +++ b/Tests/NIOFileSystemTests/FileTypeTests.swift @@ -12,9 +12,8 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) -@_spi(Testing) import _NIOFileSystem import XCTest +@_spi(Testing) import _NIOFileSystem #if canImport(Darwin) import Darwin @@ -80,4 +79,3 @@ final class FileTypeTests: XCTestCase { #endif } } -#endif diff --git a/Tests/NIOFileSystemTests/Internal/CancellationTests.swift b/Tests/NIOFileSystemTests/Internal/CancellationTests.swift index cd18efac14..ca03ae78d3 100644 --- a/Tests/NIOFileSystemTests/Internal/CancellationTests.swift +++ b/Tests/NIOFileSystemTests/Internal/CancellationTests.swift @@ -12,10 +12,9 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import Atomics -@_spi(Testing) import _NIOFileSystem import XCTest +@_spi(Testing) import _NIOFileSystem @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) final class CancellationTests: XCTestCase { @@ -88,4 +87,3 @@ final class CancellationTests: XCTestCase { } } } -#endif diff --git a/Tests/NIOFileSystemTests/Internal/Concurrency Primitives/BufferedStreamTests.swift b/Tests/NIOFileSystemTests/Internal/Concurrency Primitives/BufferedStreamTests.swift index 4791e66f0e..d3451ab243 100644 --- a/Tests/NIOFileSystemTests/Internal/Concurrency Primitives/BufferedStreamTests.swift +++ b/Tests/NIOFileSystemTests/Internal/Concurrency Primitives/BufferedStreamTests.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) import XCTest @testable import _NIOFileSystem @@ -1140,4 +1139,3 @@ extension AsyncThrowingStream { return (stream, continuation!) } } -#endif diff --git a/Tests/NIOFileSystemTests/Internal/MockingInfrastructure.swift b/Tests/NIOFileSystemTests/Internal/MockingInfrastructure.swift index b61b3444f1..9eea5fbc62 100644 --- a/Tests/NIOFileSystemTests/Internal/MockingInfrastructure.swift +++ b/Tests/NIOFileSystemTests/Internal/MockingInfrastructure.swift @@ -19,10 +19,9 @@ // //See https://swift.org/LICENSE.txt for license information -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) -@_spi(Testing) import _NIOFileSystem import SystemPackage import XCTest +@_spi(Testing) import _NIOFileSystem #if ENABLE_MOCKING internal struct Wildcard: Hashable {} @@ -253,4 +252,3 @@ internal func withWindowsPaths(enabled: Bool, _ body: () -> Void) { _withWindowsPaths(enabled: enabled, body) } #endif -#endif diff --git a/Tests/NIOFileSystemTests/Internal/SyscallTests.swift b/Tests/NIOFileSystemTests/Internal/SyscallTests.swift index 66ed1171c1..2ca5204211 100644 --- a/Tests/NIOFileSystemTests/Internal/SyscallTests.swift +++ b/Tests/NIOFileSystemTests/Internal/SyscallTests.swift @@ -12,10 +12,9 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) -@_spi(Testing) import _NIOFileSystem import SystemPackage import XCTest +@_spi(Testing) import _NIOFileSystem #if ENABLE_MOCKING final class SyscallTests: XCTestCase { @@ -519,4 +518,3 @@ extension Array where Element == MockTestCase { } } #endif -#endif diff --git a/Tests/NIOFileSystemTests/XCTestExtensions.swift b/Tests/NIOFileSystemTests/XCTestExtensions.swift index 304207df18..177a874653 100644 --- a/Tests/NIOFileSystemTests/XCTestExtensions.swift +++ b/Tests/NIOFileSystemTests/XCTestExtensions.swift @@ -12,9 +12,8 @@ // //===----------------------------------------------------------------------===// -#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) || os(Linux) || os(Android) -import _NIOFileSystem import XCTest +import _NIOFileSystem @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) func XCTAssertThrowsErrorAsync( @@ -68,4 +67,3 @@ func XCTAssertSystemCallError( XCTAssertEqual(systemCallError.systemCall, name, file: file, line: line) XCTAssertEqual(systemCallError.errno, errno, file: file, line: line) } -#endif From 1ff5fd585ce6583f96ff551473d55e9e81dc4450 Mon Sep 17 00:00:00 2001 From: Johannes Weiss Date: Fri, 25 Oct 2024 08:55:01 +0100 Subject: [PATCH 18/20] EmbeddedEventLoop/Channel: check correct thread (#2951) ### Motivation: `EmbeddedChannel` & `EmbeddedEventLoop` currently violate `Sendable`. They should store the current thread in `init` and then implement `inEventLoop` and other functions with a check that the current thread is correct. Since NIO 1.0 `Embedded*` were always documented to not be thread-safe but we should finally police this. ### Modifications: - Implement the thread check - For now, just warn (soon hopefully crash) - Delete EmbeddedScheduledCallbackTests (#2950) ### Result: - Less Embedded* abuse - fixes #2949 --- Sources/NIOEmbedded/Embedded.swift | 139 +++++++++++++++--- Sources/NIOPosix/Thread.swift | 6 +- .../DispatchQueue+WithFutureTest.swift | 13 +- Tests/NIOPosixTests/EventLoopFutureTest.swift | 19 ++- .../NIOScheduledCallbackTests.swift | 23 --- 5 files changed, 145 insertions(+), 55 deletions(-) diff --git a/Sources/NIOEmbedded/Embedded.swift b/Sources/NIOEmbedded/Embedded.swift index 3105f3af85..bf091a2894 100644 --- a/Sources/NIOEmbedded/Embedded.swift +++ b/Sources/NIOEmbedded/Embedded.swift @@ -22,6 +22,20 @@ import _NIODataStructures import Dispatch #endif +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#elseif canImport(Bionic) +import Bionic +#elseif canImport(WASILibc) +import WASILibc +#else +#error("Unknown C library.") +#endif + internal struct EmbeddedScheduledTask { let id: UInt64 let task: () -> Void @@ -99,6 +113,18 @@ public final class EmbeddedEventLoop: EventLoop, CustomStringConvertible { public let description = "EmbeddedEventLoop" + #if canImport(Darwin) || canImport(Glibc) || canImport(Musl) || canImport(Bionic) + private let myThread: pthread_t = pthread_self() + + func isCorrectThread() -> Bool { + pthread_equal(self.myThread, pthread_self()) != 0 + } + #else + func isCorrectThread() -> Bool { + true // let's be conservative + } + #endif + private func nextTaskNumber() -> UInt64 { defer { self.taskNumber += 1 @@ -108,7 +134,29 @@ public final class EmbeddedEventLoop: EventLoop, CustomStringConvertible { /// - see: `EventLoop.inEventLoop` public var inEventLoop: Bool { - true + self.checkCorrectThread() + return true + } + + public func checkCorrectThread() { + guard self.isCorrectThread() else { + if Self.strictModeEnabled { + preconditionFailure( + "EmbeddedEventLoop is not thread-safe. You can only use it from the thread you created it on." + ) + } else { + fputs( + """ + ERROR: NIO API misuse: EmbeddedEventLoop is not thread-safe. \ + You can only use it from the thread you created it on. This problem will be upgraded to a forced \ + crash in future versions of SwiftNIO. + + """, + stderr + ) + } + return + } } /// Initialize a new `EmbeddedEventLoop`. @@ -117,6 +165,7 @@ public final class EmbeddedEventLoop: EventLoop, CustomStringConvertible { /// - see: `EventLoop.scheduleTask(deadline:_:)` @discardableResult public func scheduleTask(deadline: NIODeadline, _ task: @escaping () throws -> T) -> Scheduled { + self.checkCorrectThread() let promise: EventLoopPromise = makePromise() switch self.state { @@ -157,7 +206,8 @@ public final class EmbeddedEventLoop: EventLoop, CustomStringConvertible { /// - see: `EventLoop.scheduleTask(in:_:)` @discardableResult public func scheduleTask(in: TimeAmount, _ task: @escaping () throws -> T) -> Scheduled { - scheduleTask(deadline: self._now + `in`, task) + self.checkCorrectThread() + return self.scheduleTask(deadline: self._now + `in`, task) } @preconcurrency @@ -166,15 +216,17 @@ public final class EmbeddedEventLoop: EventLoop, CustomStringConvertible { in amount: TimeAmount, handler: some (NIOScheduledCallbackHandler & Sendable) ) -> NIOScheduledCallback { + self.checkCorrectThread() /// Even though this type does not implement a custom `scheduleCallback(at:handler)`, it uses a manual clock so /// it cannot rely on the default implementation of `scheduleCallback(in:handler:)`, which computes the deadline /// as an offset from `NIODeadline.now`. This event loop needs the deadline to be offset from `self._now`. - self.scheduleCallback(at: self._now + amount, handler: handler) + return self.scheduleCallback(at: self._now + amount, handler: handler) } /// On an `EmbeddedEventLoop`, `execute` will simply use `scheduleTask` with a deadline of _now_. This means that /// `task` will be run the next time you call `EmbeddedEventLoop.run`. public func execute(_ task: @escaping () -> Void) { + self.checkCorrectThread() self.scheduleTask(deadline: self._now, task) } @@ -184,6 +236,7 @@ public final class EmbeddedEventLoop: EventLoop, CustomStringConvertible { /// /// - seealso: `EmbeddedEventLoop.advanceTime`. public func run() { + self.checkCorrectThread() // Execute all tasks that are currently enqueued to be executed *now*. self.advanceTime(to: self._now) } @@ -191,6 +244,7 @@ public final class EmbeddedEventLoop: EventLoop, CustomStringConvertible { /// Runs the event loop and moves "time" forward by the given amount, running any scheduled /// tasks that need to be run. public func advanceTime(by increment: TimeAmount) { + self.checkCorrectThread() self.advanceTime(to: self._now + increment) } @@ -199,6 +253,7 @@ public final class EmbeddedEventLoop: EventLoop, CustomStringConvertible { /// /// - Note: If `deadline` is before the current time, the current time will not be advanced. public func advanceTime(to deadline: NIODeadline) { + self.checkCorrectThread() let newTime = max(deadline, self._now) while let nextTask = self.scheduledTasks.peek() { @@ -228,6 +283,7 @@ public final class EmbeddedEventLoop: EventLoop, CustomStringConvertible { } internal func cancelRemainingScheduledTasks() { + self.checkCorrectThread() while let task = self.scheduledTasks.pop() { task.fail(EventLoopError.cancelled) } @@ -236,6 +292,7 @@ public final class EmbeddedEventLoop: EventLoop, CustomStringConvertible { #if canImport(Dispatch) /// - see: `EventLoop.shutdownGracefully` public func shutdownGracefully(queue: DispatchQueue, _ callback: @escaping (Error?) -> Void) { + self.checkCorrectThread() self.state = .closing run() cancelRemainingScheduledTasks() @@ -247,28 +304,33 @@ public final class EmbeddedEventLoop: EventLoop, CustomStringConvertible { #endif public func _preconditionSafeToWait(file: StaticString, line: UInt) { + self.checkCorrectThread() // EmbeddedEventLoop always allows a wait, as waiting will essentially always block // wait() return } public func _promiseCreated(futureIdentifier: _NIOEventLoopFutureIdentifier, file: StaticString, line: UInt) { + self.checkCorrectThread() precondition(_isDebugAssertConfiguration()) self._promiseCreationStore[futureIdentifier] = (file: file, line: line) } public func _promiseCompleted(futureIdentifier: _NIOEventLoopFutureIdentifier) -> (file: StaticString, line: UInt)? { + self.checkCorrectThread() precondition(_isDebugAssertConfiguration()) return self._promiseCreationStore.removeValue(forKey: futureIdentifier) } public func _preconditionSafeToSyncShutdown(file: StaticString, line: UInt) { + self.checkCorrectThread() // EmbeddedEventLoop always allows a sync shutdown. return } deinit { + self.checkCorrectThread() precondition(scheduledTasks.isEmpty, "Embedded event loop freed with unexecuted scheduled tasks!") } @@ -278,6 +340,18 @@ public final class EmbeddedEventLoop: EventLoop, CustomStringConvertible { "EmbeddedEventLoop is not thread safe and cannot be used as a SerialExecutor. Use NIOAsyncTestingEventLoop instead." ) } + + static let strictModeEnabled: Bool = { + for ciVar in ["SWIFTNIO_STRICT", "SWIFTNIO_CI", "SWIFTNIO_STRICT_EMBEDDED"] { + switch getenv(ciVar).map({ String.init(cString: $0).lowercased() }) { + case "true", "y", "yes", "on", "1": + return true + default: + () + } + } + return false + }() } @usableFromInline @@ -613,9 +687,11 @@ public final class EmbeddedChannel: Channel { /// - see: `ChannelOptions.Types.AllowRemoteHalfClosureOption` public var allowRemoteHalfClosure: Bool { get { - channelcore.allowRemoteHalfClosure + self.embeddedEventLoop.checkCorrectThread() + return channelcore.allowRemoteHalfClosure } set { + self.embeddedEventLoop.checkCorrectThread() channelcore.allowRemoteHalfClosure = newValue } } @@ -631,12 +707,14 @@ public final class EmbeddedChannel: Channel { /// - see: `Channel._channelCore` public var _channelCore: ChannelCore { - channelcore + self.embeddedEventLoop.checkCorrectThread() + return self.channelcore } /// - see: `Channel.pipeline` public var pipeline: ChannelPipeline { - _pipeline + self.embeddedEventLoop.checkCorrectThread() + return self._pipeline } /// - see: `Channel.isWritable` @@ -653,6 +731,7 @@ public final class EmbeddedChannel: Channel { /// writes) this will be `.clean`. If there are any unconsumed inbound, outbound, or pending outbound /// events, the `EmbeddedChannel` will returns those as `.leftOvers(inbound:outbound:pendingOutbound:)`. public func finish(acceptAlreadyClosed: Bool) throws -> LeftOverState { + self.embeddedEventLoop.checkCorrectThread() do { try close().wait() } catch let error as ChannelError { @@ -685,7 +764,8 @@ public final class EmbeddedChannel: Channel { /// writes) this will be `.clean`. If there are any unconsumed inbound, outbound, or pending outbound /// events, the `EmbeddedChannel` will returns those as `.leftOvers(inbound:outbound:pendingOutbound:)`. public func finish() throws -> LeftOverState { - try self.finish(acceptAlreadyClosed: false) + self.embeddedEventLoop.checkCorrectThread() + return try self.finish(acceptAlreadyClosed: false) } private var _pipeline: ChannelPipeline! @@ -695,7 +775,8 @@ public final class EmbeddedChannel: Channel { /// - see: `Channel.eventLoop` public var eventLoop: EventLoop { - self.embeddedEventLoop + self.embeddedEventLoop.checkCorrectThread() + return self.embeddedEventLoop } /// Returns the `EmbeddedEventLoop` that this `EmbeddedChannel` uses. This will return the same instance as @@ -705,9 +786,11 @@ public final class EmbeddedChannel: Channel { /// - see: `Channel.localAddress` public var localAddress: SocketAddress? { get { - self.channelcore.localAddress + self.embeddedEventLoop.checkCorrectThread() + return self.channelcore.localAddress } set { + self.embeddedEventLoop.checkCorrectThread() self.channelcore.localAddress = newValue } } @@ -715,9 +798,11 @@ public final class EmbeddedChannel: Channel { /// - see: `Channel.remoteAddress` public var remoteAddress: SocketAddress? { get { - self.channelcore.remoteAddress + self.embeddedEventLoop.checkCorrectThread() + return self.channelcore.remoteAddress } set { + self.embeddedEventLoop.checkCorrectThread() self.channelcore.remoteAddress = newValue } } @@ -739,7 +824,8 @@ public final class EmbeddedChannel: Channel { /// `ChannelHandler`. @inlinable public func readOutbound(as type: T.Type = T.self) throws -> T? { - try _readFromBuffer(buffer: &channelcore.outboundBuffer) + self.embeddedEventLoop.checkCorrectThread() + return try _readFromBuffer(buffer: &channelcore.outboundBuffer) } /// If available, this method reads one element of type `T` out of the `EmbeddedChannel`'s inbound buffer. If the @@ -754,7 +840,8 @@ public final class EmbeddedChannel: Channel { /// - note: `EmbeddedChannel.writeInbound` will fire data through the `ChannelPipeline` using `fireChannelRead`. @inlinable public func readInbound(as type: T.Type = T.self) throws -> T? { - try _readFromBuffer(buffer: &channelcore.inboundBuffer) + self.embeddedEventLoop.checkCorrectThread() + return try _readFromBuffer(buffer: &channelcore.inboundBuffer) } /// Sends an inbound `channelRead` event followed by a `channelReadComplete` event through the `ChannelPipeline`. @@ -768,9 +855,10 @@ public final class EmbeddedChannel: Channel { // all the way. @inlinable @discardableResult public func writeInbound(_ data: T) throws -> BufferState { - pipeline.fireChannelRead(NIOAny(data)) - pipeline.fireChannelReadComplete() - try throwIfErrorCaught() + self.embeddedEventLoop.checkCorrectThread() + self.pipeline.fireChannelRead(NIOAny(data)) + self.pipeline.fireChannelReadComplete() + try self.throwIfErrorCaught() return self.channelcore.inboundBuffer.isEmpty ? .empty : .full(Array(self.channelcore.inboundBuffer)) } @@ -786,7 +874,8 @@ public final class EmbeddedChannel: Channel { // all the way. @inlinable @discardableResult public func writeOutbound(_ data: T) throws -> BufferState { - try writeAndFlush(NIOAny(data)).wait() + self.embeddedEventLoop.checkCorrectThread() + try self.writeAndFlush(NIOAny(data)).wait() return self.channelcore.outboundBuffer.isEmpty ? .empty : .full(Array(self.channelcore.outboundBuffer)) } @@ -794,14 +883,16 @@ public final class EmbeddedChannel: Channel { /// /// The `EmbeddedChannel` will store an error some error travels the `ChannelPipeline` all the way past its end. public func throwIfErrorCaught() throws { + self.embeddedEventLoop.checkCorrectThread() if let error = channelcore.error { - channelcore.error = nil + self.channelcore.error = nil throw error } } @inlinable func _readFromBuffer(buffer: inout CircularBuffer) throws -> T? { + self.embeddedEventLoop.checkCorrectThread() if buffer.isEmpty { return nil } @@ -825,6 +916,7 @@ public final class EmbeddedChannel: Channel { public convenience init(handler: ChannelHandler? = nil, loop: EmbeddedEventLoop = EmbeddedEventLoop()) { let handlers = handler.map { [$0] } ?? [] self.init(handlers: handlers, loop: loop) + self.embeddedEventLoop.checkCorrectThread() } /// Create a new instance. @@ -842,17 +934,20 @@ public final class EmbeddedChannel: Channel { // This will never throw... try! register().wait() + self.embeddedEventLoop.checkCorrectThread() } /// - see: `Channel.setOption` @inlinable public func setOption(_ option: Option, value: Option.Value) -> EventLoopFuture { + self.embeddedEventLoop.checkCorrectThread() self.setOptionSync(option, value: value) return self.eventLoop.makeSucceededVoidFuture() } @inlinable internal func setOptionSync(_ option: Option, value: Option.Value) { + self.embeddedEventLoop.checkCorrectThread() if option is ChannelOptions.Types.AllowRemoteHalfClosureOption { self.allowRemoteHalfClosure = value as! Bool return @@ -864,11 +959,13 @@ public final class EmbeddedChannel: Channel { /// - see: `Channel.getOption` @inlinable public func getOption(_ option: Option) -> EventLoopFuture { - self.eventLoop.makeSucceededFuture(self.getOptionSync(option)) + self.embeddedEventLoop.checkCorrectThread() + return self.eventLoop.makeSucceededFuture(self.getOptionSync(option)) } @inlinable internal func getOptionSync(_ option: Option) -> Option.Value { + self.embeddedEventLoop.checkCorrectThread() if option is ChannelOptions.Types.AutoReadOption { return true as! Option.Value } @@ -894,10 +991,11 @@ public final class EmbeddedChannel: Channel { /// - address: The address to fake-bind to. /// - promise: The `EventLoopPromise` which will be fulfilled when the fake-bind operation has been done. public func bind(to address: SocketAddress, promise: EventLoopPromise?) { + self.embeddedEventLoop.checkCorrectThread() promise?.futureResult.whenSuccess { self.localAddress = address } - pipeline.bind(to: address, promise: promise) + self.pipeline.bind(to: address, promise: promise) } /// Fires the (outbound) `connect` event through the `ChannelPipeline`. If the event hits the `EmbeddedChannel` @@ -908,10 +1006,11 @@ public final class EmbeddedChannel: Channel { /// - address: The address to fake-bind to. /// - promise: The `EventLoopPromise` which will be fulfilled when the fake-bind operation has been done. public func connect(to address: SocketAddress, promise: EventLoopPromise?) { + self.embeddedEventLoop.checkCorrectThread() promise?.futureResult.whenSuccess { self.remoteAddress = address } - pipeline.connect(to: address, promise: promise) + self.pipeline.connect(to: address, promise: promise) } } diff --git a/Sources/NIOPosix/Thread.swift b/Sources/NIOPosix/Thread.swift index ca09003b14..1a5a9667ff 100644 --- a/Sources/NIOPosix/Thread.swift +++ b/Sources/NIOPosix/Thread.swift @@ -105,14 +105,14 @@ final class NIOThread { } /// Returns the current running `NIOThread`. - static var current: NIOThread { + public static var current: NIOThread { let handle = ThreadOpsSystem.currentThread return NIOThread(handle: handle, desiredName: nil) } } extension NIOThread: CustomStringConvertible { - var description: String { + public var description: String { let desiredName = self.desiredName let actualName = self.currentName @@ -232,7 +232,7 @@ public final class ThreadSpecificVariable { extension ThreadSpecificVariable: @unchecked Sendable where Value: Sendable {} extension NIOThread: Equatable { - static func == (lhs: NIOThread, rhs: NIOThread) -> Bool { + public static func == (lhs: NIOThread, rhs: NIOThread) -> Bool { lhs.withUnsafeThreadHandle { lhs in rhs.withUnsafeThreadHandle { rhs in ThreadOpsSystem.compareThreads(lhs, rhs) diff --git a/Tests/NIOCoreTests/DispatchQueue+WithFutureTest.swift b/Tests/NIOCoreTests/DispatchQueue+WithFutureTest.swift index ea2939b464..e70a30f6ca 100644 --- a/Tests/NIOCoreTests/DispatchQueue+WithFutureTest.swift +++ b/Tests/NIOCoreTests/DispatchQueue+WithFutureTest.swift @@ -15,6 +15,7 @@ import Dispatch import NIOCore import NIOEmbedded +import NIOPosix import XCTest enum DispatchQueueTestError: Error { @@ -23,7 +24,11 @@ enum DispatchQueueTestError: Error { class DispatchQueueWithFutureTest: XCTestCase { func testDispatchQueueAsyncWithFuture() { - let eventLoop = EmbeddedEventLoop() + let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) + defer { + XCTAssertNoThrow(try group.syncShutdownGracefully()) + } + let eventLoop = group.next() let sem = DispatchSemaphore(value: 0) var nonBlockingRan = false let futureResult: EventLoopFuture = DispatchQueue.global().asyncWithFuture(eventLoop: eventLoop) { @@ -46,7 +51,11 @@ class DispatchQueueWithFutureTest: XCTestCase { } func testDispatchQueueAsyncWithFutureThrows() { - let eventLoop = EmbeddedEventLoop() + let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) + defer { + XCTAssertNoThrow(try group.syncShutdownGracefully()) + } + let eventLoop = group.next() let sem = DispatchSemaphore(value: 0) var nonBlockingRan = false let futureResult: EventLoopFuture = DispatchQueue.global().asyncWithFuture(eventLoop: eventLoop) { diff --git a/Tests/NIOPosixTests/EventLoopFutureTest.swift b/Tests/NIOPosixTests/EventLoopFutureTest.swift index d0f7bb3335..8ef68fe637 100644 --- a/Tests/NIOPosixTests/EventLoopFutureTest.swift +++ b/Tests/NIOPosixTests/EventLoopFutureTest.swift @@ -12,6 +12,7 @@ // //===----------------------------------------------------------------------===// +import Atomics import Dispatch import NIOConcurrencyHelpers import NIOEmbedded @@ -1379,27 +1380,31 @@ class EventLoopFutureTest: XCTestCase { } func testFlatBlockingMapOnto() { - let eventLoop = EmbeddedEventLoop() + let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) + defer { + XCTAssertNoThrow(try group.syncShutdownGracefully()) + } + let eventLoop = group.next() let p = eventLoop.makePromise(of: String.self) let sem = DispatchSemaphore(value: 0) - var blockingRan = false - var nonBlockingRan = false + let blockingRan = ManagedAtomic(false) + let nonBlockingRan = ManagedAtomic(false) p.futureResult.map { $0.count }.flatMapBlocking(onto: DispatchQueue.global()) { value -> Int in sem.wait() // Block in chained EventLoopFuture - blockingRan = true + blockingRan.store(true, ordering: .sequentiallyConsistent) return 1 + value }.whenSuccess { XCTAssertEqual($0, 6) - XCTAssertTrue(blockingRan) - XCTAssertTrue(nonBlockingRan) + XCTAssertTrue(blockingRan.load(ordering: .sequentiallyConsistent)) + XCTAssertTrue(nonBlockingRan.load(ordering: .sequentiallyConsistent)) } p.succeed("hello") let p2 = eventLoop.makePromise(of: Bool.self) p2.futureResult.whenSuccess { _ in - nonBlockingRan = true + nonBlockingRan.store(true, ordering: .sequentiallyConsistent) } p2.succeed(true) diff --git a/Tests/NIOPosixTests/NIOScheduledCallbackTests.swift b/Tests/NIOPosixTests/NIOScheduledCallbackTests.swift index 7038b7129d..b115ac2f8f 100644 --- a/Tests/NIOPosixTests/NIOScheduledCallbackTests.swift +++ b/Tests/NIOPosixTests/NIOScheduledCallbackTests.swift @@ -54,29 +54,6 @@ final class MTELGScheduledCallbackTests: _BaseScheduledCallbackTests { } } -final class EmbeddedScheduledCallbackTests: _BaseScheduledCallbackTests { - struct Requirements: ScheduledCallbackTestRequirements { - let _loop = EmbeddedEventLoop() - var loop: (any EventLoop) { self._loop } - - func advanceTime(by amount: TimeAmount) async throws { - self._loop.advanceTime(by: amount) - } - - func shutdownEventLoop() async throws { - try await self._loop.shutdownGracefully() - } - - func maybeInContext(_ body: @escaping @Sendable () throws -> R) async throws -> R { - try body() - } - } - - override func setUp() async throws { - self.requirements = Requirements() - } -} - final class NIOAsyncTestingEventLoopScheduledCallbackTests: _BaseScheduledCallbackTests { struct Requirements: ScheduledCallbackTestRequirements { let _loop = NIOAsyncTestingEventLoop() From 02906a6c084b280f3e3e3dbffa5be62537c7943c Mon Sep 17 00:00:00 2001 From: Franz Busch Date: Mon, 28 Oct 2024 16:14:37 +0100 Subject: [PATCH 19/20] Fix `NIOAsyncSequenceProducer` watermark strategy. (#2952) # Motivation It was currently possible that the producer's delegate is getting called twice with `produceMore` even if no `yield` returned a `stopProducing`. This could happen when we expected the producer to yield elements but the consumer went below the low watermark again. Resulting in two subsequent calls. # Modification This PR stores the current demand state in the strategy which let's us avoid flipping the `hasOustandingDemand` state of the sequence. # Result Correctly, ensured the call order of `produceMore` and `stopProducing`. --- .../NIOAsyncSequenceProducerStrategies.swift | 22 ++++++++++-- ...owWatermarkBackPressureStrategyTests.swift | 4 +-- .../NIOAsyncSequenceTests.swift | 36 +++++++++++++++++++ 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/Sources/NIOCore/AsyncSequences/NIOAsyncSequenceProducerStrategies.swift b/Sources/NIOCore/AsyncSequences/NIOAsyncSequenceProducerStrategies.swift index 5fe444b9bd..6de186197c 100644 --- a/Sources/NIOCore/AsyncSequences/NIOAsyncSequenceProducerStrategies.swift +++ b/Sources/NIOCore/AsyncSequences/NIOAsyncSequenceProducerStrategies.swift @@ -22,6 +22,7 @@ public enum NIOAsyncSequenceProducerBackPressureStrategies { public struct HighLowWatermark: NIOAsyncSequenceProducerBackPressureStrategy { private let lowWatermark: Int private let highWatermark: Int + private var hasOustandingDemand: Bool = true /// Initializes a new ``NIOAsyncSequenceProducerBackPressureStrategies/HighLowWatermark``. /// @@ -36,12 +37,29 @@ public enum NIOAsyncSequenceProducerBackPressureStrategies { public mutating func didYield(bufferDepth: Int) -> Bool { // We are demanding more until we reach the high watermark - bufferDepth < self.highWatermark + if bufferDepth < self.highWatermark { + precondition(self.hasOustandingDemand) + return true + } else { + self.hasOustandingDemand = false + return false + } } public mutating func didConsume(bufferDepth: Int) -> Bool { // We start demanding again once we are below the low watermark - bufferDepth < self.lowWatermark + if bufferDepth < self.lowWatermark { + if self.hasOustandingDemand { + // We are below and have outstanding demand + return true + } else { + // We are below but don't have outstanding demand but need more + self.hasOustandingDemand = true + return true + } + } else { + return self.hasOustandingDemand + } } } } diff --git a/Tests/NIOCoreTests/AsyncSequences/NIOAsyncSequenceProducer+HighLowWatermarkBackPressureStrategyTests.swift b/Tests/NIOCoreTests/AsyncSequences/NIOAsyncSequenceProducer+HighLowWatermarkBackPressureStrategyTests.swift index 3506dc9ce8..f7a6790ecf 100644 --- a/Tests/NIOCoreTests/AsyncSequences/NIOAsyncSequenceProducer+HighLowWatermarkBackPressureStrategyTests.swift +++ b/Tests/NIOCoreTests/AsyncSequences/NIOAsyncSequenceProducer+HighLowWatermarkBackPressureStrategyTests.swift @@ -51,10 +51,10 @@ final class NIOAsyncSequenceProducerBackPressureStrategiesHighLowWatermarkTests: } func testDidConsume_whenAboveLowWatermark() { - XCTAssertFalse(self.strategy.didConsume(bufferDepth: 6)) + XCTAssertTrue(self.strategy.didConsume(bufferDepth: 6)) } func testDidConsume_whenAtLowWatermark() { - XCTAssertFalse(self.strategy.didConsume(bufferDepth: 5)) + XCTAssertTrue(self.strategy.didConsume(bufferDepth: 5)) } } diff --git a/Tests/NIOCoreTests/AsyncSequences/NIOAsyncSequenceTests.swift b/Tests/NIOCoreTests/AsyncSequences/NIOAsyncSequenceTests.swift index 3720976224..7a7583f59e 100644 --- a/Tests/NIOCoreTests/AsyncSequences/NIOAsyncSequenceTests.swift +++ b/Tests/NIOCoreTests/AsyncSequences/NIOAsyncSequenceTests.swift @@ -149,6 +149,42 @@ final class NIOAsyncSequenceProducerTests: XCTestCase { XCTAssertEqual(self.source.yield(contentsOf: [7, 8, 9, 10, 11]), .stopProducing) } + func testWatermarkBackpressure_whenBelowLowwatermark_andOutstandingDemand() async { + let newSequence = NIOAsyncSequenceProducer.makeSequence( + elementType: Int.self, + backPressureStrategy: NIOAsyncSequenceProducerBackPressureStrategies.HighLowWatermark( + lowWatermark: 2, + highWatermark: 5 + ), + finishOnDeinit: false, + delegate: self.delegate + ) + let iterator = newSequence.sequence.makeAsyncIterator() + var eventsIterator = self.delegate.events.makeAsyncIterator() + let source = newSequence.source + + XCTAssertEqual(source.yield(1), .produceMore) + XCTAssertEqual(source.yield(2), .produceMore) + XCTAssertEqual(source.yield(3), .produceMore) + XCTAssertEqual(source.yield(4), .produceMore) + XCTAssertEqual(source.yield(5), .stopProducing) + XCTAssertEqualWithoutAutoclosure(await iterator.next(), 1) + XCTAssertEqualWithoutAutoclosure(await iterator.next(), 2) + XCTAssertEqualWithoutAutoclosure(await iterator.next(), 3) + XCTAssertEqualWithoutAutoclosure(await iterator.next(), 4) + XCTAssertEqualWithoutAutoclosure(await iterator.next(), 5) + XCTAssertEqualWithoutAutoclosure(await eventsIterator.next(), .produceMore) + XCTAssertEqual(source.yield(6), .produceMore) + XCTAssertEqual(source.yield(7), .produceMore) + XCTAssertEqual(source.yield(8), .produceMore) + XCTAssertEqualWithoutAutoclosure(await iterator.next(), 6) + XCTAssertEqualWithoutAutoclosure(await iterator.next(), 7) + XCTAssertEqualWithoutAutoclosure(await iterator.next(), 8) + source.finish() + XCTAssertEqualWithoutAutoclosure(await iterator.next(), nil) + XCTAssertEqualWithoutAutoclosure(await eventsIterator.next(), .didTerminate) + } + // MARK: - Yield func testYield_whenInitial_andStopDemanding() async { From 411c2c553c177801aecb4f1a393e076aea62e3a1 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 29 Oct 2024 10:40:47 +0000 Subject: [PATCH 20/20] Handle Sendability of RemovableChannelHandler (#2953) Motivation: RemovableChannelHandlers have a large API surface in NIOCore. That API surface is a bit awkward with regard to strict concurrency, and needs some cleanup. Modifications: This patch adds some new API that is necessary to safely work with RemovableChannelHandlers, deprecates some API that cannot plausibly be used, and cleans up some other parts of the API. Result: Easier to work with RemovableChannelHandlers --- Sources/NIOCore/AsyncAwaitSupport.swift | 5 + Sources/NIOCore/ChannelPipeline.swift | 161 ++++++++++++++---- .../NIOHTTP1/HTTPServerUpgradeHandler.swift | 4 +- .../NIOHTTPClientUpgradeHandler.swift | 4 +- ...pplicationProtocolNegotiationHandler.swift | 10 +- ...pplicationProtocolNegotiationHandler.swift | 6 +- Sources/NIOTLS/SNIHandler.swift | 6 +- .../AsyncChannel/AsyncChannelTests.swift | 2 +- .../HTTPServerUpgradeTests.swift | 2 +- .../AcceptBackoffHandlerTest.swift | 4 +- Tests/NIOPosixTests/ChannelPipelineTest.swift | 10 +- Tests/NIOPosixTests/CodecTest.swift | 10 +- Tests/NIOPosixTests/MulticastTest.swift | 2 +- 13 files changed, 167 insertions(+), 59 deletions(-) diff --git a/Sources/NIOCore/AsyncAwaitSupport.swift b/Sources/NIOCore/AsyncAwaitSupport.swift index e8018d6e08..7a8b0448b7 100644 --- a/Sources/NIOCore/AsyncAwaitSupport.swift +++ b/Sources/NIOCore/AsyncAwaitSupport.swift @@ -188,6 +188,11 @@ extension ChannelPipeline { } @available(macOS 10.15, iOS 13, tvOS 13, watchOS 6, *) + @available( + *, + deprecated, + message: "Use .syncOperations.removeHandler(context:) instead, this method is not Sendable-safe." + ) public func removeHandler(context: ChannelHandlerContext) async throws { try await self.removeHandler(context: context).get() } diff --git a/Sources/NIOCore/ChannelPipeline.swift b/Sources/NIOCore/ChannelPipeline.swift index b2c90c142d..ae6b1f9aff 100644 --- a/Sources/NIOCore/ChannelPipeline.swift +++ b/Sources/NIOCore/ChannelPipeline.swift @@ -167,8 +167,9 @@ public final class ChannelPipeline: ChannelInvoker { /// - handler: the `ChannelHandler` to add /// - position: The position in the `ChannelPipeline` to add `handler`. Defaults to `.last`. /// - returns: the `EventLoopFuture` which will be notified once the `ChannelHandler` was added. + @preconcurrency public func addHandler( - _ handler: ChannelHandler, + _ handler: ChannelHandler & Sendable, name: String? = nil, position: ChannelPipeline.Position = .last ) -> EventLoopFuture { @@ -349,7 +350,8 @@ public final class ChannelPipeline: ChannelInvoker { /// - parameters: /// - handler: the `ChannelHandler` to remove. /// - returns: the `EventLoopFuture` which will be notified once the `ChannelHandler` was removed. - public func removeHandler(_ handler: RemovableChannelHandler) -> EventLoopFuture { + @preconcurrency + public func removeHandler(_ handler: RemovableChannelHandler & Sendable) -> EventLoopFuture { let promise = self.eventLoop.makePromise(of: Void.self) self.removeHandler(handler, promise: promise) return promise.futureResult @@ -371,6 +373,11 @@ public final class ChannelPipeline: ChannelInvoker { /// - parameters: /// - context: the `ChannelHandlerContext` that belongs to `ChannelHandler` that should be removed. /// - returns: the `EventLoopFuture` which will be notified once the `ChannelHandler` was removed. + @available( + *, + deprecated, + message: "Use .syncOperations.removeHandler(context:) instead, this method is not Sendable-safe." + ) public func removeHandler(context: ChannelHandlerContext) -> EventLoopFuture { let promise = self.eventLoop.makePromise(of: Void.self) self.removeHandler(context: context, promise: promise) @@ -382,14 +389,11 @@ public final class ChannelPipeline: ChannelInvoker { /// - parameters: /// - handler: the `ChannelHandler` to remove. /// - promise: An `EventLoopPromise` that will complete when the `ChannelHandler` is removed. - public func removeHandler(_ handler: RemovableChannelHandler, promise: EventLoopPromise?) { + @preconcurrency + public func removeHandler(_ handler: RemovableChannelHandler & Sendable, promise: EventLoopPromise?) { + @Sendable func removeHandler0() { - switch self.contextSync(handler: handler) { - case .success(let context): - self.removeHandler(context: context, promise: promise) - case .failure(let error): - promise?.fail(error) - } + self.syncOperations.removeHandler(handler, promise: promise) } if self.eventLoop.inEventLoop { @@ -407,13 +411,9 @@ public final class ChannelPipeline: ChannelInvoker { /// - name: the name that was used to add the `ChannelHandler` to the `ChannelPipeline` before. /// - promise: An `EventLoopPromise` that will complete when the `ChannelHandler` is removed. public func removeHandler(name: String, promise: EventLoopPromise?) { + @Sendable func removeHandler0() { - switch self.contextSync(name: name) { - case .success(let context): - self.removeHandler(context: context, promise: promise) - case .failure(let error): - promise?.fail(error) - } + self.syncOperations.removeHandler(name: name, promise: promise) } if self.eventLoop.inEventLoop { @@ -430,13 +430,22 @@ public final class ChannelPipeline: ChannelInvoker { /// - parameters: /// - context: the `ChannelHandlerContext` that belongs to `ChannelHandler` that should be removed. /// - promise: An `EventLoopPromise` that will complete when the `ChannelHandler` is removed. + @available( + *, + deprecated, + message: "Use .syncOperations.removeHandler(context:) instead, this method is not Sendable-safe." + ) public func removeHandler(context: ChannelHandlerContext, promise: EventLoopPromise?) { - guard context.handler is RemovableChannelHandler else { + let sendableView = context.sendableView + + guard sendableView.channelHandlerIsRemovable else { promise?.fail(ChannelError._unremovableHandler) return } + + @Sendable func removeHandler0() { - context.startUserTriggeredRemoval(promise: promise) + sendableView.wrappedValue.startUserTriggeredRemoval(promise: promise) } if self.eventLoop.inEventLoop { @@ -453,7 +462,13 @@ public final class ChannelPipeline: ChannelInvoker { /// - parameters: /// - handler: the `ChannelHandler` for which the `ChannelHandlerContext` should be returned /// - returns: the `EventLoopFuture` which will be notified once the the operation completes. - public func context(handler: ChannelHandler) -> EventLoopFuture { + @available( + *, + deprecated, + message: "This method is not strict concurrency safe. Prefer .syncOperations.context(handler:)" + ) + @preconcurrency + public func context(handler: ChannelHandler & Sendable) -> EventLoopFuture { let promise = self.eventLoop.makePromise(of: ChannelHandlerContext.self) if self.eventLoop.inEventLoop { @@ -1005,8 +1020,9 @@ extension ChannelPipeline { /// - position: The position in the `ChannelPipeline` to add `handlers`. Defaults to `.last`. /// /// - returns: A future that will be completed when all of the supplied `ChannelHandler`s were added. + @preconcurrency public func addHandlers( - _ handlers: [ChannelHandler], + _ handlers: [ChannelHandler & Sendable], position: ChannelPipeline.Position = .last ) -> EventLoopFuture { let future: EventLoopFuture @@ -1030,8 +1046,9 @@ extension ChannelPipeline { /// - position: The position in the `ChannelPipeline` to add `handlers`. Defaults to `.last`. /// /// - returns: A future that will be completed when all of the supplied `ChannelHandler`s were added. + @preconcurrency public func addHandlers( - _ handlers: ChannelHandler..., + _ handlers: (ChannelHandler & Sendable)..., position: ChannelPipeline.Position = .last ) -> EventLoopFuture { self.addHandlers(handlers, position: position) @@ -1149,18 +1166,51 @@ extension ChannelPipeline { /// - parameters: /// - handler: the `ChannelHandler` to remove. /// - returns: the `EventLoopFuture` which will be notified once the `ChannelHandler` was removed. - @preconcurrency public func removeHandler(_ handler: RemovableChannelHandler) -> EventLoopFuture { let promise = self.eventLoop.makePromise(of: Void.self) + self.removeHandler(handler, promise: promise) + return promise.futureResult + } + + /// Remove a ``ChannelHandler`` from the ``ChannelPipeline``. + /// + /// - parameters: + /// - handler: the ``ChannelHandler`` to remove. + /// - promise: an ``EventLoopPromise`` to notify when the ``ChannelHandler`` was removed. + public func removeHandler(_ handler: RemovableChannelHandler, promise: EventLoopPromise?) { switch self._pipeline.contextSync(handler: handler) { case .success(let context): - self._pipeline.removeHandler(context: context, promise: promise) + self.removeHandler(context: context, promise: promise) case .failure(let error): - promise.fail(error) + promise?.fail(error) } + } + + /// Remove a `ChannelHandler` from the `ChannelPipeline`. + /// + /// - parameters: + /// - name: the name that was used to add the `ChannelHandler` to the `ChannelPipeline` before. + /// - returns: the `EventLoopFuture` which will be notified once the `ChannelHandler` was removed. + public func removeHandler(name: String) -> EventLoopFuture { + let promise = self.eventLoop.makePromise(of: Void.self) + self.removeHandler(name: name, promise: promise) return promise.futureResult } + /// Remove a ``ChannelHandler`` from the ``ChannelPipeline``. + /// + /// - parameters: + /// - name: the name that was used to add the `ChannelHandler` to the `ChannelPipeline` before. + /// - promise: an ``EventLoopPromise`` to notify when the ``ChannelHandler`` was removed. + public func removeHandler(name: String, promise: EventLoopPromise?) { + switch self._pipeline.contextSync(name: name) { + case .success(let context): + self.removeHandler(context: context, promise: promise) + case .failure(let error): + promise?.fail(error) + } + } + /// Remove a `ChannelHandler` from the `ChannelPipeline`. /// /// - parameters: @@ -1168,10 +1218,23 @@ extension ChannelPipeline { /// - returns: the `EventLoopFuture` which will be notified once the `ChannelHandler` was removed. public func removeHandler(context: ChannelHandlerContext) -> EventLoopFuture { let promise = self.eventLoop.makePromise(of: Void.self) - self._pipeline.removeHandler(context: context, promise: promise) + self.removeHandler(context: context, promise: promise) return promise.futureResult } + /// Remove a `ChannelHandler` from the `ChannelPipeline`. + /// + /// - parameters: + /// - context: the `ChannelHandlerContext` that belongs to `ChannelHandler` that should be removed. + /// - promise: an ``EventLoopPromise`` to notify when the ``ChannelHandler`` was removed. + public func removeHandler(context: ChannelHandlerContext, promise: EventLoopPromise?) { + if context.handler is RemovableChannelHandler { + context.startUserTriggeredRemoval(promise: promise) + } else { + promise?.fail(ChannelError.unremovableHandler) + } + } + /// Returns the `ChannelHandlerContext` for the given handler instance if it is in /// the `ChannelPipeline`, if it exists. /// @@ -1367,7 +1430,8 @@ extension ChannelPipeline.SynchronousOperations: Sendable {} extension ChannelPipeline { /// A `Position` within the `ChannelPipeline` used to insert handlers into the `ChannelPipeline`. - public enum Position { + @preconcurrency + public enum Position: Sendable { /// The first `ChannelHandler` -- the front of the `ChannelPipeline`. case first @@ -1375,18 +1439,15 @@ extension ChannelPipeline { case last /// Before the given `ChannelHandler`. - case before(ChannelHandler) + case before(ChannelHandler & Sendable) /// After the given `ChannelHandler`. - case after(ChannelHandler) + case after(ChannelHandler & Sendable) } } -@available(*, unavailable) -extension ChannelPipeline.Position: Sendable {} - /// Special `ChannelHandler` that forwards all events to the `Channel.Unsafe` implementation. -final class HeadChannelHandler: _ChannelOutboundHandler { +final class HeadChannelHandler: _ChannelOutboundHandler, Sendable { static let name = "head" static let sharedInstance = HeadChannelHandler() @@ -1442,7 +1503,7 @@ extension CloseMode { } /// Special `ChannelInboundHandler` which will consume all inbound events. -final class TailChannelHandler: _ChannelInboundHandler { +final class TailChannelHandler: _ChannelInboundHandler, Sendable { static let name = "tail" static let sharedInstance = TailChannelHandler() @@ -1977,6 +2038,42 @@ extension ChannelHandlerContext { } } +extension ChannelHandlerContext { + var sendableView: SendableView { + SendableView(wrapping: self) + } + + /// A wrapper over ``ChannelHandlerContext`` that allows access to the thread-safe API + /// surface on the type. + /// + /// Very little of ``ChannelHandlerContext`` is thread-safe, but in a rare few places + /// there are things we can access. This type makes those available. + struct SendableView: @unchecked Sendable { + private let context: ChannelHandlerContext + + fileprivate init(wrapping context: ChannelHandlerContext) { + self.context = context + } + + /// Whether the ``ChannelHandler`` associated with this context conforms to + /// ``RemovableChannelHandler``. + var channelHandlerIsRemovable: Bool { + // `context.handler` is not mutable, and set at construction, so this access is + // acceptable. The protocol conformance check is also safe. + self.context.handler is RemovableChannelHandler + } + + /// Grabs the underlying ``ChannelHandlerContext``. May only be called on the + /// event loop. + var wrappedValue: ChannelHandlerContext { + // The event loop lookup here is also thread-safe, so we can grab the value out + // and use it. + self.context.eventLoop.preconditionInEventLoop() + return self.context + } + } +} + extension ChannelPipeline: CustomDebugStringConvertible { public var debugDescription: String { // This method forms output in the following format: diff --git a/Sources/NIOHTTP1/HTTPServerUpgradeHandler.swift b/Sources/NIOHTTP1/HTTPServerUpgradeHandler.swift index 0c691cc5df..1a92370c3d 100644 --- a/Sources/NIOHTTP1/HTTPServerUpgradeHandler.swift +++ b/Sources/NIOHTTP1/HTTPServerUpgradeHandler.swift @@ -290,7 +290,7 @@ public final class HTTPServerUpgradeHandler: ChannelInboundHandler, RemovableCha ) self.upgradeState = .upgradeComplete // When we remove ourselves we'll be delivering any buffered data. - context.pipeline.removeHandler(context: context, promise: nil) + context.pipeline.syncOperations.removeHandler(context: context, promise: nil) case .failure(let error): // Remain in the '.upgrading' state. @@ -357,7 +357,7 @@ public final class HTTPServerUpgradeHandler: ChannelInboundHandler, RemovableCha context.fireChannelReadComplete() // Ok, we've delivered all the parts. We can now remove ourselves, which should happen synchronously. - context.pipeline.removeHandler(context: context, promise: nil) + context.pipeline.syncOperations.removeHandler(context: context, promise: nil) } /// Builds the initial mandatory HTTP headers for HTTP upgrade responses. diff --git a/Sources/NIOHTTP1/NIOHTTPClientUpgradeHandler.swift b/Sources/NIOHTTP1/NIOHTTPClientUpgradeHandler.swift index b092512f16..dbc403f69f 100644 --- a/Sources/NIOHTTP1/NIOHTTPClientUpgradeHandler.swift +++ b/Sources/NIOHTTP1/NIOHTTPClientUpgradeHandler.swift @@ -356,7 +356,7 @@ public final class NIOHTTPClientUpgradeHandler: ChannelDuplexHandler, RemovableC self.upgradeState = .upgradeComplete } .whenComplete { _ in - context.pipeline.removeHandler(context: context, promise: nil) + context.pipeline.syncOperations.removeHandler(context: context, promise: nil) } } } @@ -397,7 +397,7 @@ public final class NIOHTTPClientUpgradeHandler: ChannelDuplexHandler, RemovableC context.fireChannelRead(Self.wrapInboundOut(data)) // We've delivered the data. We can now remove ourselves, which should happen synchronously. - context.pipeline.removeHandler(context: context, promise: nil) + context.pipeline.syncOperations.removeHandler(context: context, promise: nil) } } diff --git a/Sources/NIOTLS/ApplicationProtocolNegotiationHandler.swift b/Sources/NIOTLS/ApplicationProtocolNegotiationHandler.swift index 40c799ddfa..3d6859dec9 100644 --- a/Sources/NIOTLS/ApplicationProtocolNegotiationHandler.swift +++ b/Sources/NIOTLS/ApplicationProtocolNegotiationHandler.swift @@ -128,10 +128,12 @@ public final class ApplicationProtocolNegotiationHandler: ChannelInboundHandler, } private func userFutureCompleted(context: ChannelHandlerContext, result: Result) { + context.eventLoop.assertInEventLoop() + switch self.stateMachine.userFutureCompleted(with: result) { case .fireErrorCaughtAndRemoveHandler(let error): context.fireErrorCaught(error) - context.pipeline.removeHandler(self, promise: nil) + context.pipeline.syncOperations.removeHandler(self, promise: nil) case .fireErrorCaughtAndStartUnbuffering(let error): context.fireErrorCaught(error) @@ -141,7 +143,7 @@ public final class ApplicationProtocolNegotiationHandler: ChannelInboundHandler, self.unbuffer(context: context) case .removeHandler: - context.pipeline.removeHandler(self, promise: nil) + context.pipeline.syncOperations.removeHandler(self, promise: nil) case .none: break @@ -149,6 +151,8 @@ public final class ApplicationProtocolNegotiationHandler: ChannelInboundHandler, } private func unbuffer(context: ChannelHandlerContext) { + context.eventLoop.assertInEventLoop() + while true { switch self.stateMachine.unbuffer() { case .fireChannelRead(let data): @@ -156,7 +160,7 @@ public final class ApplicationProtocolNegotiationHandler: ChannelInboundHandler, case .fireChannelReadCompleteAndRemoveHandler: context.fireChannelReadComplete() - context.pipeline.removeHandler(self, promise: nil) + context.pipeline.syncOperations.removeHandler(self, promise: nil) return } } diff --git a/Sources/NIOTLS/NIOTypedApplicationProtocolNegotiationHandler.swift b/Sources/NIOTLS/NIOTypedApplicationProtocolNegotiationHandler.swift index c4a2984143..4749be12ec 100644 --- a/Sources/NIOTLS/NIOTypedApplicationProtocolNegotiationHandler.swift +++ b/Sources/NIOTLS/NIOTypedApplicationProtocolNegotiationHandler.swift @@ -138,7 +138,7 @@ public final class NIOTypedApplicationProtocolNegotiationHandler.self).flatMap { context in - channel.pipeline.removeHandler(context: context) + channel.pipeline.syncOperations.removeHandler(context: context) }.whenFailure { error in XCTFail("unexpected error: \(error)") } @@ -834,7 +834,7 @@ public final class ByteToMessageDecoderTest: XCTestCase { mutating func decode(context: ChannelHandlerContext, buffer: inout ByteBuffer) throws -> DecodingState { if let slice = buffer.readSlice(length: 16) { context.fireChannelRead(Self.wrapInboundOut(slice)) - context.pipeline.removeHandler(context: context).whenFailure { error in + context.pipeline.syncOperations.removeHandler(context: context).whenFailure { error in XCTFail("unexpected error: \(error)") } return .continue @@ -943,7 +943,7 @@ public final class ByteToMessageDecoderTest: XCTestCase { ) ) ) - context.pipeline.removeHandler(context: context).whenFailure { error in + context.pipeline.syncOperations.removeHandler(context: context).whenFailure { error in XCTFail("unexpected error: \(error)") } return .continue @@ -1101,7 +1101,7 @@ public final class ByteToMessageDecoderTest: XCTestCase { buffer.writeString("x") XCTAssertNoThrow(try channel.writeInbound(buffer)) let removalFuture = channel.pipeline.context(handlerType: ByteToMessageHandler.self).flatMap { - channel.pipeline.removeHandler(context: $0) + channel.pipeline.syncOperations.removeHandler(context: $0) } channel.embeddedEventLoop.run() XCTAssertNoThrow(try removalFuture.wait()) @@ -1135,7 +1135,7 @@ public final class ByteToMessageDecoderTest: XCTestCase { let channel = EmbeddedChannel(handler: ByteToMessageHandler(decoder)) XCTAssertNoThrow(try channel.connect(to: SocketAddress(ipAddress: "1.2.3.4", port: 5678)).wait()) let removalFuture = channel.pipeline.context(handlerType: ByteToMessageHandler.self).flatMap { - channel.pipeline.removeHandler(context: $0) + channel.pipeline.syncOperations.removeHandler(context: $0) } channel.embeddedEventLoop.run() XCTAssertNoThrow(try removalFuture.wait()) diff --git a/Tests/NIOPosixTests/MulticastTest.swift b/Tests/NIOPosixTests/MulticastTest.swift index 22a3e4d0a8..c9ae19d291 100644 --- a/Tests/NIOPosixTests/MulticastTest.swift +++ b/Tests/NIOPosixTests/MulticastTest.swift @@ -27,7 +27,7 @@ final class PromiseOnReadHandler: ChannelInboundHandler { func channelRead(context: ChannelHandlerContext, data: NIOAny) { self.promise.succeed(Self.unwrapInboundIn(data)) - _ = context.pipeline.removeHandler(context: context) + context.pipeline.syncOperations.removeHandler(context: context, promise: nil) } }