diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..aaac325 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,8 @@ +root = true + +[*] +indent_style = tab +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true diff --git a/Package.resolved b/Package.resolved index ebdcc9d..7e9a3d0 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,12 +1,13 @@ { + "originHash" : "a818c4732d60e59901e640cf8c5de3900e8ec0ffaa1daa0a631b8676b0b17c4b", "pins" : [ { "identity" : "rearrange", "kind" : "remoteSourceControl", "location" : "https://github.com/ChimeHQ/Rearrange", "state" : { - "revision" : "5ff7f3363f7a08f77e0d761e38e6add31c2136e1", - "version" : "1.8.1" + "revision" : "f1d74e1642956f0300756ad8d1d64e9034857bc3", + "version" : "2.0.0" } }, { @@ -14,8 +15,7 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/ChimeHQ/SwiftTreeSitter", "state" : { - "revision" : "36aa61d1b531f744f35229f010efba9c6d6cbbdd", - "version" : "0.9.0" + "revision" : "55f2c2fdaf859f86e4ea8513b9934badc7894019" } }, { @@ -28,5 +28,5 @@ } } ], - "version" : 2 + "version" : 3 } diff --git a/Package.swift b/Package.swift index 8cc2a91..5512d3f 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 5.8 +// swift-tools-version: 6.0 import PackageDescription @@ -15,8 +15,8 @@ let package = Package( .library(name: "Neon", targets: ["Neon"]), ], dependencies: [ - .package(url: "https://github.com/ChimeHQ/SwiftTreeSitter", from: "0.9.0"), - .package(url: "https://github.com/ChimeHQ/Rearrange", from: "1.8.1"), + .package(url: "https://github.com/ChimeHQ/SwiftTreeSitter", revision: "55f2c2fdaf859f86e4ea8513b9934badc7894019"), + .package(url: "https://github.com/ChimeHQ/Rearrange", from: "2.0.0"), ], targets: [ .target(name: "RangeState", dependencies: ["Rearrange"]), @@ -24,11 +24,11 @@ let package = Package( .target( name: "Neon", dependencies: [ - "RangeState", - "Rearrange", - "TreeSitterClient", - .product(name: "SwiftTreeSitterLayer", package: "SwiftTreeSitter"), - ] + "RangeState", + "Rearrange", + "TreeSitterClient", + .product(name: "SwiftTreeSitterLayer", package: "SwiftTreeSitter"), + ] ), .target( name: "TreeSitterClient", @@ -36,7 +36,7 @@ let package = Package( "RangeState", "Rearrange", "SwiftTreeSitter", - .product(name: "SwiftTreeSitterLayer", package: "SwiftTreeSitter"), + .product(name: "SwiftTreeSitterLayer", package: "SwiftTreeSitter"), ] ), .target( @@ -50,13 +50,3 @@ let package = Package( .testTarget(name: "TreeSitterClientTests", dependencies: ["TreeSitterClient", "NeonTestsTreeSitterSwift"]) ] ) - -let swiftSettings: [SwiftSetting] = [ - .enableExperimentalFeature("StrictConcurrency") -] - -for target in package.targets { - var settings = target.swiftSettings ?? [] - settings.append(contentsOf: swiftSettings) - target.swiftSettings = settings -} diff --git a/Projects/NeonExample.xcodeproj/project.pbxproj b/Projects/NeonExample.xcodeproj/project.pbxproj index 5de9fe9..e8a1a81 100644 --- a/Projects/NeonExample.xcodeproj/project.pbxproj +++ b/Projects/NeonExample.xcodeproj/project.pbxproj @@ -259,6 +259,10 @@ "INFOPLIST_KEY_UILaunchScreen_Generation[sdk=iphonesimulator*]" = YES; "INFOPLIST_KEY_UIStatusBarStyle[sdk=iphoneos*]" = UIStatusBarStyleDefault; "INFOPLIST_KEY_UIStatusBarStyle[sdk=iphonesimulator*]" = UIStatusBarStyleDefault; + IPHONEOS_DEPLOYMENT_TARGET = 16.0; + MACOSX_DEPLOYMENT_TARGET = 13.0; + TVOS_DEPLOYMENT_TARGET = 16.0; + WATCHOS_DEPLOYMENT_TARGET = 9.0; }; name = Debug; }; @@ -275,6 +279,10 @@ "INFOPLIST_KEY_UILaunchScreen_Generation[sdk=iphonesimulator*]" = YES; "INFOPLIST_KEY_UIStatusBarStyle[sdk=iphoneos*]" = UIStatusBarStyleDefault; "INFOPLIST_KEY_UIStatusBarStyle[sdk=iphonesimulator*]" = UIStatusBarStyleDefault; + IPHONEOS_DEPLOYMENT_TARGET = 16.0; + MACOSX_DEPLOYMENT_TARGET = 13.0; + TVOS_DEPLOYMENT_TARGET = 16.0; + WATCHOS_DEPLOYMENT_TARGET = 9.0; }; name = Release; }; diff --git a/Projects/NeonExample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Projects/NeonExample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 34ae22d..4c7ecde 100644 --- a/Projects/NeonExample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/Projects/NeonExample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -1,4 +1,5 @@ { + "originHash" : "88a29eea1c1b10215ec2071d3cc04cae5d3b2418319c7e017e284ba383823031", "pins" : [ { "identity" : "nsui", @@ -13,8 +14,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/ChimeHQ/Rearrange", "state" : { - "revision" : "5ff7f3363f7a08f77e0d761e38e6add31c2136e1", - "version" : "1.8.1" + "revision" : "f1d74e1642956f0300756ad8d1d64e9034857bc3", + "version" : "2.0.0" } }, { @@ -22,7 +23,16 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/ChimeHQ/SwiftTreeSitter", "state" : { - "revision" : "b01904a3737649c1d8520106bbb285724fe5b0bb" + "revision" : "55f2c2fdaf859f86e4ea8513b9934badc7894019" + } + }, + { + "identity" : "tree-sitter", + "kind" : "remoteSourceControl", + "location" : "https://github.com/tree-sitter/tree-sitter", + "state" : { + "revision" : "d97db6d63507eb62c536bcb2c4ac7d70c8ec665e", + "version" : "0.23.2" } }, { @@ -44,5 +54,5 @@ } } ], - "version" : 2 + "version" : 3 } diff --git a/Projects/NeonExample/TextViewController.swift b/Projects/NeonExample/TextViewController.swift index 0ef04da..9346286 100644 --- a/Projects/NeonExample/TextViewController.swift +++ b/Projects/NeonExample/TextViewController.swift @@ -11,18 +11,14 @@ final class TextViewController: NSUIViewController { private let highlighter: TextViewHighlighter init() { - if #available(iOS 16.0, *) { - self.textView = NSUITextView(usingTextLayoutManager: false) - } else { - self.textView = NSUITextView() - } + self.textView = NSUITextView(usingTextLayoutManager: false) self.highlighter = try! Self.makeHighlighter(for: textView) super.init(nibName: nil, bundle: nil) // enable non-continguous layout for TextKit 1 - if #available(macOS 12.0, iOS 16.0, tvOS 15.0, *), textView.textLayoutManager == nil { + if textView.textLayoutManager == nil { textView.nsuiLayoutManager?.allowsNonContiguousLayout = true } } @@ -119,17 +115,17 @@ final class TextViewController: NSUIViewController { // it wasn't that way on creation highlighter.observeEnclosingScrollView() - regularTest() + regularTestWithSwiftCode() } - func regularTest() { + func regularTestWithSwiftCode() { let url = Bundle.main.url(forResource: "test", withExtension: "code")! let content = try! String(contentsOf: url) textView.text = content } - func doBigTest() { + func doBigMarkdownTest() { let url = Bundle.main.url(forResource: "big_test", withExtension: "md")! let content = try! String(contentsOf: url) diff --git a/README.md b/README.md index a981ca9..069e5fa 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ Neon is made up of three parts: the core library, `RangeState` and `TreeSitterCl Neon's lowest-level component is called RangeState. This module contains the core building blocks used for the rest of the system. RangeState is built around the idea of hybrid synchronous/asynchronous execution. Making everything async is a lot easier, but that makes it impossible to provide a low-latency path for small documents. It is content-independent. -- `Hybrid(Throwing)ValueProvider`: a fundamental type that defines work in terms of both synchronous and asynchronous functions +- `HybridSyncAsyncValueProvider`: a fundamental type that defines work in terms of both synchronous and asynchronous functions - `RangeProcessor`: performs on-demand processing of range-based content (think parsing) - `RangeValidator`: building block for managing the validation of range-based content - `RangeInvalidationBuffer`: buffer and consolidate invalidations so they can be applied at the optimal time @@ -60,8 +60,6 @@ Neon's lowest-level component is called RangeState. This module contains the cor Many of these support versionable content. If you are working with a backing store structure that supports efficient versioning, like a [piece table](https://en.wikipedia.org/wiki/Piece_table), expressing this to RangeState can improve its efficiency. -It might be surprising to see that many of the types in RangeState are marked `@MainActor`. Right now, I have found no way to both support the hybrid sync/async functionality while also not being tied to a global actor. I think this is the most resonable trade-off, but I would very much like to lift this restriction. However, I believe it will require [language changes](https://forums.swift.org/t/isolation-assumptions/69514/47). - ### Neon The top-level module includes systems for managing text styling. It is also text-system independent. It makes very few assumptions about how text is stored, displayed, or styled. It also includes some components for use with stock AppKit and UIKit systems. These are provided for easy integration, not maximum performance. @@ -84,7 +82,9 @@ I have not yet figured out a way to do this with TextKit 2, and it may not be po #### Performance -Neon's performance is highly dependant on the text system integration. Every aspect is important as there are performance cliffs all around. But, priority range calcations (the visible set for most text views) are of particular importance. This is surprisingly challenging to do correctly with TextKit 1, and extremely hard with TextKit 2. +Neon's performance is highly dependant on the text system integration. Every aspect is important as there are performance cliffs all around. But, priority range calculations (the visible set for most text views) are of particular importance. This is surprisingly challenging to do correctly with TextKit 1, and extremely hard with TextKit 2. + +Bottom line: Neon is extremely efficient. The bottlenecks will probably be your parsing system and text view. It can do a decent job of hiding parsing performance issues too. However, that doesn't mean it's perfect. There's always room for improvement and if you suspect a problem please do file an issue. ### TreeSitterClient @@ -106,12 +106,16 @@ Neon was designed to accept and overlay token data from multiple sources simulta - Second pass: [tree-sitter](https://tree-sitter.github.io/tree-sitter/), which has good quality and **could** be low-latency - Third pass: [Language Server Protocol](https://microsoft.github.io/language-server-protocol/)'s [semantic tokens](https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textDocument_semanticTokens), which can augment existing highlighting, but is high-latency +An example of a first-pass system you might be interested in is [Lowlight](https://github.com/ChimeHQ/Lowlight). + ### Theming A highlighting theme is really just a mapping from semantic labels to styles. Token data sources apply the semantic labels and the `TextSystemInterface` uses those labels to look up styling. This separation makes it very easy for you to do this look-up in a way that makes the most sense for whatever theming formats you'd like to support. This is also a convenient spot to adapt/modify the semantic labels coming from your data sources into a normalized form. +If you are looking for some help here, check out [ThemePark](https://github.com/ChimeHQ/ThemePark). + ## Usage ### TreeSitterClient @@ -139,16 +143,15 @@ let clientConfig = TreeSitterClient.Configuration( // `languageConfigurationChanged(for:)` return nil }, - contentProvider: { [textView] length in - // given a maximum needed length, produce a `Content` structure - // that will be used to access the text data + contentSnapshotProvider: { [textView] length in + // given a maximum needed length, produce a `ContentSnapshot` structure + // that will be used to access immutable text data // this can work for any system that efficiently produce a `String` return .init(string: textView.string) }, lengthProvider: { [textView] in textView.string.utf16.count - }, invalidationHandler: { set in // take action on invalidated regions of the text diff --git a/Sources/Neon/TextSystemInterface+Validation.swift b/Sources/Neon/TextSystemInterface+Validation.swift index 72cd8ae..92755d9 100644 --- a/Sources/Neon/TextSystemInterface+Validation.swift +++ b/Sources/Neon/TextSystemInterface+Validation.swift @@ -44,7 +44,10 @@ extension TextSystemInterface { return validation(for: application, in: contentRange) }, mainActorAsyncValue: { contentRange in - return await asyncValidate(contentRange, provider: provider.mainActorAsync) + await asyncValidate( + contentRange, + provider: { range in await provider.async(range) } + ) } ) } @@ -69,6 +72,10 @@ extension TextSystemInterface { func validatorSecondaryHandler( with provider: @escaping Styler.SecondaryValidationProvider ) -> SecondaryValidationProvider { - { await asyncValidate($0, provider: provider) } + { range in + await asyncValidate(range, provider: { + await provider($0) + }) + } } } diff --git a/Sources/Neon/TextSystemStyler.swift b/Sources/Neon/TextSystemStyler.swift index fc9184f..73d1973 100644 --- a/Sources/Neon/TextSystemStyler.swift +++ b/Sources/Neon/TextSystemStyler.swift @@ -29,7 +29,8 @@ public final class TextSystemStyler { versionedContent: textSystem.content, provider: tokenValidator.validationProvider, prioritySetProvider: { textSystem.visibleSet } - ) + ), + isolation: MainActor.shared ) } @@ -57,7 +58,7 @@ public final class TextSystemStyler { public func visibleContentDidChange() { let prioritySet = textSystem.visibleSet - validator.validate(.set(prioritySet), prioritizing: prioritySet) + validator.validate(.set(prioritySet), prioritizing: prioritySet, isolation: MainActor.shared) } @@ -68,12 +69,12 @@ public final class TextSystemStyler { public func validate(_ target: RangeTarget) { let prioritySet = textSystem.visibleSet - validator.validate(target, prioritizing: prioritySet) + validator.validate(target, prioritizing: prioritySet, isolation: MainActor.shared) } public func validate() { let prioritySet = textSystem.visibleSet - validator.validate(.set(prioritySet), prioritizing: prioritySet) + validator.validate(.set(prioritySet), prioritizing: prioritySet, isolation: MainActor.shared) } } diff --git a/Sources/Neon/TextViewHighlighter.swift b/Sources/Neon/TextViewHighlighter.swift index 94f7876..fe32c62 100644 --- a/Sources/Neon/TextViewHighlighter.swift +++ b/Sources/Neon/TextViewHighlighter.swift @@ -35,6 +35,7 @@ extension TextView { /// A class that can connect `NSTextView`/`UITextView` to `TreeSitterClient` /// /// This class is a minimal implementation that can help perform highlighting for a TextView. It is compatible with both TextKit 1 and 2 views, and uses single-phase pass with tree-sitter. The created instance will become the delegate of the view's `NSTextStorage`. +@available(macOS 13.0, macCatalyst 16.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) @MainActor public final class TextViewHighlighter { private typealias Styler = TextSystemStyler @@ -87,6 +88,7 @@ public final class TextViewHighlighter { configuration: .init( languageProvider: configuration.languageProvider, contentProvider: { [interface] in interface.languageLayerContent(with: $0) }, + contentSnapshopProvider: { [interface] in interface.languageLayerContentSnapshot(with: $0) }, lengthProvider: { [interface] in interface.content.currentLength }, invalidationHandler: { [buffer] in buffer.invalidate(.set($0)) }, locationTransformer: configuration.locationTransformer @@ -131,7 +133,7 @@ public final class TextViewHighlighter { try textView.getTextStorage().delegate = storageDelegate - observeEnclosingScrollView() + observeEnclosingScrollView() invalidate(.all) } @@ -147,6 +149,7 @@ public final class TextViewHighlighter { } } +@available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) extension TextViewHighlighter { /// Begin monitoring for containing scroll view changes. /// @@ -157,7 +160,7 @@ extension TextViewHighlighter { print("warning: there is no enclosing scroll view") return } - + NotificationCenter.default.addObserver( self, selector: #selector(visibleContentChanged(_:)), @@ -173,7 +176,7 @@ extension TextViewHighlighter { ) #elseif os(iOS) || os(visionOS) self.frameObservation = textView.observe(\.contentOffset) { [weak self] view, _ in - MainActor.assumeIsolated { + MainActor.assumeIsolated { guard let self = self else { return } self.lastVisibleRange = self.textView.visibleTextRange diff --git a/Sources/Neon/ThreePhaseTextSystemStyler.swift b/Sources/Neon/ThreePhaseTextSystemStyler.swift index b63ee34..655976c 100644 --- a/Sources/Neon/ThreePhaseTextSystemStyler.swift +++ b/Sources/Neon/ThreePhaseTextSystemStyler.swift @@ -31,7 +31,8 @@ public final class ThreePhaseTextSystemStyler { secondaryProvider: textSystem.validatorSecondaryHandler(with: secondaryValidationProvider), secondaryValidationDelay: 3.0, prioritySetProvider: { textSystem.visibleSet } - ) + ), + isolation: MainActor.shared ) } @@ -46,12 +47,12 @@ public final class ThreePhaseTextSystemStyler { public func validate(_ target: RangeTarget) { let prioritySet = textSystem.visibleSet - validator.validate(target, prioritizing: prioritySet) + validator.validate(target, prioritizing: prioritySet, isolation: MainActor.shared) } public func validate() { let prioritySet = textSystem.visibleSet - validator.validate(.set(prioritySet), prioritizing: prioritySet) + validator.validate(.set(prioritySet), prioritizing: prioritySet, isolation: MainActor.shared) } } diff --git a/Sources/Neon/Token.swift b/Sources/Neon/Token.swift index ec0995e..a7fdbf8 100644 --- a/Sources/Neon/Token.swift +++ b/Sources/Neon/Token.swift @@ -48,7 +48,7 @@ public struct TokenApplication: Hashable, Sendable { /// The underlying parsing system must be able to translate a request for tokens expressed as an `NSRange` into a `TokenApplication`. /// /// This would be a lot easier to implement if the interface was purely asynchronous. However, Neon provides a fully synchronous styling path. Avoiding the need for an async context can be very useful, and makes it possible to provide a flicker-free guarantee if the underlying parsing system can process the work required in reasonable time. Your actual implementation, however, does not actually have to implement the synchronous path if that's too difficult. -public typealias TokenProvider = HybridValueProvider +public typealias TokenProvider = HybridSyncAsyncValueProvider extension TokenProvider { /// A TokenProvider that returns an empty set of tokens for all requests. @@ -62,4 +62,16 @@ extension TokenProvider { } ) } + + /// A TokenProvider that returns an empty set of tokens for all async requests, but fails to resolve tokens synchronously. + public static var asyncOnlyNone: TokenProvider { + .init( + syncValue: { _ in + return nil + }, + asyncValue: { _, _ in + return .noChange + } + ) + } } diff --git a/Sources/Neon/TokenSystemValidator.swift b/Sources/Neon/TokenSystemValidator.swift index 2ed76b9..83b8e22 100644 --- a/Sources/Neon/TokenSystemValidator.swift +++ b/Sources/Neon/TokenSystemValidator.swift @@ -14,10 +14,10 @@ final class TokenSystemValidator { self.tokenProvider = tokenProvider } - var validationProvider: HybridValueProvider { + var validationProvider: HybridSyncAsyncValueProvider { .init( syncValue: { self.validate($0) }, - asyncValue: { range, _ in await self.validate(range)} + asyncValue: { _, range in await self.validate(range)} ) } @@ -41,7 +41,7 @@ final class TokenSystemValidator { guard range.version == currentVersion else { return .stale } // https://github.com/apple/swift/pull/71143 - let application = await tokenProvider.mainActorAsync(range.value) + let application = await tokenProvider.async(isolation: MainActor.shared, range.value) applyStyles(for: application) diff --git a/Sources/Neon/TreeSitterClient+Neon.swift b/Sources/Neon/TreeSitterClient+Neon.swift index 55a9672..c5743d3 100644 --- a/Sources/Neon/TreeSitterClient+Neon.swift +++ b/Sources/Neon/TreeSitterClient+Neon.swift @@ -15,8 +15,19 @@ extension TokenApplication { self.init(tokens: tokens, range: range) } + + public init(namedRanges: [NamedRange], range: NSRange) { + let tokens = namedRanges.map { + let name = $0.name + + return Token(name: name, range: $0.range) + } + + self.init(tokens: tokens, range: range) + } } +@available(macOS 13.0, macCatalyst 16.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) extension TreeSitterClient { @MainActor public func tokenProvider(with provider: @escaping TextProvider, nameMap: [String : String] = [:]) -> TokenProvider { @@ -36,7 +47,7 @@ extension TreeSitterClient { mainActorAsyncValue: { [highlightsProvider] range in do { let params = TreeSitterClient.ClientQueryParams(range: range, textProvider: provider) - let namedRanges = try await highlightsProvider.mainActorAsync(params) + let namedRanges = try await highlightsProvider.async(params) return TokenApplication(namedRanges: namedRanges, nameMap: nameMap, range: range) } catch { @@ -47,26 +58,18 @@ extension TreeSitterClient { } } -extension LanguageLayer.Content { - /// this should probably move into SwiftTreeSitterLayer - init(string: String, limit: Int) { - let read = Parser.readFunction(for: string, limit: limit) - - self.init( - readHandler: read, - textProvider: string.predicateTextProvider - ) - } -} - #if os(macOS) || os(iOS) || os(visionOS) extension TextViewSystemInterface { func languageLayerContent(with limit: Int) -> LanguageLayer.Content { LanguageLayer.Content(string: textStorage.string, limit: limit) } + + func languageLayerContentSnapshot(with limit: Int) -> LanguageLayer.ContentSnapshot { + LanguageLayer.ContentSnapshot(string: textStorage.string, limit: limit) + } } -@available(macOS 12.0, macCatalyst 15.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *) +@available(macOS 13.0, macCatalyst 16.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) extension TreeSitterClient { /// Highlight an input string. public static func highlight( @@ -75,14 +78,14 @@ extension TreeSitterClient { rootLanguageConfig: LanguageConfiguration, languageProvider: @escaping LanguageLayer.LanguageProvider ) async throws -> AttributedString { - let content = LanguageLayer.Content(string: string) + let content = LanguageLayer.ContentSnapshot(string: string) let length = string.utf16.count let client = try TreeSitterClient( rootLanguageConfig: rootLanguageConfig, configuration: Configuration( languageProvider: languageProvider, - contentProvider: { _ in content }, + contentSnapshopProvider: { _ in content }, lengthProvider: { length }, invalidationHandler: { _ in }, locationTransformer: { _ in nil } diff --git a/Sources/RangeState/HybridSyncAsyncValueProvider.swift b/Sources/RangeState/HybridSyncAsyncValueProvider.swift new file mode 100644 index 0000000..99569d6 --- /dev/null +++ b/Sources/RangeState/HybridSyncAsyncValueProvider.swift @@ -0,0 +1,79 @@ +import Foundation + +/// A type that can perform work both synchronously and asynchronously. +public struct HybridSyncAsyncValueProvider { + public typealias SyncValueProvider = (Input) throws(Failure) -> Output? + public typealias AsyncValueProvider = (isolated (any Actor)?, sending Input) async throws(Failure) -> sending Output + + public let syncValueProvider: SyncValueProvider + public let asyncValueProvider: AsyncValueProvider + + public init( + syncValue: @escaping SyncValueProvider = { _ in nil }, + asyncValue: @escaping AsyncValueProvider + ) { + self.syncValueProvider = syncValue + self.asyncValueProvider = asyncValue + } + + public func async(isolation: isolated (any Actor)? = #isolation, _ input: sending Input) async throws(Failure) -> sending Output { + try await asyncValueProvider(isolation, input) + } + + + public func sync(_ input: Input) throws(Failure) -> Output? { + try syncValueProvider(input) + } +} + +extension HybridSyncAsyncValueProvider { + /// Create an instance that can statically prove to the compiler that asyncValueProvider is isolated to the MainActor. + public init( + syncValue: @escaping SyncValueProvider = { _ in nil }, + mainActorAsyncValue: @escaping @MainActor (Input) async throws(Failure) -> sending Output + ) { + self.syncValueProvider = syncValue + self.asyncValueProvider = { _, input async throws(Failure) in + try await mainActorAsyncValue(input) + } + } +} + +// I've not yet gotten these working right, but I think there could be something here. +extension HybridSyncAsyncValueProvider { + // Returns a new `HybridSyncAsyncValueProvider` with a new output type. +// func map(_ transform: @escaping (isolated (any Actor)?, Output) throws -> T) -> HybridSyncAsyncValueProvider { +// .init( +// syncValue: { input in +// guard let output = try sync(input) else { +// return nil +// } +// +// return try transform(#isolation, output) +// }, +// asyncValue: { (isolation, input) in +// try transform(isolation, try await self.async(isolation: isolation, input)) +// } +// ) +// } + +// /// Transforms the `Failure` type of `HybridSyncAsyncValueProvider` to `Never`, +// func catching(_ block: @escaping (Input, Error) -> Output) -> HybridSyncAsyncValueProvider { +// .init( +// syncValue: { +// do { +// return try self.sync($0) +// } catch { +// return block($0, error) +// } +// }, +// asyncValue: { +// do { +// return try await self.async(isolation: $0, $1) +// } catch { +// return block($1, error) +// } +// } +// ) +// } +} diff --git a/Sources/RangeState/HybridValueProvider+RangeProcessor.swift b/Sources/RangeState/HybridValueProvider+RangeProcessor.swift index 7866567..c22c075 100644 --- a/Sources/RangeState/HybridValueProvider+RangeProcessor.swift +++ b/Sources/RangeState/HybridValueProvider+RangeProcessor.swift @@ -1,63 +1,37 @@ import Foundation -extension HybridValueProvider { - /// Construct a `HybridValueProvider` that will first attempt to process a location using a `RangeProcessor`. - @MainActor +extension HybridSyncAsyncValueProvider { + /// Construct a `HybridSyncAsyncValueProvider` that will first attempt to process a location using a `RangeProcessor`. public init( + isolation: isolated (any Actor)? = #isolation, rangeProcessor: RangeProcessor, inputTransformer: @escaping (Input) -> (Int, RangeFillMode), syncValue: @escaping SyncValueProvider, - asyncValue: @escaping @MainActor (Input) async -> Output + asyncValue: @escaping (Input) async throws(Failure) -> sending Output ) { - self.init( - syncValue: { input in - let (location, fill) = inputTransformer(input) - - if rangeProcessor.processLocation(location, mode: fill) { - return syncValue(input) - } - - return nil - }, - asyncValue: { input, actor in - let (location, fill) = inputTransformer(input) - - await rangeProcessor.processLocation(location, mode: fill) - await rangeProcessor.processingCompleted() + // bizarre local-function workaround https://github.com/swiftlang/swift/issues/77067 + func _syncVersion(input: Input) throws(Failure) -> Output? { + let (location, fill) = inputTransformer(input) - return await asyncValue(input) + if rangeProcessor.processLocation(isolation: isolation, location, mode: fill) { + return try syncValue(input) } - ) - } -} -extension HybridThrowingValueProvider { - /// Construct a `HybridThrowingValueProvider` that will first attempt to process a location using a `RangeProcessor`. - @MainActor - public init( - rangeProcessor: RangeProcessor, - inputTransformer: @escaping (Input) -> (Int, RangeFillMode), - syncValue: @escaping SyncValueProvider, - asyncValue: @escaping @MainActor (Input) async throws -> Output - ) { - self.init( - syncValue: { input in - let (location, fill) = inputTransformer(input) + return nil + } - if rangeProcessor.processLocation(location, mode: fill) { - return try syncValue(input) - } + func _asyncVersion(isolation: isolated(any Actor)?, input: sending Input) async throws(Failure) -> sending Output { + let (location, fill) = inputTransformer(input) - return nil - }, - asyncValue: { input, actor in - let (location, fill) = inputTransformer(input) + rangeProcessor.processLocation(isolation: isolation, location, mode: fill) + await rangeProcessor.processingCompleted() - await rangeProcessor.processLocation(location, mode: fill) - await rangeProcessor.processingCompleted() + return try await asyncValue(input) + } - return try await asyncValue(input) - } + self.init( + syncValue: _syncVersion, + asyncValue: _asyncVersion ) } } diff --git a/Sources/RangeState/HybridValueProvider.swift b/Sources/RangeState/HybridValueProvider.swift deleted file mode 100644 index 3092a0d..0000000 --- a/Sources/RangeState/HybridValueProvider.swift +++ /dev/null @@ -1,139 +0,0 @@ -import Foundation - -/// A type that can perform work both synchronously and asynchronously. -public struct HybridValueProvider { - public typealias SyncValueProvider = (Input) -> Output? - public typealias AsyncValueProvider = (Input, isolated any Actor) async -> Output - - public let syncValueProvider: SyncValueProvider - public let asyncValueProvider: AsyncValueProvider - - public init( - syncValue: @escaping SyncValueProvider = { _ in nil }, - asyncValue: @escaping AsyncValueProvider - ) { - self.syncValueProvider = syncValue - self.asyncValueProvider = asyncValue - } - - public func async(_ input: Input, isolatedTo actor: any Actor) async -> Output { - await asyncValueProvider(input, actor) - } - - - public func sync(_ input: Input) -> Output? { - syncValueProvider(input) - } -} - -extension HybridValueProvider { - /// Create an instance that can statically prove to the compiler that asyncValueProvider is isolated to the MainActor. - public init( - syncValue: @escaping SyncValueProvider = { _ in nil }, - mainActorAsyncValue: @escaping @MainActor (Input) async -> Output - ) { - self.syncValueProvider = syncValue - self.asyncValueProvider = { input, _ in - return await mainActorAsyncValue(input) - } - } - - /// Hopefully temporary until https://github.com/apple/swift/pull/71143 is available. - @MainActor - public func mainActorAsync(_ input: Input) async -> Output { - await asyncValueProvider(input, MainActor.shared) - } -} - -/// A type that can perform failable work both synchronously and asynchronously. -public struct HybridThrowingValueProvider { - public typealias SyncValueProvider = (Input) throws -> Output? - public typealias AsyncValueProvider = (Input, isolated any Actor) async throws -> Output - - public let syncValueProvider: SyncValueProvider - public let asyncValueProvider: AsyncValueProvider - - public init( - syncValue: @escaping SyncValueProvider = { _ in nil }, - asyncValue: @escaping AsyncValueProvider - ) { - self.syncValueProvider = syncValue - self.asyncValueProvider = asyncValue - } - - public func async(_ input: Input, isolatedTo actor: any Actor) async throws -> Output { - try await asyncValueProvider(input, actor) - } - - public func sync(_ input: Input) throws -> Output? { - try syncValueProvider(input) - } -} - -extension HybridThrowingValueProvider { - /// Create an instance that can statically prove to the compiler that asyncValueProvider is isolated to the MainActor. - public init( - syncValue: @escaping SyncValueProvider = { _ in nil }, - mainActorAsyncValue: @escaping @MainActor (Input) async -> Output - ) { - self.syncValueProvider = syncValue - self.asyncValueProvider = { input, _ in - return await mainActorAsyncValue(input) - } - } - - /// Hopefully temporary until https://github.com/apple/swift/pull/71143 is available. - @MainActor - public func mainActorAsync(_ input: Input) async throws -> Output { - try await asyncValueProvider(input, MainActor.shared) - } -} - -// I believe these may still be implementable when https://github.com/apple/swift/pull/71143 is available. -//extension HybridValueProvider { -// /// Returns a new `HybridValueProvider` with a new output type. -// public func map(_ transform: @escaping (Output) -> T) -> HybridValueProvider where T: Sendable { -// .init( -// syncValue: { self.sync($0).map(transform) }, -// asyncValue: { transform(await self.async($0, isolatedTo: $1)) } -// ) -// } -// -// /// Convert to a `HybridThrowingValueProvider`. -// public var throwing: HybridThrowingValueProvider { -// .init( -// syncValue: self.syncValueProvider, -// asyncValue: self.asyncValueProvider -// ) -// } -//} - -//extension HybridThrowingValueProvider { - /// Returns a new `HybridThrowingValueProvider` with a new output type. -// public func map(_ transform: @escaping (Output, isolated (any Actor)?) throws -> T) -> HybridThrowingValueProvider where T: Sendable { -// .init( -// syncValue: { input in try self.sync($0).map({ transform($0, nil) }) }, -// asyncValue: { try transform(try await self.async($0, isolatedTo: $1), $1) } -// ) -// } - -// /// Transforms a `HybridThrowingValueProvider` into a `HybridValueProvider`. -// public func catching(_ block: @escaping (Input, Error) -> Output) -> HybridValueProvider { -// .init( -// syncValue: { -// do { -// return try self.sync($0) -// } catch { -// return block($0, error) -// } -// }, -// asyncValue: { -// do { -// return try await self.async($0, isolatedTo: $1) -// } catch { -// return block($0, error) -// } -// } -// ) -// } -//} diff --git a/Sources/RangeState/RangeProcessor.swift b/Sources/RangeState/RangeProcessor.swift index 5aad898..baaf80d 100644 --- a/Sources/RangeState/RangeProcessor.swift +++ b/Sources/RangeState/RangeProcessor.swift @@ -18,7 +18,6 @@ public enum RangeFillMode: Sendable, Hashable { } /// A type that can perform on-demand processing of range-based data. -@MainActor public final class RangeProcessor { private typealias Continuation = CheckedContinuation<(), Never> private typealias VersionedMutation = Versioned @@ -31,7 +30,7 @@ public final class RangeProcessor { /// Function to apply changes. /// /// These mutations can come from the content being modified or from operations that require lazy processing. The parameter's postApplyLimit property defines the maximum read location and needs to be respected to preserve the lazy access semantics. - public typealias ChangeHandler = (RangeMutation, @MainActor @escaping () -> Void) -> Void + public typealias ChangeHandler = (RangeMutation, @escaping () -> Void) -> Void public typealias LengthProvider = () -> Int public struct Configuration { @@ -109,7 +108,7 @@ extension RangeProcessor { /// - Returns: true if the location has been processed @discardableResult - public func processLocation(_ location: Int, mode: RangeFillMode = .required) -> Bool { + public func processLocation(isolation: isolated (any Actor)? = #isolation, _ location: Int, mode: RangeFillMode = .required) -> Bool { switch mode { case .none: break @@ -117,14 +116,14 @@ extension RangeProcessor { // update our target self.targetProcessingLocation = max(location, targetProcessingLocation) - scheduleFilling() + scheduleFilling(in: isolation) case .required: if hasPendingChanges { break } if let mutation = fillMutationNeeded(for: location, mode: mode) { - processMutation(mutation) + processMutation(mutation, in: isolation) } } @@ -158,7 +157,7 @@ extension RangeProcessor { } } - public func processingCompleted() async { + public func processingCompleted(isolation: isolated (any Actor)? = #isolation) async { if hasPendingChanges == false { return } @@ -186,7 +185,7 @@ extension RangeProcessor { didChangeContent(in: mutation.range, delta: mutation.delta) } - public func didChangeContent(in range: NSRange, delta: Int) { + public func didChangeContent(isolation: isolated (any Actor)? = #isolation, in range: NSRange, delta: Int) { if processed(range.location) == false { return } @@ -216,21 +215,24 @@ extension RangeProcessor { let mutation = RangeMutation(range: visibleRange, delta: visibleDelta, limit: effectiveLimit) - processMutation(mutation) + processMutation(mutation, in: isolation) } - private func processMutation(_ mutation: RangeMutation) { + private func processMutation(_ mutation: RangeMutation, in isolation: isolated (any Actor)?) { self.pendingEvents.append(.change(VersionedMutation(mutation, version: version))) self.version += 1 - // at this point, it is possible that the target location is no longer meaningful. But that's ok, because it will be clamped and possibly overwritten anyways + // this requires a very strange workaround to get the correct isolation inheritance from this changeHandler arrangement. I believe this is a bug. + // https://github.com/swiftlang/swift/issues/77067 + func _completeContentChanged() { + self.completeContentChanged(mutation, in: isolation) + } - configuration.changeHandler(mutation, { - self.completeContentChanged(mutation) - }) + // at this point, it is possible that the target location is no longer meaningful. But that's ok, because it will be clamped and possibly overwritten anyways + configuration.changeHandler(mutation, _completeContentChanged) } - private func completeContentChanged(_ mutation: RangeMutation) { + private func completeContentChanged(_ mutation: RangeMutation, in isolation: isolated (any Actor)?) { self.processedVersion += 1 resumeLeadingContinuations() @@ -249,10 +251,10 @@ extension RangeProcessor { updateProcessedLocation(by: mutation.delta) - scheduleFilling() + scheduleFilling(in: isolation) } - public func continueFillingIfNeeded() { + public func continueFillingIfNeeded(isolation: isolated (any Actor)? = #isolation) { if hasPendingChanges { return } @@ -262,12 +264,12 @@ extension RangeProcessor { guard let mutation else { return } - processMutation(mutation) + processMutation(mutation, in: isolation) } - private func scheduleFilling() { - DispatchQueue.main.async { - self.continueFillingIfNeeded() + private func scheduleFilling(in isolation: isolated (any Actor)?) { + Task { + self.continueFillingIfNeeded(isolation: isolation) } } } diff --git a/Sources/RangeState/RangeValidator.swift b/Sources/RangeState/RangeValidator.swift index ee1d0d4..83c3c51 100644 --- a/Sources/RangeState/RangeValidator.swift +++ b/Sources/RangeState/RangeValidator.swift @@ -10,7 +10,7 @@ public enum Validation: Sendable, Hashable { /// A type that manages the validation of range-based content. public final class RangeValidator { public typealias ContentRange = VersionedRange - public typealias ValidationProvider = HybridValueProvider + public typealias ValidationProvider = HybridSyncAsyncValueProvider public enum Action: Sendable, Equatable { case none diff --git a/Sources/RangeState/SinglePhaseRangeValidator.swift b/Sources/RangeState/SinglePhaseRangeValidator.swift index 2d98b37..f8e052f 100644 --- a/Sources/RangeState/SinglePhaseRangeValidator.swift +++ b/Sources/RangeState/SinglePhaseRangeValidator.swift @@ -2,11 +2,9 @@ import Foundation import Rearrange -@MainActor public final class SinglePhaseRangeValidator { - public typealias ContentRange = RangeValidator.ContentRange - public typealias Provider = HybridValueProvider + public typealias Provider = HybridSyncAsyncValueProvider public typealias PrioritySetProvider = () -> IndexSet private typealias Sequence = AsyncStream @@ -33,6 +31,24 @@ public final class SinglePhaseRangeValidator { public let configuration: Configuration public var validationHandler: (NSRange) -> Void = { _ in } + public init(configuration: Configuration, isolation: isolated (any Actor)) { + self.configuration = configuration + self.primaryValidator = RangeValidator(content: configuration.versionedContent) + + let (stream, continuation) = Sequence.makeStream() + + self.continuation = continuation + + Task { [weak self] in + _ = isolation + + for await versionedRange in stream { + await self?.validateRangeAsync(versionedRange, isolation: isolation) + } + } + } + + @MainActor public init(configuration: Configuration) { self.configuration = configuration self.primaryValidator = RangeValidator(content: configuration.versionedContent) @@ -43,7 +59,7 @@ public final class SinglePhaseRangeValidator { Task { [weak self] in for await versionedRange in stream { - await self?.validateRangeAsync(versionedRange) + await self?.validateRangeAsync(versionedRange, isolation: MainActor.shared) } } } @@ -62,7 +78,11 @@ public final class SinglePhaseRangeValidator { } @discardableResult - public func validate(_ target: RangeTarget, prioritizing set: IndexSet? = nil) -> RangeValidator.Action { + public func validate( + _ target: RangeTarget, + prioritizing set: IndexSet? = nil, + isolation: isolated (any Actor) + ) -> RangeValidator.Action { // capture this first, because we're about to start one let outstanding = primaryValidator.hasOutstandingValidations @@ -84,26 +104,38 @@ public final class SinglePhaseRangeValidator { return action } - completePrimaryValidation(of: contentRange, with: validation) + completePrimaryValidation(of: contentRange, with: validation, isolation: isolation) return .none } } - private func completePrimaryValidation(of contentRange: ContentRange, with validation: Validation) { + @MainActor + @discardableResult + public func validate( + _ target: RangeTarget, + prioritizing set: IndexSet? = nil + ) -> RangeValidator.Action { + validate(target, prioritizing: set, isolation: MainActor.shared) + } + + private func completePrimaryValidation(of contentRange: ContentRange, with validation: Validation, isolation: isolated (any Actor)) { primaryValidator.completeValidation(of: contentRange, with: validation) switch validation { case .stale: - DispatchQueue.main.async { + Task { + _ = isolation + if contentRange.version == self.version { print("version unchanged after stale results, stopping validation") return } - + let prioritySet = self.configuration.prioritySetProvider?() ?? IndexSet(contentRange.value) - self.validate(.set(prioritySet)) + self.validate(.set(prioritySet), isolation: isolation) + } case let .success(range): validationHandler(range) @@ -131,9 +163,9 @@ public final class SinglePhaseRangeValidator { continuation.yield(contentRange) } - private func validateRangeAsync(_ contentRange: ContentRange) async { - let validation = await self.configuration.provider.mainActorAsync(contentRange) + private func validateRangeAsync(_ contentRange: ContentRange, isolation: isolated (any Actor)) async { + let validation = await self.configuration.provider.async(contentRange) - completePrimaryValidation(of: contentRange, with: validation) + completePrimaryValidation(of: contentRange, with: validation, isolation: isolation) } } diff --git a/Sources/RangeState/ThreePhaseRangeValidator.swift b/Sources/RangeState/ThreePhaseRangeValidator.swift index 014de83..3b22eb5 100644 --- a/Sources/RangeState/ThreePhaseRangeValidator.swift +++ b/Sources/RangeState/ThreePhaseRangeValidator.swift @@ -2,7 +2,6 @@ import Foundation import Rearrange -@MainActor public final class ThreePhaseRangeValidator { public typealias PrimaryValidator = SinglePhaseRangeValidator private typealias InternalValidator = RangeValidator @@ -48,20 +47,30 @@ public final class ThreePhaseRangeValidator { public let configuration: Configuration - public init(configuration: Configuration) { + public init(configuration: Configuration, isolation: isolated (any Actor)) { self.configuration = configuration self.primaryValidator = PrimaryValidator( configuration: .init( versionedContent: configuration.versionedContent, provider: configuration.provider, prioritySetProvider: configuration.prioritySetProvider - ) + ), + isolation: isolation ) self.fallbackValidator = InternalValidator(content: configuration.versionedContent) self.secondaryValidator = InternalValidator(content: configuration.versionedContent) - primaryValidator.validationHandler = { [unowned self] in self.handlePrimaryValidation(of: $0) } + func _validationHandler(_ range: NSRange) { + handlePrimaryValidation(of: range, isolation: isolation) + } + + primaryValidator.validationHandler = _validationHandler + } + + @MainActor + public convenience init(configuration: Configuration) { + self.init(configuration: configuration, isolation: MainActor.shared) } private var version: Content.Version { @@ -75,17 +84,22 @@ public final class ThreePhaseRangeValidator { secondaryValidator?.invalidate(target) } - public func validate(_ target: RangeTarget, prioritizing set: IndexSet? = nil) { - let action = primaryValidator.validate(target, prioritizing: set) + public func validate(_ target: RangeTarget, prioritizing set: IndexSet? = nil, isolation: isolated (any Actor)) { + let action = primaryValidator.validate(target, prioritizing: set, isolation: isolation) switch action { case .none: - scheduleSecondaryValidation(of: target, prioritizing: set) + scheduleSecondaryValidation(of: target, prioritizing: set, isolation: isolation) case let .needed(contentRange): fallbackValidate(contentRange.value, prioritizing: set) } } + @MainActor + public func validate(_ target: RangeTarget, prioritizing set: IndexSet? = nil) { + validate(target, prioritizing: set, isolation: MainActor.shared) + } + private func fallbackValidate(_ targetRange: NSRange, prioritizing set: IndexSet?) -> Void { guard let provider = configuration.fallbackHandler else { return } @@ -124,17 +138,17 @@ public final class ThreePhaseRangeValidator { } extension ThreePhaseRangeValidator { - private func handlePrimaryValidation(of range: NSRange) { + private func handlePrimaryValidation(of range: NSRange, isolation: isolated (any Actor)) { let target = RangeTarget.range(range) let prioritySet = configuration.prioritySetProvider?() ?? IndexSet(range) fallbackValidator.invalidate(target) secondaryValidator?.invalidate(target) - scheduleSecondaryValidation(of: target, prioritizing: prioritySet) + scheduleSecondaryValidation(of: target, prioritizing: prioritySet, isolation: isolation) } - private func scheduleSecondaryValidation(of target: RangeTarget, prioritizing set: IndexSet?) { + private func scheduleSecondaryValidation(of target: RangeTarget, prioritizing set: IndexSet?, isolation: isolated (any Actor)) { if configuration.secondaryProvider == nil || secondaryValidator == nil { return } @@ -145,13 +159,20 @@ extension ThreePhaseRangeValidator { let delay = max(UInt64(configuration.secondaryValidationDelay * 1_000_000_000), 0) self.task = Task { + _ = isolation + try await Task.sleep(nanoseconds: delay) - await secondaryValidate(target: target, requestingVersion: requestingVersion, prioritizing: set) + await secondaryValidate(target: target, requestingVersion: requestingVersion, prioritizing: set, isolation: isolation) } } - private func secondaryValidate(target: RangeTarget, requestingVersion: Content.Version, prioritizing set: IndexSet?) async { + private func secondaryValidate( + target: RangeTarget, + requestingVersion: Content.Version, + prioritizing set: IndexSet?, + isolation: isolated (any Actor) + ) async { guard requestingVersion == self.version, let validator = secondaryValidator, diff --git a/Sources/RangeState/Versioned.swift b/Sources/RangeState/Versioned.swift index 681340c..de0cf3b 100644 --- a/Sources/RangeState/Versioned.swift +++ b/Sources/RangeState/Versioned.swift @@ -1,5 +1,6 @@ import Foundation +/// Represents a value with changes that can be tracked over time. public struct Versioned { public var value: Value public var version: Version @@ -20,26 +21,26 @@ public typealias VersionedRange = Versioned /// /// This can be used to model text storage. If your backing store supports versioning, this can be used to improve efficiency. public protocol VersionedContent { - associatedtype Version: Equatable & Sendable + associatedtype Version: Equatable & Sendable - var currentVersion: Version { get } - func length(for version: Version) -> Int? + var currentVersion: Version { get } + func length(for version: Version) -> Int? } extension VersionedContent { - public var currentVersionedLength: Versioned { - let vers = currentVersion + public var currentVersionedLength: Versioned { + let vers = currentVersion - guard let value = length(for: vers) else { - preconditionFailure("length of current version must always be available") - } + guard let value = length(for: vers) else { + preconditionFailure("length of current version must always be available") + } - return .init(value, version: vers) - } + return .init(value, version: vers) + } - public var currentLength: Int { - currentVersionedLength.value - } + public var currentLength: Int { + currentVersionedLength.value + } } /// Content where only the current version is valid. diff --git a/Sources/TreeSitterClient/BackgroundProcessor.swift b/Sources/TreeSitterClient/BackgroundProcessor.swift new file mode 100644 index 0000000..8a1a0c1 --- /dev/null +++ b/Sources/TreeSitterClient/BackgroundProcessor.swift @@ -0,0 +1,84 @@ +import Dispatch + +fileprivate struct UnsafeContainer: @unchecked Sendable { + let value: T +} + +final class BackgroundProcessor { + private let valueContainer: UnsafeContainer + private let queue = DispatchQueue(label: "com.chimehq.Neon.BackgroundProcessor") + private var pendingCount = 0 + + public init(value: Value) { + self.valueContainer = UnsafeContainer(value: value) + } + + public var hasPendingWork: Bool { + pendingCount > 0 + } + + private func beginBackgroundWork() { + precondition(pendingCount >= 0) + pendingCount += 1 + } + + private func endBackgroundWork() { + pendingCount -= 1 + precondition(pendingCount >= 0) + } + + public func accessValueSynchronously() -> Value? { + if hasPendingWork == false { + return valueContainer.value + } + + return nil + } + + // I would like to downgrade T: Sendable to sending but that seems to not work + public func accessValue( + isolation: isolated (any Actor), + preferSynchronous: Bool, + operation: @escaping @Sendable (Value) throws -> sending T, + completion: @escaping (Result) -> Void + ) { + if preferSynchronous, let v = accessValueSynchronously() { + precondition(hasPendingWork == false) + + let result = Result { try operation(v) } + completion(result) + + precondition(hasPendingWork == false) + + return + } + + + self.beginBackgroundWork() + + Task { + _ = isolation + + let result = await runOperation(operation: operation) + + self.endBackgroundWork() + + completion(result) + } + } + + private nonisolated func runOperation(operation: @escaping @Sendable (Value) throws -> sending T) async -> sending Result { + Result { try operation(valueContainer.value) } + } + + public func accessValue( + isolation: isolated (any Actor), + operation: @escaping @Sendable (Value) throws -> sending T + ) async throws -> T { + try await withCheckedThrowingContinuation(isolation: isolation) { continuation in + accessValue(isolation: isolation, preferSynchronous: false, operation: operation, completion: { result in + continuation.resume(with: result) + }) + } + } +} diff --git a/Sources/TreeSitterClient/BackgroundingLanguageLayerTree.swift b/Sources/TreeSitterClient/BackgroundingLanguageLayerTree.swift index a48eac2..bc7484e 100644 --- a/Sources/TreeSitterClient/BackgroundingLanguageLayerTree.swift +++ b/Sources/TreeSitterClient/BackgroundingLanguageLayerTree.swift @@ -8,7 +8,7 @@ enum BackgroundingLanguageLayerTreeError: Error { case unableToSnapshot } -@MainActor +@available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) final class BackgroundingLanguageLayerTree { public static let synchronousLengthThreshold = 2048 public static let synchronousDocumentSize = 2048*512 @@ -33,86 +33,74 @@ final class BackgroundingLanguageLayerTree { } } - private let queue = DispatchQueue(label: "com.chimehq.QueuedLanguageLayerTree") + private let queue = DispatchQueue(label: "com.chimehq.BackgroundingLanguageLayerTree") private var currentVersion = 0 private var committedVersion = 0 private var pendingOldPoint: Point? - private let rootLayer: LanguageLayer private let configuration: Configuration + private let backgroundProcessor: BackgroundProcessor public init(rootLanguageConfig: LanguageConfiguration, configuration: Configuration) throws { self.configuration = configuration - self.rootLayer = try LanguageLayer(languageConfig: rootLanguageConfig, configuration: configuration.layerConfiguration) - } - - private func accessTreeSynchronously(version: Int) -> LanguageLayer? { - guard version == committedVersion else { return nil } + let rootLayer = try LanguageLayer(languageConfig: rootLanguageConfig, configuration: configuration.layerConfiguration) - return rootLayer + self.backgroundProcessor = BackgroundProcessor(value: rootLayer) } - private func accessTree( - version: Int, - preferSynchronous: Bool, - operation: @escaping (LanguageLayer) throws -> T, - completion: @escaping @MainActor (Result) -> Void - ) { - if preferSynchronous, let tree = accessTreeSynchronously(version: version) { - let result = Result(catching: { try operation(tree) }) - completion(result) - return - } - - // this must be unsafe because LanguageLayerTree is not Sendable. However access is gated through the main actor/queue. - queue.backport.asyncUnsafe { [rootLayer] in - let result = Result(catching: { try operation(rootLayer) }) - - DispatchQueue.main.async { - completion(result) - } - } + private func accessTreeSynchronously(version: Int) -> LanguageLayer? { + backgroundProcessor.accessValueSynchronously() } public func willChangeContent(in range: NSRange) { self.pendingOldPoint = configuration.locationTransformer(range.max) } - public func didChangeContent(_ content: LanguageLayer.Content, in range: NSRange, delta: Int, completion: @escaping @MainActor (IndexSet) -> Void) { + public func didChangeContent( + _ snapshot: LanguageLayer.ContentSnapshot, + in range: NSRange, + delta: Int, + isolation: isolated (any Actor), + completion: @escaping (IndexSet) -> Void + ) { let transformer = configuration.locationTransformer - let upToDate = currentVersion == committedVersion let smallChange = delta < Self.synchronousLengthThreshold && range.length < Self.synchronousLengthThreshold let smallDoc = range.max < Self.synchronousDocumentSize - let sync = upToDate && smallChange && smallDoc - - let version = currentVersion - self.currentVersion += 1 + let sync = smallChange && smallDoc let oldEndPoint = pendingOldPoint ?? transformer(range.max) ?? .zero let edit = InputEdit(range: range, delta: delta, oldEndPoint: oldEndPoint, transformer: transformer) - accessTree(version: version, preferSynchronous: sync) { tree in - tree.didChangeContent(content, using: edit, resolveSublayers: false) - } completion: { result in - self.committedVersion += 1 - - do { - completion(try result.get()) - } catch { - preconditionFailure("didChangeContent should not be able to fail: \(error)") + backgroundProcessor.accessValue( + isolation: isolation, + preferSynchronous: sync, + operation: { $0.didChangeContent(snapshot.content, using: edit, resolveSublayers: false) }, + completion: { result in + do { + completion(try result.get()) + } catch { + preconditionFailure("didChangeContent should not be able to fail: \(error)") + } } - } + ) } - public func languageConfigurationChanged(for name: String, content: LanguageLayer.Content, completion: @escaping @MainActor (Result) -> Void) { - accessTree(version: currentVersion, preferSynchronous: true) { tree in - try tree.languageConfigurationChanged(for: name, content: content) - } completion: { - completion($0) - } + public func languageConfigurationChanged( + for name: String, + content snapshot: LanguageLayer.ContentSnapshot, + isolation: isolated (any Actor), + completion: @escaping (Result) -> Void + ) { + backgroundProcessor.accessValue( + isolation: isolation, + preferSynchronous: true, + operation: { try $0.languageConfigurationChanged(for: name, content: snapshot.content) }, + completion: { result in completion(result) } + ) } } +@available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) extension BackgroundingLanguageLayerTree { public func executeQuery(_ queryDef: Query.Definition, in set: IndexSet) throws -> LanguageTreeQueryCursor { guard let tree = accessTreeSynchronously(version: currentVersion) else { @@ -122,32 +110,27 @@ extension BackgroundingLanguageLayerTree { return try tree.executeQuery(queryDef, in: set) } - public func executeQuery(_ queryDef: Query.Definition, in set: IndexSet) async throws -> [QueryMatch] { - try await withCheckedThrowingContinuation { continuation in - accessTree(version: currentVersion, preferSynchronous: false) { tree in - guard let snapshot = tree.snapshot(in: set) else { - throw BackgroundingLanguageLayerTreeError.unableToSnapshot - } + public func executeQuery(_ queryDef: Query.Definition, in set: IndexSet, isolation: isolated (any Actor)) async throws -> [QueryMatch] { + let snapshot = try await backgroundProcessor.accessValue(isolation: isolation) { layer in + guard let snapshot = layer.snapshot(in: set) else { + throw BackgroundingLanguageLayerTreeError.unableToSnapshot + } - return snapshot - } completion: { result in - DispatchQueue.global().backport.asyncUnsafe { - let cursorResult = result.flatMap { snapshot in - Result(catching: { - let cursor = try snapshot.executeQuery(queryDef, in: set) + return snapshot + } + + return try await Self.processSnapshot(queryDef, in: set, snapshot: snapshot) + } - // this prefetches results in the background - return cursor.map { $0 } - }) - } + private nonisolated static func processSnapshot(_ queryDef: Query.Definition, in set: IndexSet, snapshot: LanguageLayerTreeSnapshot) async throws -> sending [QueryMatch] { + let cursor = try snapshot.executeQuery(queryDef, in: set) - continuation.resume(with: cursorResult) - } - } - } + // this prefetches results in the background + return cursor.map { $0 } } } +@available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) extension BackgroundingLanguageLayerTree { public func resolveSublayers(with content: LanguageLayer.Content, in set: IndexSet) throws -> IndexSet { guard let tree = accessTreeSynchronously(version: currentVersion) else { @@ -157,13 +140,9 @@ extension BackgroundingLanguageLayerTree { return try tree.resolveSublayers(with: content, in: set) } - public func resolveSublayers(with content: LanguageLayer.Content, in set: IndexSet) async throws -> IndexSet { - try await withCheckedThrowingContinuation { continuation in - accessTree(version: currentVersion, preferSynchronous: false) { tree in - try tree.resolveSublayers(with: content, in: set) - } completion: { result in - continuation.resume(with: result) - } + public func resolveSublayers(with snapshot: LanguageLayer.ContentSnapshot, in set: IndexSet, isolation: isolated (any Actor)) async throws -> IndexSet { + try await backgroundProcessor.accessValue(isolation: isolation) { layer in + try layer.resolveSublayers(with: snapshot.content, in: set) } } } diff --git a/Sources/TreeSitterClient/DispatchQueueBackport.swift b/Sources/TreeSitterClient/DispatchQueueBackport.swift deleted file mode 100644 index 4d7d316..0000000 --- a/Sources/TreeSitterClient/DispatchQueueBackport.swift +++ /dev/null @@ -1,21 +0,0 @@ -import Foundation - -struct DispatchQueueBackport { - private let queue: DispatchQueue - - init(_ queue: DispatchQueue) { - self.queue = queue - } - - func asyncUnsafe(group: DispatchGroup? = nil, qos: DispatchQoS = .unspecified, flags: DispatchWorkItemFlags = [], execute unsafeWork: @escaping @convention(block) () -> Void) { - let work = unsafeBitCast(unsafeWork, to: (@Sendable @convention(block) () -> Void).self) - - queue.async(group: group, qos: qos, flags: flags, execute: work) - } -} - -extension DispatchQueue { - var backport: DispatchQueueBackport { - DispatchQueueBackport(self) - } -} diff --git a/Sources/TreeSitterClient/HybridSyncAsyncVersionedResource.swift b/Sources/TreeSitterClient/HybridSyncAsyncVersionedResource.swift new file mode 100644 index 0000000..2850f60 --- /dev/null +++ b/Sources/TreeSitterClient/HybridSyncAsyncVersionedResource.swift @@ -0,0 +1,69 @@ +import RangeState + +public struct HybridSyncAsyncLanguageLayer { + let provider: HybridSyncAsyncValueProvider + + +} + +extension HybridSyncAsyncValueProvider { + func access( + isolation: isolated (any Actor)? = #isolation, + input: sending Input, + operation: @escaping (Bool, Output) throws -> sending Success, + completion: @escaping (Result) -> Void + ) where Input: Sendable { + // make a synchronous attempt + do { + if let output = try sync(input) { + let result = Result { try operation(true, output) } + + completion(result) + } + } catch { + completion(.failure(error)) + return + } + + Task { + _ = isolation + + do { + let output = try await self.async(input) + + let result = Result { try operation(false, output) } + + completion(result) + } + } + } +} + +final class HybridSyncAsyncVersionedResource { + typealias Version = Int + typealias VersionedResource = Versioned + typealias SyncAvailable = (Version) -> Bool + typealias Provider = HybridSyncAsyncValueProvider + + private let resource: VersionedResource + public let syncAvailable: SyncAvailable + + init(resouce: Resource, syncAvailable: @escaping SyncAvailable) { + self.resource = VersionedResource(resouce, version: 0) + self.syncAvailable = syncAvailable + } + + func access( + version: Version, + operation: @escaping (Bool, Resource) throws -> sending Success, + completion: @escaping (Result) -> Void + ) { + if syncAvailable(resource.version) { + let result = Result(catching: { try operation(true, resource.value) }) + completion(result) + return + } + + + } +} diff --git a/Sources/TreeSitterClient/TreeSitterClient.swift b/Sources/TreeSitterClient/TreeSitterClient.swift index 147d2e2..8644fe8 100644 --- a/Sources/TreeSitterClient/TreeSitterClient.swift +++ b/Sources/TreeSitterClient/TreeSitterClient.swift @@ -14,13 +14,20 @@ enum TreeSitterClientError: Error { /// Interface with the tree-sitter parsing query system. /// -/// TreeSitterClient supports arbitrary language nesting and unified queries across the document. +/// TreeSitterClient supports arbitrary language nesting and unified queries across the document. Nesting is a fairly expensive thing to do, as it makes queries, edits, and invalidation calculations significantly more complex, on top of language layer resolution. /// /// Tree-sitter ultimately resolves to a single semantic view of the text, and is considered a single phase. However, it may require numerous validation/invalidation passes before fully resolving a document's content. +/// +/// Today, compiler limitations mean that this type must be @MainActor +@available(macOS 13.0, macCatalyst 16.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) public final class TreeSitterClient { public typealias TextProvider = SwiftTreeSitter.Predicate.TextProvider + /// Produces synchronously-accessible content that covers the range of `0.. LanguageLayer.Content + /// Produces a immutable snapshot of the existing content that covers the range of `0.. LanguageLayer.ContentSnapshot + public typealias HighlightsProvider = HybridSyncAsyncValueProvider private typealias SublayerValidator = SinglePhaseRangeValidator private static let deltaRange = 128.. Void public let locationTransformer: (Int) -> Point? @@ -45,7 +53,8 @@ public final class TreeSitterClient { /// public init( languageProvider: @escaping LanguageLayer.LanguageProvider = { _ in nil }, - contentProvider: @escaping (Int) -> LanguageLayer.Content, + contentProvider: @escaping ContentProvider, + contentSnapshopProvider: @escaping ContentSnapshotProvider, lengthProvider: @escaping RangeProcessor.LengthProvider, invalidationHandler: @escaping (IndexSet) -> Void, locationTransformer: @escaping (Int) -> Point?, @@ -53,6 +62,24 @@ public final class TreeSitterClient { ) { self.languageProvider = languageProvider self.contentProvider = contentProvider + self.contentSnapshotProvider = contentSnapshopProvider + self.lengthProvider = lengthProvider + self.invalidationHandler = invalidationHandler + self.locationTransformer = locationTransformer + self.maximumLanguageDepth = maximumLanguageDepth + } + + public init( + languageProvider: @escaping LanguageLayer.LanguageProvider = { _ in nil }, + contentSnapshopProvider: @escaping ContentSnapshotProvider, + lengthProvider: @escaping RangeProcessor.LengthProvider, + invalidationHandler: @escaping (IndexSet) -> Void, + locationTransformer: @escaping (Int) -> Point?, + maximumLanguageDepth: Int = 4 + ) { + self.languageProvider = languageProvider + self.contentProvider = nil + self.contentSnapshotProvider = contentSnapshopProvider self.lengthProvider = lengthProvider self.invalidationHandler = invalidationHandler self.locationTransformer = locationTransformer @@ -73,7 +100,8 @@ public final class TreeSitterClient { configuration: .init( versionedContent: versionedContent, provider: validatorProvider - ) + ), + isolation: MainActor.shared ) private let layerTree: BackgroundingLanguageLayerTree @@ -115,9 +143,9 @@ public final class TreeSitterClient { /// Inform the client that calls to `languageConfiguration` may change. public func languageConfigurationChanged(for name: String) { - let content = maximumProcessedContent + let content = maximumProcessedContentSnapshot - layerTree.languageConfigurationChanged(for: name, content: content) { result in + layerTree.languageConfigurationChanged(for: name, content: content, isolation: MainActor.shared) { result in do { let invalidated = try result.get() @@ -129,21 +157,30 @@ public final class TreeSitterClient { } private var maximumProcessedContent: LanguageLayer.Content { - configuration.contentProvider(rangeProcessor.maximumProcessedLocation ?? 0) + if let content = configuration.contentProvider?(rangeProcessor.maximumProcessedLocation ?? 0) { + return content + } + + return maximumProcessedContentSnapshot.content + } + + private var maximumProcessedContentSnapshot: LanguageLayer.ContentSnapshot { + configuration.contentSnapshotProvider(rangeProcessor.maximumProcessedLocation ?? 0) } } +@available(macOS 13.0, macCatalyst 16.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) extension TreeSitterClient { private var hasPendingChanges: Bool { rangeProcessor.hasPendingChanges } - private func didChange(_ mutation: RangeMutation, completion: @MainActor @escaping () -> Void) { + private func didChange(_ mutation: RangeMutation, completion: @escaping () -> Void) { let limit = mutation.postApplyLimit - let content = configuration.contentProvider(limit) + let content = configuration.contentSnapshotProvider(limit) - layerTree.didChangeContent(content, in: mutation.range, delta: mutation.delta, completion: { invalidated in + layerTree.didChangeContent(content, in: mutation.range, delta: mutation.delta, isolation: MainActor.shared, completion: { invalidated in completion() self.handleInvalidation(invalidated, sublayers: false) }) @@ -164,6 +201,7 @@ extension TreeSitterClient { } } +@available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) extension TreeSitterClient { private func resolveSublayers(in range: NSRange) -> Bool { guard self.canAttemptSynchronousAccess(in: .range(range)) else { @@ -190,10 +228,10 @@ extension TreeSitterClient { private func resolveSublayers(in range: NSRange) async { let set = IndexSet(integersIn: range) - let content = self.maximumProcessedContent + let content = self.maximumProcessedContentSnapshot do { - let invalidatedSet = try await self.layerTree.resolveSublayers(with: content, in: set) + let invalidatedSet = try await self.layerTree.resolveSublayers(with: content, in: set, isolation: MainActor.shared) self.handleInvalidation(invalidatedSet, sublayers: false) } catch { @@ -239,6 +277,7 @@ extension TreeSitterClient { } } +@available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) extension TreeSitterClient { @MainActor public struct ClientQueryParams { @@ -284,7 +323,7 @@ extension TreeSitterClient { } private func validateSublayers(in set: IndexSet) { - sublayerValidator.validate(.set(set)) + sublayerValidator.validate(.set(set), isolation: MainActor.shared) } private func executeQuery(_ clientQuery: ClientQuery) async throws -> some Sequence { @@ -294,12 +333,12 @@ extension TreeSitterClient { validateSublayers(in: clientQuery.params.indexSet) - let matches = try await layerTree.executeQuery(clientQuery.query, in: clientQuery.params.indexSet) + let matches = try await layerTree.executeQuery(clientQuery.query, in: clientQuery.params.indexSet, isolation: MainActor.shared) return matches.resolve(with: .init(textProvider: clientQuery.params.textProvider)) } - public var highlightsProvider: HybridThrowingValueProvider { + public var highlightsProvider: HighlightsProvider { .init( rangeProcessor: rangeProcessor, inputTransformer: { ($0.maxLocation, $0.mode) }, @@ -320,10 +359,11 @@ extension TreeSitterClient { } } +@available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) extension TreeSitterClient { /// Execute a standard highlights.scm query. public func highlights(in set: IndexSet, provider: @escaping TextProvider, mode: RangeFillMode = .required) async throws -> [NamedRange] { - try await highlightsProvider.mainActorAsync(.init(indexSet: set, textProvider: provider, mode: mode)) + try await highlightsProvider.async(.init(indexSet: set, textProvider: provider, mode: mode)) } /// Execute a standard highlights.scm query. @@ -333,6 +373,6 @@ extension TreeSitterClient { /// Execute a standard highlights.scm query. public func highlights(in range: NSRange, provider: @escaping TextProvider, mode: RangeFillMode = .required) async throws -> [NamedRange] { - try await highlightsProvider.mainActorAsync(.init(range: range, textProvider: provider, mode: mode)) + try await highlightsProvider.async(.init(range: range, textProvider: provider, mode: mode)) } } diff --git a/Tests/RangeStateTests/RangeProcessorTests.swift b/Tests/RangeStateTests/RangeProcessorTests.swift index 4f9a06f..d7350e9 100644 --- a/Tests/RangeStateTests/RangeProcessorTests.swift +++ b/Tests/RangeStateTests/RangeProcessorTests.swift @@ -9,7 +9,7 @@ final class MockChangeHandler { var changeCompleted: @MainActor () -> Void = { } - func handleChange(_ mutation: RangeMutation, completion: @MainActor @escaping () -> Void) { + func handleChange(_ mutation: RangeMutation, completion: @escaping () -> Void) { mutations.append(mutation) changeCompleted() diff --git a/Tests/RangeStateTests/SinglePhaseRangeValidatorTests.swift b/Tests/RangeStateTests/SinglePhaseRangeValidatorTests.swift index 3c37536..c407395 100644 --- a/Tests/RangeStateTests/SinglePhaseRangeValidatorTests.swift +++ b/Tests/RangeStateTests/SinglePhaseRangeValidatorTests.swift @@ -16,7 +16,7 @@ final class SinglePhaseRangeValidatorTests: XCTestCase { return .success($0.value) }, - asyncValue: { contentRange, _ in + asyncValue: { _, contentRange in return .success(contentRange.value) }) @@ -46,7 +46,7 @@ final class SinglePhaseRangeValidatorTests: XCTestCase { return .success($0.value) }, - asyncValue: { contentRange, _ in + asyncValue: { _, contentRange in return .success(contentRange.value) }) @@ -76,7 +76,7 @@ final class SinglePhaseRangeValidatorTests: XCTestCase { return .success($0.value) }, - asyncValue: { contentRange, _ in + asyncValue: { _, contentRange in return .success(contentRange.value) } ) @@ -95,4 +95,34 @@ final class SinglePhaseRangeValidatorTests: XCTestCase { content.string = "ab" validator.contentChanged(in: NSRange(2..<3), delta: -1) } + + @MainActor + func testContentAddedAtEndAsync() async { + let validationExp = expectation(description: "validation") + + let content = StringContent(string: "abc") + let provider = StringValidator.Provider( + syncValue: { _ in + return nil + }, + asyncValue: { _, contentRange in + validationExp.fulfill() + + return .success(contentRange.value) + }) + + let validator = StringValidator( + configuration: .init( + versionedContent: content, + provider: provider + ) + ) + + validator.validate(.all) + + await fulfillment(of: [validationExp], timeout: 1.0) + + content.string = "abcd" + validator.contentChanged(in: NSRange(3..<3), delta: 1) + } } diff --git a/Tests/TreeSitterClientTests/TreeSitterClientTests.swift b/Tests/TreeSitterClientTests/TreeSitterClientTests.swift index 37caf25..ce5b7f7 100644 --- a/Tests/TreeSitterClientTests/TreeSitterClientTests.swift +++ b/Tests/TreeSitterClientTests/TreeSitterClientTests.swift @@ -5,6 +5,7 @@ import SwiftTreeSitter import TreeSitterClient import NeonTestsTreeSitterSwift +@available(macOS 13.0, macCatalyst 16.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) final class TreeSitterClientTests: XCTestCase { @MainActor func testSynchronousQuery() throws { @@ -23,13 +24,13 @@ final class TreeSitterClientTests: XCTestCase { let source = """ func main() { - print("hello!") + print("hello!") } """ let clientConfig = TreeSitterClient.Configuration( languageProvider: { _ in nil }, - contentProvider: { _ in .init(string: source) }, + contentSnapshopProvider: { _ in .init(string: source) }, lengthProvider: { source.utf16.count }, invalidationHandler: { _ in }, locationTransformer: { _ in nil } @@ -142,7 +143,7 @@ func main() { //let content = """ //func main() { print("hello" } //""" -// +// // let queryText = """ //("func" @keyword.function) //""" @@ -164,7 +165,7 @@ func main() { // client.didChangeContent(to: content, in: .zero, delta: content.utf16.count, limit: content.utf16.count) // // wait(for: [queryExpectation], timeout: 2.0) -// +// // switch result { // case .failure(.staleContent): // break