Objective-C marks and non-ASCII characters

Source/SourceKittenFramework/SourceDeclaration.swift

@@ -11,10 +11,13 @@ public func insertMarks(declarations: [SourceDeclaration], limitRange: NSRange?
     guard let path = declarations.first?.location.file, let file = File(path: path) else {
         fatalError("can't extract marks without a file.")
     }
-    let currentMarks = file.contents.pragmaMarks(path, excludeRanges: declarations.map({ $0.range }), limitRange: limitRange)
+    let currentMarks = file.contents.pragmaMarks(path, excludeRanges: declarations.map({
+        file.contents.byteRangeToNSRange(start: $0.range.location, length: $0.range.length) ?? NSRange()
+    }), limitRange: limitRange)
     let newDeclarations: [SourceDeclaration] = declarations.map { declaration in
         var varDeclaration = declaration
-        varDeclaration.children = insertMarks(declaration.children, limitRange: declaration.range)
+        let range = file.contents.byteRangeToNSRange(start: declaration.range.location, length: declaration.range.length)
+        varDeclaration.children = insertMarks(declaration.children, limitRange: range)
         return varDeclaration
     }
     return (newDeclarations + currentMarks).sort()

Source/SourceKittenFramework/String+SourceKitten.swift

@@ -87,6 +87,24 @@ extension NSString {
         }
     }
 
+    /**
+    Converts an `NSRange` suitable for filtering `self` as an
+    `NSString` to a range of byte offsets in `self`.
+
+    - parameter start: Starting character index in the string.
+    - parameter length: Number of characters to include in range.
+
+    - returns: An equivalent `NSRange`.
+    */
+    public func NSRangeToByteRange(start start: Int, length: Int) -> NSRange? {
+        let string = self as String
+        return string.byteOffsetAtIndex(start).flatMap { stringStart in
+            return string.byteOffsetAtIndex(start + length).map { stringEnd in
+                return NSRange(location: stringStart, length: stringEnd - stringStart)
+            }
+        }
+    }
+
     /**
     Returns a substring with the provided byte range.
 
@@ -181,6 +199,17 @@ extension String {
         return utf8.startIndex.advancedBy(offset).samePositionIn(utf16).map(utf16.startIndex.distanceTo)
     }
 
+    /**
+     Byte offset equivalent to UTF16 index.
+
+     - parameter index: UTF16 index.
+
+     - returns: Byte offset, if any.
+     */
+    private func byteOffsetAtIndex(index: Int) -> Int? {
+        return utf16.startIndex.advancedBy(index).samePositionIn(utf8).map(utf8.startIndex.distanceTo)
+    }
+
     /// Returns the `#pragma mark`s in the string.
     /// Just the content; no leading dashes or leading `#pragma mark`.
     public func pragmaMarks(filename: String, excludeRanges: [NSRange], limitRange: NSRange?) -> [SourceDeclaration] {
@@ -205,9 +234,13 @@ extension String {
             if markString.isEmpty {
                 return nil
             }
+            let markByteRange = self.NSRangeToByteRange(start: markRange.location, length: markRange.length)
+            if markByteRange == nil {
+                return nil
+            }
             let location = SourceLocation(file: filename,
-                line: UInt32((self as NSString).lineRangeWithByteRange(start: markRange.location, length: 0)!.start),
-                column: 1, offset: UInt32(markRange.location))
+                line: UInt32((self as NSString).lineRangeWithByteRange(start: markByteRange!.location, length: 0)!.start),
+                column: 1, offset: UInt32(markByteRange!.location))
             return SourceDeclaration(type: .Mark, location: location, extent: (location, location), name: markString,
                 usr: nil, declaration: nil, documentation: nil, commentBody: nil, children: [])
         }

Source/SourceKittenFrameworkTests/ClangTranslationUnitTests.swift

@@ -35,6 +35,15 @@ class ClangTranslationUnitTests: XCTestCase {
         let comparisonString = (tu.description + "\n").stringByReplacingOccurrencesOfString(escapedFixturesDirectory, withString: "")
         compareJSONStringWithFixturesName("Musician", jsonString: comparisonString)
     }
+
+    func testUnicodeInObjectiveCDocs() {
+        let headerFiles = [fixturesDirectory + "SuperScript.h"]
+        let compilerArguments = ["-x", "objective-c", "-isysroot", sdkPath(), "-I", fixturesDirectory]
+        let tu = ClangTranslationUnit(headerFiles: headerFiles, compilerArguments: compilerArguments)
+        let escapedFixturesDirectory = fixturesDirectory.stringByReplacingOccurrencesOfString("/", withString: "\\/")
+        let comparisonString = (tu.description + "\n").stringByReplacingOccurrencesOfString(escapedFixturesDirectory, withString: "")
+        compareJSONStringWithFixturesName("SuperScript", jsonString: comparisonString)
+    }
 
     func testRealmObjectiveCDocs() {
         let headerFiles = [fixturesDirectory + "/Realm/Realm.h"]

Source/SourceKittenFrameworkTests/Fixtures/SuperScript.h

@@ -0,0 +1,15 @@
+#import <Foundation/Foundation.h>
+
+/**
+ ᴬ 	ᴮ 		ᴰ 	ᴱ 		ᴳ 	ᴴ 	ᴵ 	ᴶ 	ᴷ 	ᴸ 	ᴹ 	ᴺ 	ᴼ 	ᴾ 		ᴿ 		ᵀ 	ᵁ 	ⱽ 	ᵂ 		
+ */
+@interface SuperScript : NSObject
+
+#pragma mark ᵃ	ᵇ	ᶜ	ᵈ	ᵉ	ᶠ	ᵍ	ʰ	ⁱ	ʲ	ᵏ	ˡ	ᵐ	ⁿ	ᵒ	ᵖ		ʳ	ˢ	ᵗ	ᵘ	ᵛ	ʷ	ˣ	ʸ	ᶻ
+
+/**
+  	ᵝ 	ᵞ 	ᵟ 	ᵋ 			ᶿ 	ᶥ 											ᶹ 	ᵠ 	ᵡ
+ */
+- (IBAction)superSize:(id)sender;
+
+@end

Source/SourceKittenFrameworkTests/Fixtures/SuperScript.json

@@ -0,0 +1,49 @@
+[
+  {
+    "SuperScript.h" : {
+      "key.substructure" : [
+        {
+          "key.kind" : "sourcekitten.source.lang.objc.decl.class",
+          "key.parsed_declaration" : "@interface SuperScript : NSObject",
+          "key.doc.comment" : "ᴬ \tᴮ \t\tᴰ \tᴱ \t\tᴳ \tᴴ \tᴵ \tᴶ \tᴷ \tᴸ \tᴹ \tᴺ \tᴼ \tᴾ \t\tᴿ \t\tᵀ \tᵁ \tⱽ \tᵂ",
+          "key.doc.line" : 6,
+          "key.name" : "SuperScript",
+          "key.doc.column" : 12,
+          "key.parsed_scope.end" : 15,
+          "key.usr" : "c:objc(cs)SuperScript",
+          "key.doc.file" : "SuperScript.h",
+          "key.doc.full_as_xml" : "",
+          "key.substructure" : [
+            {
+              "key.kind" : "sourcekitten.source.lang.objc.mark",
+              "key.doc.file" : "SuperScript.h",
+              "key.doc.line" : 8,
+              "key.name" : "ᵃ\tᵇ\tᶜ\tᵈ\tᵉ\tᶠ\tᵍ\tʰ\tⁱ\tʲ\tᵏ\tˡ\tᵐ\tⁿ\tᵒ\tᵖ\t\tʳ\tˢ\tᵗ\tᵘ\tᵛ\tʷ\tˣ\tʸ\tᶻ",
+              "key.parsed_scope.start" : 8,
+              "key.doc.column" : 1,
+              "key.parsed_scope.end" : 8,
+              "key.filepath" : "SuperScript.h"
+            },
+            {
+              "key.kind" : "sourcekitten.source.lang.objc.decl.method.instance",
+              "key.doc.file" : "SuperScript.h",
+              "key.parsed_declaration" : "- (void)superSize:(id)sender;",
+              "key.doc.comment" : "ᵝ \tᵞ \tᵟ \tᵋ \t\t\tᶿ \tᶥ \t\t\t\t\t\t\t\t\t\t\tᶹ \tᵠ \tᵡ",
+              "key.doc.full_as_xml" : "",
+              "key.doc.line" : 13,
+              "key.name" : "-superSize:",
+              "key.parsed_scope.start" : 13,
+              "key.doc.column" : 13,
+              "key.parsed_scope.end" : 13,
+              "key.filepath" : "SuperScript.h",
+              "key.usr" : "c:objc(cs)SuperScript(im)superSize:"
+            }
+          ],
+          "key.parsed_scope.start" : 6,
+          "key.filepath" : "SuperScript.h"
+        }
+      ],
+      "key.diagnostic_stage" : ""
+    }
+  }
+]