Prioritize BOM check in encoding detection

Author: 1024jp

CHANGELOG.md

@@ -11,7 +11,7 @@
 
 ### Improvements
 
-- Improve the algorithm to scan encoding decralations.
+- Improve the algorithm of text encoding detection.
 - Update the C++ syntax to include the .cu and .cuh filename extensions.
 
 

Packages/EditorCore/Sources/FileEncoding/String+Encoding.swift

@@ -38,7 +38,7 @@ public extension String {
 
     struct DetectionOptions: Sendable {
 
-        /// The list of encodings to test the encoding.
+        /// The list of encodings to test decoding.
         public var candidates: [String.Encoding]
 
         /// The text encoding read from the file's extended attributes.
@@ -115,14 +115,25 @@ public extension String {
 
 extension String {
 
-    /// Reads string from data by detecting the text encoding automatically.
+    /// Reads string from data by detecting the text encoding based on the detection options.
     ///
     /// - Parameters:
     ///   - data: The data to encode.
     ///   - options: The options for encoding detection.
     /// - Returns: The decoded string and used encoding.
+    /// - Throws: `CocoaError(.fileReadUnknownStringEncoding)`
     static func string(data: Data, options: String.DetectionOptions) throws(CocoaError) -> (String, String.Encoding) {
 
+        // check BOMs
+        for bom in Unicode.BOM.allCases {
+            if options.candidates.contains(bom.encoding),
+               data.starts(with: bom.sequence),
+               let string = String(bomCapableData: data, encoding: bom.encoding)
+            {
+                return (string, bom.encoding)
+            }
+        }
+        
         // try interpreting with xattr encoding
         if let xattrEncoding = options.xattrEncoding {
             // just trust xattr encoding if the content is empty
@@ -140,45 +151,11 @@ extension String {
             return (string, encoding)
         }
 
-        // detect encoding from data
-        var usedEncoding: String.Encoding?
-        let string = try String(data: data, suggestedEncodings: options.candidates, usedEncoding: &usedEncoding)
-        if let encoding = usedEncoding {
-            return (string, encoding)
-        }
-        
-        throw CocoaError(.fileReadUnknownStringEncoding)
-    }
-    
-    
-    /// Returns a  `String` initialized by converting given `data` into Unicode characters using an intelligent encoding detection.
-    ///
-    /// - Parameters:
-    ///   - data: The data object containing the string data.
-    ///   - suggestedEncodings: The prioritized list of encoding candidates.
-    ///   - usedEncoding: The encoding used to interpret the data.
-    /// - Throws: `CocoaError(.fileReadUnknownStringEncoding)`
-    init(data: Data, suggestedEncodings: [String.Encoding], usedEncoding: inout String.Encoding?) throws(CocoaError) {
-        
-        // check BOMs
-        for bom in Unicode.BOM.allCases {
-            guard
-                data.starts(with: bom.sequence),
-                let string = String(bomCapableData: data, encoding: bom.encoding)
-            else { continue }
-            
-            usedEncoding = bom.encoding
-            self = string
-            return
-        }
-        
-        // try encodings in order from the top of the encoding list
-        for encoding in suggestedEncodings {
-            guard let string = String(data: data, encoding: encoding) else { continue }
-            
-            usedEncoding = encoding
-            self = string
-            return
+        // try applying encodings in order from the top of the candidates
+        for encoding in options.candidates {
+            if let string = String(data: data, encoding: encoding) {
+                return (string, encoding)
+            }
         }
 
         throw CocoaError(.fileReadUnknownStringEncoding)

Packages/EditorCore/Tests/FileEncodingTests/EncodingDetectionTests.swift

@@ -46,8 +46,7 @@ struct EncodingDetectionTests {
         }
         #expect(String(bomCapableData: data, encoding: .utf8) == "0")
 
-        var encoding: String.Encoding?
-        let string = try self.encodedStringForFileName("UTF-8 BOM", usedEncoding: &encoding)
+        let (string, encoding) = try self.encodedStringForFileName("UTF-8 BOM")
 
         #expect(string == "0")
         #expect(encoding == .utf8)
@@ -89,8 +88,7 @@ struct EncodingDetectionTests {
 
     @Test func utf16() throws {
 
-        var encoding: String.Encoding?
-        let string = try self.encodedStringForFileName("UTF-16", usedEncoding: &encoding)
+        let (string, encoding) = try self.encodedStringForFileName("UTF-16")
 
         #expect(string == "0")
         #expect(encoding == .utf16)
@@ -99,8 +97,7 @@ struct EncodingDetectionTests {
 
     @Test func utf32() throws {
 
-        var encoding: String.Encoding?
-        let string = try self.encodedStringForFileName("UTF-32", usedEncoding: &encoding)
+        let (string, encoding) = try self.encodedStringForFileName("UTF-32")
 
         #expect(string == "0")
         #expect(encoding == .utf32)
@@ -112,52 +109,42 @@ struct EncodingDetectionTests {
         let data = try self.dataForFileName("ISO 2022-JP")
         let encodings: [String.Encoding] = [.iso2022JP, .utf16]
 
-        var encoding: String.Encoding?
-        let string = try String(data: data, suggestedEncodings: encodings, usedEncoding: &encoding)
+        let (string, encoding) = try String.string(data: data, options: .init(candidates: encodings))
 
         #expect(string == "dog犬")
         #expect(encoding == .iso2022JP)
     }
 
 
-    @Test func utf8() throws {
+    @Test func emptySuggestion() throws {
 
         let data = try self.dataForFileName("UTF-8")
 
-        var encoding: String.Encoding?
         #expect(throws: CocoaError(.fileReadUnknownStringEncoding)) {
-            try String(data: data, suggestedEncodings: [], usedEncoding: &encoding)
+            try String.string(data: data, options: .init(candidates: []))
         }
-        #expect(encoding == nil)
     }
 
 
-    @Test func suggestedEncoding() throws {
+    @Test func utf8() throws {
 
         let data = try self.dataForFileName("UTF-8")
 
-        var encoding: String.Encoding?
         let invalidEncoding = String.Encoding(cfEncoding: kCFStringEncodingInvalidId)
-        let string = try String(data: data, suggestedEncodings: [invalidEncoding, .utf8], usedEncoding: &encoding)
+        let (string, encoding) = try String.string(data: data, options: .init(candidates: [invalidEncoding, .utf8, .utf16]))
 
         #expect(string == "0")
         #expect(encoding == .utf8)
     }
 
 
-    @Test func emptyData() {
+    @Test func emptyData() throws {
 
         let data = Data()
+        let (string, encoding) = try String.string(data: data, options: .init(candidates: [.shiftJIS]))
 
-        var encoding: String.Encoding?
-        var string: String?
-        
-        #expect(throws: CocoaError(.fileReadUnknownStringEncoding)) {
-            string = try String(data: data, suggestedEncodings: [], usedEncoding: &encoding)
-        }
-        
-        #expect(string == nil)
-        #expect(encoding == nil)
+        #expect(string.isEmpty)
+        #expect(encoding == .shiftJIS)
         #expect(!data.starts(with: Unicode.BOM.utf8.sequence))
     }
 
@@ -291,11 +278,19 @@ private extension String.Encoding {
 
 private extension EncodingDetectionTests {
 
-    func encodedStringForFileName(_ fileName: String, usedEncoding: inout String.Encoding?) throws -> String {
-        
-        let data = try self.dataForFileName(fileName)
-        
-        return try String(data: data, suggestedEncodings: [], usedEncoding: &usedEncoding)
+    func encodedStringForFileName(_ fileName: String) throws -> (String, String.Encoding) {
+        
+        try String.string(
+            data: try self.dataForFileName(fileName),
+            options: .init(candidates: [
+                .utf8,
+                .utf16,
+                .utf16BigEndian,
+                .utf16LittleEndian,
+                .utf32,
+                .utf32BigEndian,
+                .utf32LittleEndian,
+            ]))
     }