Merge pull request #307 from robUx4/utf8-406

Update utf8 library to latest 4.0.6 version

.gitignore

@@ -11,6 +11,7 @@
 *.pc
 Makefile
 .dirstamp
+.DS_Store
 
 [Dd]ebug/
 [Dd]ebugPublic/

src/lib/utf8-cpp/LICENSE

@@ -0,0 +1,23 @@
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

src/lib/utf8-cpp/source/utf8.h

@@ -28,6 +28,18 @@ DEALINGS IN THE SOFTWARE.
 #ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
 #define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
 
+/*
+To control the C++ language version used by the library, you can define UTF_CPP_CPLUSPLUS macro
+and set it to one of the values used by the __cplusplus predefined macro.
+
+For instance,
+    #define UTF_CPP_CPLUSPLUS 199711L
+will cause the UTF-8 CPP library to use only types and language features available in the C++ 98 standard.
+Some library features will be disabled.
+
+If you leave UTF_CPP_CPLUSPLUS undefined, it will be internally assigned to __cplusplus.
+*/
+
 #include "utf8/checked.h"
 #include "utf8/unchecked.h"
 

src/lib/utf8-cpp/source/utf8/checked.h

@@ -39,27 +39,28 @@ namespace utf8
 
     // Exceptions that may be thrown from the library functions.
     class invalid_code_point : public exception {
-        uint32_t cp;
+        utfchar32_t cp;
     public:
-        invalid_code_point(uint32_t codepoint) : cp(codepoint) {}
+        invalid_code_point(utfchar32_t codepoint) : cp(codepoint) {}
         virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid code point"; }
-        uint32_t code_point() const {return cp;}
+        utfchar32_t code_point() const {return cp;}
     };
 
     class invalid_utf8 : public exception {
-        uint8_t u8;
+        utfchar8_t u8;
     public:
-        invalid_utf8 (uint8_t u) : u8(u) {}
+        invalid_utf8 (utfchar8_t u) : u8(u) {}
+        invalid_utf8 (char c) : u8(static_cast<utfchar8_t>(c)) {}
         virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid UTF-8"; }
-        uint8_t utf8_octet() const {return u8;}
+        utfchar8_t utf8_octet() const {return u8;}
     };
 
     class invalid_utf16 : public exception {
-        uint16_t u16;
+        utfchar16_t u16;
     public:
-        invalid_utf16 (uint16_t u) : u16(u) {}
+        invalid_utf16 (utfchar16_t u) : u16(u) {}
         virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid UTF-16"; }
-        uint16_t utf16_word() const {return u16;}
+        utfchar16_t utf16_word() const {return u16;}
     };
 
     class not_enough_room : public exception {
@@ -70,33 +71,30 @@ namespace utf8
     /// The library API - functions intended to be called by the users
 
     template <typename octet_iterator>
-    octet_iterator append(uint32_t cp, octet_iterator result)
+    octet_iterator append(utfchar32_t cp, octet_iterator result)
     {
         if (!utf8::internal::is_code_point_valid(cp))
             throw invalid_code_point(cp);
 
-        if (cp < 0x80)                        // one octet
-            *(result++) = static_cast<uint8_t>(cp);
-        else if (cp < 0x800) {                // two octets
-            *(result++) = static_cast<uint8_t>((cp >> 6)            | 0xc0);
-            *(result++) = static_cast<uint8_t>((cp & 0x3f)          | 0x80);
-        }
-        else if (cp < 0x10000) {              // three octets
-            *(result++) = static_cast<uint8_t>((cp >> 12)           | 0xe0);
-            *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f)   | 0x80);
-            *(result++) = static_cast<uint8_t>((cp & 0x3f)          | 0x80);
-        }
-        else {                                // four octets
-            *(result++) = static_cast<uint8_t>((cp >> 18)           | 0xf0);
-            *(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f)  | 0x80);
-            *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f)   | 0x80);
-            *(result++) = static_cast<uint8_t>((cp & 0x3f)          | 0x80);
-        }
-        return result;
+        return internal::append(cp, result);
+    }
+
+    inline void append(utfchar32_t cp, std::string& s)
+    {
+        append(cp, std::back_inserter(s));
+    }
+
+    template <typename word_iterator>
+    word_iterator append16(utfchar32_t cp, word_iterator result)
+    {
+        if (!utf8::internal::is_code_point_valid(cp))
+            throw invalid_code_point(cp);
+
+        return internal::append16(cp, result);
     }
 
     template <typename octet_iterator, typename output_iterator>
-    output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement)
+    output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, utfchar32_t replacement)
     {
         while (start != end) {
             octet_iterator sequence_start = start;
@@ -131,14 +129,28 @@ namespace utf8
     template <typename octet_iterator, typename output_iterator>
     inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out)
     {
-        static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd);
+        static const utfchar32_t replacement_marker = utf8::internal::mask16(0xfffd);
         return utf8::replace_invalid(start, end, out, replacement_marker);
     }
 
+    inline std::string replace_invalid(const std::string& s, utfchar32_t replacement)
+    {
+        std::string result;
+        replace_invalid(s.begin(), s.end(), std::back_inserter(result), replacement);
+        return result;
+    }
+
+    inline std::string replace_invalid(const std::string& s)
+    {
+        std::string result;
+        replace_invalid(s.begin(), s.end(), std::back_inserter(result));
+        return result;
+    }
+
     template <typename octet_iterator>
-    uint32_t next(octet_iterator& it, octet_iterator end)
+    utfchar32_t next(octet_iterator& it, octet_iterator end)
     {
-        uint32_t cp = 0;
+        utfchar32_t cp = 0;
         internal::utf_error err_code = utf8::internal::validate_next(it, end, cp);
         switch (err_code) {
             case internal::UTF8_OK :
@@ -148,21 +160,31 @@ namespace utf8
             case internal::INVALID_LEAD :
             case internal::INCOMPLETE_SEQUENCE :
             case internal::OVERLONG_SEQUENCE :
-                throw invalid_utf8(*it);
+                throw invalid_utf8(static_cast<utfchar8_t>(*it));
             case internal::INVALID_CODE_POINT :
                 throw invalid_code_point(cp);
         }
         return cp;
     }
 
+    template <typename word_iterator>
+    utfchar32_t next16(word_iterator& it, word_iterator end)
+    {
+        utfchar32_t cp = 0;
+        internal::utf_error err_code = utf8::internal::validate_next16(it, end, cp);
+        if (err_code == internal::NOT_ENOUGH_ROOM)
+            throw not_enough_room();
+        return cp;
+    }
+
     template <typename octet_iterator>
-    uint32_t peek_next(octet_iterator it, octet_iterator end)
+    utfchar32_t peek_next(octet_iterator it, octet_iterator end)
     {
         return utf8::next(it, end);
     }
 
     template <typename octet_iterator>
-    uint32_t prior(octet_iterator& it, octet_iterator start)
+    utfchar32_t prior(octet_iterator& it, octet_iterator start)
     {
         // can't do much if it == start
         if (it == start)
@@ -205,23 +227,23 @@ namespace utf8
     octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
     {
         while (start != end) {
-            uint32_t cp = utf8::internal::mask16(*start++);
+            utfchar32_t cp = utf8::internal::mask16(*start++);
             // Take care of surrogate pairs first
             if (utf8::internal::is_lead_surrogate(cp)) {
                 if (start != end) {
-                    uint32_t trail_surrogate = utf8::internal::mask16(*start++);
+                    const utfchar32_t trail_surrogate = utf8::internal::mask16(*start++);
                     if (utf8::internal::is_trail_surrogate(trail_surrogate))
                         cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
                     else
-                        throw invalid_utf16(static_cast<uint16_t>(trail_surrogate));
+                        throw invalid_utf16(static_cast<utfchar16_t>(trail_surrogate));
                 }
                 else
-                    throw invalid_utf16(static_cast<uint16_t>(cp));
+                    throw invalid_utf16(static_cast<utfchar16_t>(cp));
 
             }
             // Lone trail surrogate
             else if (utf8::internal::is_trail_surrogate(cp))
-                throw invalid_utf16(static_cast<uint16_t>(cp));
+                throw invalid_utf16(static_cast<utfchar16_t>(cp));
 
             result = utf8::append(cp, result);
         }
@@ -232,13 +254,13 @@ namespace utf8
     u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
     {
         while (start < end) {
-            uint32_t cp = utf8::next(start, end);
+            const utfchar32_t cp = utf8::next(start, end);
             if (cp > 0xffff) { //make a surrogate pair
-                *result++ = static_cast<uint16_t>((cp >> 10)   + internal::LEAD_OFFSET);
-                *result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
+                *result++ = static_cast<utfchar16_t>((cp >> 10)   + internal::LEAD_OFFSET);
+                *result++ = static_cast<utfchar16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
             }
             else
-                *result++ = static_cast<uint16_t>(cp);
+                *result++ = static_cast<utfchar16_t>(cp);
         }
         return result;
     }
@@ -268,9 +290,9 @@ namespace utf8
       octet_iterator range_start;
       octet_iterator range_end;
       public:
-      typedef uint32_t value_type;
-      typedef uint32_t* pointer;
-      typedef uint32_t& reference;
+      typedef utfchar32_t value_type;
+      typedef utfchar32_t* pointer;
+      typedef utfchar32_t& reference;
       typedef std::ptrdiff_t difference_type;
       typedef std::bidirectional_iterator_tag iterator_category;
       iterator () {}
@@ -284,7 +306,7 @@ namespace utf8
       }
       // the default "big three" are OK
       octet_iterator base () const { return it; }
-      uint32_t operator * () const
+      utfchar32_t operator * () const
       {
           octet_iterator temp = it;
           return utf8::next(temp, range_end);
@@ -325,7 +347,9 @@ namespace utf8
 
 } // namespace utf8
 
-#if UTF_CPP_CPLUSPLUS >= 201703L // C++ 17 or later
+#if UTF_CPP_CPLUSPLUS >= 202002L // C++ 20 or later
+#include "cpp20.h"
+#elif UTF_CPP_CPLUSPLUS >= 201703L // C++ 17 or later
 #include "cpp17.h"
 #elif UTF_CPP_CPLUSPLUS >= 201103L // C++ 11 or later
 #include "cpp11.h"