Render non-BMP CJKV characters locally

CHANGELOG.md

@@ -4,14 +4,14 @@
 
 - Add constants `MAX_TILE_ZOOM = 25` and `MIN_TILE_ZOOM = 0` as maximum and minimum world tile zoom (Z) values; replace hardcoded instances with those constants.
 - Add functions `isInBoundsForTileZoomXY` and `isInBoundsForZoomLngLat` to check whether a tile ZXY or a zoom+LngLat is in the world bounds; use `MAX_TILE_ZOOM` and `MIN_TILE_ZOOM` in those checks; replace existing hardcoded checks with those functions.
+- Render uncommon Chinese, Japanese, Korean, and Vietnamese characters. Prefer local glyph rendering for all CJKV characters, not just those in the CJK Unified Ideographs, Hiragana, Katakana, and Hangul Syllables blocks. Request other characters beyond U+FFFF from the server instead of throwing an error. ([#4550](https://github.com/maplibre/maplibre-gl-js/pull/4550), [#4560](https://github.com/maplibre/maplibre-gl-js/pull/4560))
 - _...Add new stuff here..._
 
 ### 🐞 Bug fixes
 
 - Fix right-to-left layout of labels that contain characters in the Arabic Extended-B code block. ([#4536](https://github.com/maplibre/maplibre-gl-js/pull/4536))
 - Fix 3D map freezing when camera is adjusted against map bounds. ([#4537](https://github.com/maplibre/maplibre-gl-js/issues/4537))
 - Fix `getStyle()` to return a clone so the object cannot be internally changed ([#4488](https://github.com/maplibre/maplibre-gl-js/issues/4488))
-- Prefer local glyph rendering for all CJKV characters, not just those in the CJK Unified Ideographs, Hiragana, Katakana, and Hangul Syllables blocks. ([#4560](https://github.com/maplibre/maplibre-gl-js/pull/4560)))
 - - _...Add new stuff here..._
 
 ## 4.5.2

src/data/bucket/symbol_bucket.ts

@@ -424,12 +424,12 @@ export class SymbolBucket implements Bucket {
         allowVerticalPlacement: boolean,
         doesAllowVerticalWritingMode: boolean) {
 
-        for (let i = 0; i < text.length; i++) {
-            stack[text.charCodeAt(i)] = true;
+        for (const char of text) {
+            stack[char.codePointAt(0)] = true;
             if ((textAlongLine || allowVerticalPlacement) && doesAllowVerticalWritingMode) {
-                const verticalChar = verticalizedCharacterMap[text.charAt(i)];
+                const verticalChar = verticalizedCharacterMap[char];
                 if (verticalChar) {
-                    stack[verticalChar.charCodeAt(0)] = true;
+                    stack[verticalChar.codePointAt(0)] = true;
                 }
             }
         }

src/render/glyph_manager.test.ts

@@ -66,6 +66,18 @@ describe('GlyphManager', () => {
         expect(returnedGlyphs['Arial Unicode MS'][0x5e73]).toBeNull(); // The fixture returns a PBF without the glyph we requested
     });
 
+    test('GlyphManager requests remote non-BMP, non-CJK PBF', async () => {
+        jest.spyOn(GlyphManager, 'loadGlyphRange').mockImplementation((_stack, _range, _urlTemplate, _transform) => {
+            return Promise.resolve(GLYPHS);
+        });
+
+        const manager = createGlyphManager();
+
+        // Request Egyptian hieroglyph 𓃰
+        const returnedGlyphs = await manager.getGlyphs({'Arial Unicode MS': [0x1e0f0]});
+        expect(returnedGlyphs['Arial Unicode MS'][0x1e0f0]).toBeNull(); // The fixture returns a PBF without the glyph we requested
+    });
+
     test('GlyphManager does not cache CJK chars that should be rendered locally', async () => {
         jest.spyOn(GlyphManager, 'loadGlyphRange').mockImplementation((_stack, range, _urlTemplate, _transform) => {
             const overlappingGlyphs = {};
@@ -98,6 +110,14 @@ describe('GlyphManager', () => {
         expect(returnedGlyphs['Arial Unicode MS'][0x5e73].metrics.advance).toBe(0.5);
     });
 
+    test('GlyphManager generates non-BMP CJK PBF locally', async () => {
+        const manager = createGlyphManager('sans-serif');
+
+        // Chinese character biáng 𰻞
+        const returnedGlyphs = await manager.getGlyphs({'Arial Unicode MS': [0x30EDE]});
+        expect(returnedGlyphs['Arial Unicode MS'][0x30EDE].metrics.advance).toBe(1);
+    });
+
     test('GlyphManager generates Katakana PBF locally', async () => {
         const manager = createGlyphManager('sans-serif');
 

src/render/glyph_manager.ts

@@ -1,6 +1,7 @@
 import {loadGlyphRange} from '../style/load_glyph_range';
 
 import TinySDF from '@mapbox/tiny-sdf';
+import {charAllowsIdeographicBreaking} from '../util/script_detection';
 import {AlphaImage} from '../util/image';
 
 import type {StyleGlyph} from '../style/style_glyph';
@@ -91,10 +92,6 @@ export class GlyphManager {
         }
 
         const range = Math.floor(id / 256);
-        if (range * 256 > 65535) {
-            throw new Error('glyphs > 65535 not supported');
-        }
-
         if (entry.ranges[range]) {
             return {stack, id, glyph};
         }
@@ -118,15 +115,21 @@ export class GlyphManager {
         return {stack, id, glyph: response[id] || null};
     }
 
+    /**
+     * Returns whether the given codepoint should be rendered locally.
+     *
+     * Local rendering is preferred for Unicode code blocks that represent writing systems for
+     * which TinySDF produces optimal results and greatly reduces bandwidth consumption. In
+     * general, TinySDF is best for any writing system typically set in a monospaced font. With
+     * more than 99,000 codepoints accessed essentially at random, Hanzi/Kanji/Hanja (from the CJK
+     * Unified Ideographs blocks) is the canonical example of wasteful bandwidth consumption when
+     * rendered remotely. For visual consistency within CJKV text, even relatively small CJKV and
+     * other siniform code blocks prefer local rendering.
+     */
     _doesCharSupportLocalGlyph(id: number): boolean {
-        // The CJK Unified Ideographs blocks and Hangul Syllables blocks are
-        // spread across many glyph PBFs and are typically accessed very
-        // randomly. Preferring local rendering for these blocks reduces
-        // wasteful bandwidth consumption. For visual consistency within CJKV
-        // text, also include any other CJKV or siniform ideograph or hangul,
-        // hiragana, or katakana character.
         return !!this.localIdeographFontFamily &&
-            /\p{Ideo}|\p{sc=Hang}|\p{sc=Hira}|\p{sc=Kana}/u.test(String.fromCodePoint(id));
+            (/\p{Ideo}|\p{sc=Hang}|\p{sc=Hira}|\p{sc=Kana}/u.test(String.fromCodePoint(id)) ||
+             charAllowsIdeographicBreaking(id));
     }
 
     _tinySDF(entry: Entry, stack: string, id: number): StyleGlyph {
@@ -163,7 +166,7 @@ export class GlyphManager {
             });
         }
 
-        const char = tinySDF.draw(String.fromCharCode(id));
+        const char = tinySDF.draw(String.fromCodePoint(id));
 
         /**
          * TinySDF's "top" is the distance from the alphabetic baseline to the top of the glyph.

src/symbol/shaping.test.ts

@@ -1,7 +1,119 @@
-import {type PositionedIcon, type Box,  type Shaping, applyTextFit, shapeIcon, fitIconToText} from './shaping';
+import {type PositionedIcon, type Box, type Shaping, SectionOptions, TaggedString, determineLineBreaks, applyTextFit, shapeIcon, fitIconToText} from './shaping';
 import {ImagePosition} from '../render/image_atlas';
+import type {StyleGlyph} from '../style/style_glyph';
 import {StyleImage, TextFit} from '../style/style_image';
 
+describe('TaggedString', () => {
+    describe('length', () => {
+        test('counts a surrogate pair as a single character', () => {
+            const tagged = new TaggedString();
+            tagged.text = '茹𦨭';
+            expect(tagged.length()).toBe(2);
+        });
+    });
+
+    describe('trim', () => {
+        test('turns a whitespace-only string into the empty string', () => {
+            const tagged = new TaggedString();
+            tagged.text = '  \t    \v ';
+            tagged.sections = [new SectionOptions()];
+            tagged.sectionIndex = Array(9).fill(0);
+            tagged.trim();
+            expect(tagged.text).toBe('');
+            expect(tagged.sectionIndex).toHaveLength(0);
+        });
+
+        test('trims whitespace around a surrogate pair', () => {
+            const tagged = new TaggedString();
+            tagged.text = ' 茹𦨭 ';
+            tagged.sections = [new SectionOptions()];
+            tagged.sectionIndex = Array(4).fill(0);
+            tagged.trim();
+            expect(tagged.text).toBe('茹𦨭');
+            expect(tagged.sectionIndex).toHaveLength(2);
+        });
+    });
+
+    describe('substring', () => {
+        test('avoids splitting a surrogate pair', () => {
+            const tagged = new TaggedString();
+            tagged.text = '𰻞𰻞麵𪚥𪚥';
+            tagged.sections = [new SectionOptions()];
+            tagged.sectionIndex = Array(5).fill(0);
+            expect(tagged.substring(0, 1).text).toBe('𰻞');
+            expect(tagged.substring(0, 1).sectionIndex).toEqual([0]);
+            expect(tagged.substring(0, 2).text).toBe('𰻞𰻞');
+            expect(tagged.substring(0, 2).sectionIndex).toEqual([0, 0]);
+            expect(tagged.substring(1, 2).text).toBe('𰻞');
+            expect(tagged.substring(1, 2).sectionIndex).toEqual([0]);
+            expect(tagged.substring(1, 3).text).toBe('𰻞麵');
+            expect(tagged.substring(1, 3).sectionIndex).toEqual([0, 0]);
+            expect(tagged.substring(2, 5).text).toBe('麵𪚥𪚥');
+            expect(tagged.substring(2, 5).sectionIndex).toEqual([0, 0, 0]);
+        });
+    });
+
+    describe('codeUnitIndex', () => {
+        test('splits surrogate pairs', () => {
+            const tagged = new TaggedString();
+            tagged.text = '𰻞𰻞麵𪚥𪚥';
+            expect(tagged.toCodeUnitIndex(0)).toBe(0);
+            expect(tagged.toCodeUnitIndex(1)).toBe(2);
+            expect(tagged.toCodeUnitIndex(2)).toBe(4);
+            expect(tagged.toCodeUnitIndex(3)).toBe(5);
+            expect(tagged.toCodeUnitIndex(4)).toBe(7);
+            expect(tagged.toCodeUnitIndex(5)).toBe(9);
+        });
+    });
+});
+
+describe('determineLineBreaks', () => {
+    const metrics = {
+        width: 22,
+        height: 18,
+        left: 0,
+        top: -8,
+        advance: 22,
+    };
+    const rect = {
+        x: 0,
+        y: 0,
+        w: 32,
+        h: 32,
+    };
+    const glyphs = {
+        'Test': {
+            '97': {id: 0x61, metrics, rect},
+            '98': {id: 0x62, metrics, rect},
+            '99': {id: 0x63, metrics, rect},
+            '40629': {id: 0x9EB5, metrics, rect},
+            '200414': {id: 0x30EDE, metrics, rect},
+        } as any as StyleGlyph,
+    };
+
+    test('keeps alphabetic characters together', () => {
+        const tagged = new TaggedString();
+        tagged.text = 'abc';
+        const section = new SectionOptions();
+        section.fontStack = 'Test';
+        tagged.sections = [section];
+        tagged.sectionIndex = Array(3).fill(0);
+
+        expect(determineLineBreaks(tagged, 0, 300, glyphs, {}, 30)).toEqual([3]);
+    });
+
+    test('keeps ideographic characters together', () => {
+        const tagged = new TaggedString();
+        tagged.text = '𰻞𰻞麵';
+        const section = new SectionOptions();
+        section.fontStack = 'Test';
+        tagged.sections = [section];
+        tagged.sectionIndex = Array(3).fill(0);
+
+        expect(determineLineBreaks(tagged, 0, 300, glyphs, {}, 30)).toEqual([3]);
+    });
+});
+
 describe('applyTextFit', () => {
 
     describe('applyTextFitHorizontal', () => {