From c65cd2018785ca8238c06f4e10c05f374ae61dea Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Tue, 6 Jul 2021 22:50:59 -0500 Subject: [PATCH 01/11] perf: use release-nobounds --- Readme.md | 4 ++-- package.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Readme.md b/Readme.md index b3a0115..fff60f5 100644 --- a/Readme.md +++ b/Readme.md @@ -23,9 +23,9 @@ https://github.com/aminya/minijson/releases/tag/v0.3.1 - Dub ``` -dub build --config=library --build=release --compiler=ldc2 +dub build --config=library --build=release-nobounds --compiler=ldc2 # or -dub build --config=executable --build=release --compiler=ldc2 +dub build --config=executable --build=release-nobounds --compiler=ldc2 ``` ### CLI Usage diff --git a/package.json b/package.json index 5df7bca..cb7b0d1 100644 --- a/package.json +++ b/package.json @@ -20,8 +20,8 @@ "test": "jasmine ./test/index-test.mjs", "clean": "shx rm -rf dist", "build.native": "dub build --config=executable", - "build.native.release": "pnpm build.native -- --build release", - "build.native.profile": "pnpm build.native -- --build profile", + "build.native.release": "pnpm build.native -- --build release-nobounds --compiler=ldc2", + "build.native.profile": "pnpm build.native -- --build profile --compiler=ldc2", "start.profile": "cd dist && minijson.exe --file ../benchmark/fixtures/1.0.0.json && profdump.exe --dot trace.log trace.dot && dot -Tsvg trace.dot -o trace.svg && ./trace.svg", "build.node": "npm run build.native.release && node ./src/node/build.js && (tsc -p ./src/node/tsconfig.json || echo done)", "build.wasm": "ldc2 ./src/wasm/wasm.d ./src/native/lib.d --od ./dist --O3 --mtriple=wasm32-unknown-unknown-wasm", From fa2bdf6c40047a5935a8a469761152b01dfa0da8 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Tue, 6 Jul 2021 23:18:44 -0500 Subject: [PATCH 02/11] perf: use automem - reduce gc --- dub.sdl | 2 ++ dub.selections.json | 8 ++++++++ 2 files changed, 10 insertions(+) create mode 100644 dub.selections.json diff --git a/dub.sdl b/dub.sdl index 635b811..ae0980f 100644 --- a/dub.sdl +++ b/dub.sdl @@ -9,6 +9,8 @@ targetPath "./dist" sourcePaths "./src/native" importPaths "./src/native" +dependency "automem" version="~>0.6.6" + configuration "executable" { targetType "executable" mainSourceFile "./src/native/cli.d" diff --git a/dub.selections.json b/dub.selections.json new file mode 100644 index 0000000..3f9d51f --- /dev/null +++ b/dub.selections.json @@ -0,0 +1,8 @@ +{ + "fileVersion": 1, + "versions": { + "automem": "0.6.6", + "test_allocator": "0.3.3", + "unit-threaded": "2.0.0" + } +} From 61c6ee88eb41516e158d5e12ea4199db81aa0702 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Wed, 7 Jul 2021 00:35:07 -0500 Subject: [PATCH 03/11] chore: use 0.4.2.json for profiling --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index cb7b0d1..e77cfde 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "build.native": "dub build --config=executable", "build.native.release": "pnpm build.native -- --build release-nobounds --compiler=ldc2", "build.native.profile": "pnpm build.native -- --build profile --compiler=ldc2", - "start.profile": "cd dist && minijson.exe --file ../benchmark/fixtures/1.0.0.json && profdump.exe --dot trace.log trace.dot && dot -Tsvg trace.dot -o trace.svg && ./trace.svg", + "start.profile": "cd dist && minijson.exe --file ../test/fixtures/0.4.2.json && profdump.exe --dot trace.log trace.dot && dot -Tsvg trace.dot -o trace.svg && ./trace.svg", "build.node": "npm run build.native.release && node ./src/node/build.js && (tsc -p ./src/node/tsconfig.json || echo done)", "build.wasm": "ldc2 ./src/wasm/wasm.d ./src/native/lib.d --od ./dist --O3 --mtriple=wasm32-unknown-unknown-wasm", "build.browser": "npm run build.wasm && parcel build --target browser ./src/browser/index.html", From 358326d6c424c21a5b56ca124a62314faf18f8e7 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Tue, 6 Jul 2021 23:28:26 -0500 Subject: [PATCH 04/11] perf: use Vector instead of Appender --- src/native/lib.d | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/native/lib.d b/src/native/lib.d index 3701b6d..44ae8c6 100644 --- a/src/native/lib.d +++ b/src/native/lib.d @@ -1,6 +1,7 @@ module minijson.lib; -import std : ctRegex, replaceAll, join, appender, array, matchAll, matchFirst, RegexMatch; +import std : ctRegex, replaceAll, join, array, matchAll, matchFirst, RegexMatch; +import automem : Vector; const tokenizer = ctRegex!(`"|(/\*)|(\*/)|(//)|\n|\r|[|]`, "g"); @@ -18,12 +19,12 @@ const repeatingBackSlashRegex = ctRegex!(`(\\)*$`); Return: the minified json string */ -string minifyString(string jsonString, bool hasComments = false) @safe +string minifyString(string jsonString, bool hasComments = false) @trusted { auto in_string = false; auto in_multiline_comment = false; auto in_singleline_comment = false; - auto new_str = appender!(string[]); + Vector!string new_str; size_t from = 0; auto rightContext = ""; @@ -46,7 +47,7 @@ string minifyString(string jsonString, bool hasComments = false) @safe { leftContextSubstr = leftContextSubstr.replaceAll(spaceOrBreakRegex, ""); } - new_str.put(leftContextSubstr); + new_str ~= leftContextSubstr; } from = lastIndex; @@ -64,7 +65,7 @@ string minifyString(string jsonString, bool hasComments = false) @safe } else if (matchFrontHit.matchFirst(spaceOrBreakRegex).empty()) { - new_str.put(matchFrontHit); + new_str ~= matchFrontHit; } } // comments @@ -93,7 +94,7 @@ string minifyString(string jsonString, bool hasComments = false) @safe match.popFront(); } - new_str.put(rightContext); + new_str ~= rightContext; return new_str.array().join(""); } From 020f065af2044cb72279bc7a82532dd728500c61 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Wed, 7 Jul 2021 00:56:30 -0500 Subject: [PATCH 05/11] test: try-catch test failures --- .prettierignore | 1 + test/helper.mjs | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.prettierignore b/.prettierignore index 80d2c46..5bef592 100644 --- a/.prettierignore +++ b/.prettierignore @@ -3,3 +3,4 @@ pnpm-lock.yaml package-lock.json CHANGELOG.md dist +test/fixtures/*-minified.json diff --git a/test/helper.mjs b/test/helper.mjs index abecb9a..9d39c05 100644 --- a/test/helper.mjs +++ b/test/helper.mjs @@ -30,7 +30,13 @@ export async function minifyFixtures(jsonFiles) { const resultInfo = await Promise.all( minifiedFiles.map(async (minifiedFile) => { const minifiedString = await readFile(minifiedFile, "utf8") - const minifiedObject = JSON.parse(minifiedString) + let minifiedObject + try { + minifiedObject = JSON.parse(minifiedString) + } catch(e) { + console.error(`The minified file is not valid for: ${minifiedFile}`) + throw e; + } return { minifiedString, minifiedObject } }) ) From 392ca723d291cfbb01c0ea3c3903904c30501ba7 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Wed, 7 Jul 2021 00:56:26 -0500 Subject: [PATCH 06/11] perf: use std Mallocator --- dub.selections.json | 12 ++++++------ src/native/lib.d | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/dub.selections.json b/dub.selections.json index 3f9d51f..e2bc00f 100644 --- a/dub.selections.json +++ b/dub.selections.json @@ -1,8 +1,8 @@ { - "fileVersion": 1, - "versions": { - "automem": "0.6.6", - "test_allocator": "0.3.3", - "unit-threaded": "2.0.0" - } + "fileVersion": 1, + "versions": { + "automem": "0.6.6", + "test_allocator": "0.3.3", + "unit-threaded": "2.0.0" + } } diff --git a/src/native/lib.d b/src/native/lib.d index 44ae8c6..bbfdbd3 100644 --- a/src/native/lib.d +++ b/src/native/lib.d @@ -2,6 +2,7 @@ module minijson.lib; import std : ctRegex, replaceAll, join, array, matchAll, matchFirst, RegexMatch; import automem : Vector; +import std.experimental.allocator.mallocator : Mallocator; const tokenizer = ctRegex!(`"|(/\*)|(\*/)|(//)|\n|\r|[|]`, "g"); @@ -24,7 +25,7 @@ string minifyString(string jsonString, bool hasComments = false) @trusted auto in_string = false; auto in_multiline_comment = false; auto in_singleline_comment = false; - Vector!string new_str; + Vector!(string, Mallocator) new_str; size_t from = 0; auto rightContext = ""; From b1594cfa5494359d7ade109e2497e521396fd306 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Wed, 7 Jul 2021 01:07:19 -0500 Subject: [PATCH 07/11] Revert "perf: use std Mallocator" This reverts commit 392ca723d291cfbb01c0ea3c3903904c30501ba7. --- dub.selections.json | 12 ++++++------ src/native/lib.d | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/dub.selections.json b/dub.selections.json index e2bc00f..3f9d51f 100644 --- a/dub.selections.json +++ b/dub.selections.json @@ -1,8 +1,8 @@ { - "fileVersion": 1, - "versions": { - "automem": "0.6.6", - "test_allocator": "0.3.3", - "unit-threaded": "2.0.0" - } + "fileVersion": 1, + "versions": { + "automem": "0.6.6", + "test_allocator": "0.3.3", + "unit-threaded": "2.0.0" + } } diff --git a/src/native/lib.d b/src/native/lib.d index bbfdbd3..44ae8c6 100644 --- a/src/native/lib.d +++ b/src/native/lib.d @@ -2,7 +2,6 @@ module minijson.lib; import std : ctRegex, replaceAll, join, array, matchAll, matchFirst, RegexMatch; import automem : Vector; -import std.experimental.allocator.mallocator : Mallocator; const tokenizer = ctRegex!(`"|(/\*)|(\*/)|(//)|\n|\r|[|]`, "g"); @@ -25,7 +24,7 @@ string minifyString(string jsonString, bool hasComments = false) @trusted auto in_string = false; auto in_multiline_comment = false; auto in_singleline_comment = false; - Vector!(string, Mallocator) new_str; + Vector!string new_str; size_t from = 0; auto rightContext = ""; From da8540dffddd1a641f2f9ccc5579db0bbda62abc Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Wed, 7 Jul 2021 01:08:16 -0500 Subject: [PATCH 08/11] perf: use separate tokenizers for comment-less json files --- src/native/lib.d | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/native/lib.d b/src/native/lib.d index 44ae8c6..4230837 100644 --- a/src/native/lib.d +++ b/src/native/lib.d @@ -3,7 +3,8 @@ module minijson.lib; import std : ctRegex, replaceAll, join, array, matchAll, matchFirst, RegexMatch; import automem : Vector; -const tokenizer = ctRegex!(`"|(/\*)|(\*/)|(//)|\n|\r|[|]`, "g"); +const tokenizerWithComment = ctRegex!(`"|(/\*)|(\*/)|(//)|\n|\r|[|]`, "g"); +const tokenizerNoComment = ctRegex!(`"|\n|\r|[|]`, "g"); const spaceOrBreakRegex = ctRegex!(`\s`); @@ -28,6 +29,8 @@ string minifyString(string jsonString, bool hasComments = false) @trusted size_t from = 0; auto rightContext = ""; + const tokenizer = !hasComments ? tokenizerNoComment : tokenizerWithComment; + auto match = jsonString.matchAll(tokenizer); while (!match.empty()) From db5165c3b2a05726366551733137ac15956f4d5e Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Wed, 7 Jul 2021 01:14:35 -0500 Subject: [PATCH 09/11] fix: escape [ in the tokenizers --- src/native/lib.d | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/native/lib.d b/src/native/lib.d index 4230837..c606897 100644 --- a/src/native/lib.d +++ b/src/native/lib.d @@ -3,8 +3,8 @@ module minijson.lib; import std : ctRegex, replaceAll, join, array, matchAll, matchFirst, RegexMatch; import automem : Vector; -const tokenizerWithComment = ctRegex!(`"|(/\*)|(\*/)|(//)|\n|\r|[|]`, "g"); -const tokenizerNoComment = ctRegex!(`"|\n|\r|[|]`, "g"); +const tokenizerWithComment = ctRegex!(`"|(/\*)|(\*/)|(//)|\n|\r|\[|]`, "g"); +const tokenizerNoComment = ctRegex!(`[\n\r"[]]`, "g"); const spaceOrBreakRegex = ctRegex!(`\s`); From fb359046b0d7ccf1803bbe5d2a68a2538d179757 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Wed, 7 Jul 2021 01:33:19 -0500 Subject: [PATCH 10/11] perf: remove redundant check Reduces a duplicate branch from the code --- src/native/lib.d | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/native/lib.d b/src/native/lib.d index c606897..ef1596a 100644 --- a/src/native/lib.d +++ b/src/native/lib.d @@ -104,7 +104,8 @@ string minifyString(string jsonString, bool hasComments = false) @trusted bool hasNoSlashOrEvenNumberOfSlashes(string leftContext) @safe { auto leftContextMatch = leftContext.matchFirst(repeatingBackSlashRegex); - return leftContextMatch.empty() || (leftContextMatch.hit().length % 2 == 0); + // if not matched the hit length will be 0 (== leftContextMatch.empty()) + return leftContextMatch.hit().length % 2 == 0; } /** From 9be67f4898e943f8904b242389734594df8b28c5 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Wed, 7 Jul 2021 01:48:19 -0500 Subject: [PATCH 11/11] perf: only check for \\ in leftContextSubstr This significantly speeds up the code --- .prettierignore | 1 + Readme.md | 8 ++++---- benchmark/js-benchmark.mjs | 2 +- benchmark/native-benchmark.mjs | 2 +- package.json | 2 +- src/native/lib.d | 5 +++-- test/helper.mjs | 4 ++-- 7 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.prettierignore b/.prettierignore index 5bef592..71c69ae 100644 --- a/.prettierignore +++ b/.prettierignore @@ -4,3 +4,4 @@ package-lock.json CHANGELOG.md dist test/fixtures/*-minified.json +dub.selections.json diff --git a/Readme.md b/Readme.md index fff60f5..130e349 100644 --- a/Readme.md +++ b/Readme.md @@ -1,8 +1,8 @@ # minijson -Minify JSON files **fast**! Written in D. +Minify JSON files **blazing fast**! Written in D. -1.5 times faster than jsonminify. +55 times faster than jsonminify! [![CI](https://github.com/aminya/minijson/actions/workflows/CI.yml/badge.svg)](https://github.com/aminya/minijson/actions/workflows/CI.yml) @@ -70,10 +70,10 @@ minifyFiles(["file1.json", "file2.json"]); ``` ❯ node .\benchmark\native-benchmark.mjs -38,823 ms +1.086 seconds ❯ node .\benchmark\js-benchmark.mjs -58,686 ms +58.686 seconds ``` ### Contributing diff --git a/benchmark/js-benchmark.mjs b/benchmark/js-benchmark.mjs index 79dd7fe..e5e8ab5 100644 --- a/benchmark/js-benchmark.mjs +++ b/benchmark/js-benchmark.mjs @@ -18,4 +18,4 @@ await Promise.all( ) const t2 = performance.now() -console.log(t2 - t1) +console.log(((t2 - t1)/1000).toFixed(3), "seconds") diff --git a/benchmark/native-benchmark.mjs b/benchmark/native-benchmark.mjs index d068675..b5f0e10 100644 --- a/benchmark/native-benchmark.mjs +++ b/benchmark/native-benchmark.mjs @@ -8,4 +8,4 @@ const t1 = performance.now() await minifyFiles(jsonFiles) const t2 = performance.now() -console.log(t2 - t1) +console.log(((t2 - t1)/1000).toFixed(3), "seconds") diff --git a/package.json b/package.json index e77cfde..e56784a 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "@aminya/minijson", "author": "Amin Yahyaabdi", "version": "0.3.1", - "description": "Minify JSON files fast! Written in D", + "description": "Minify JSON files blazing fast! Written in D", "homepage": "https://github.com/aminya/minijson", "license": "MIT", "files": [ diff --git a/src/native/lib.d b/src/native/lib.d index ef1596a..3c9a324 100644 --- a/src/native/lib.d +++ b/src/native/lib.d @@ -43,9 +43,10 @@ string minifyString(string jsonString, bool hasComments = false) @trusted const lastIndex = jsonString.length - rightContext.length; const noCommentOrNotInComment = !hasComments || (!in_multiline_comment && !in_singleline_comment); + + auto leftContextSubstr = leftContext[from .. $]; if (noCommentOrNotInComment) { - auto leftContextSubstr = leftContext[from .. $]; if (!in_string) { leftContextSubstr = leftContextSubstr.replaceAll(spaceOrBreakRegex, ""); @@ -58,7 +59,7 @@ string minifyString(string jsonString, bool hasComments = false) @trusted { if (matchFrontHit == "\"") { - if (!in_string || hasNoSlashOrEvenNumberOfSlashes(leftContext)) + if (!in_string || hasNoSlashOrEvenNumberOfSlashes(leftContextSubstr)) { // start of string with ", or unescaped " character found to end string in_string = !in_string; diff --git a/test/helper.mjs b/test/helper.mjs index 9d39c05..573dbee 100644 --- a/test/helper.mjs +++ b/test/helper.mjs @@ -33,9 +33,9 @@ export async function minifyFixtures(jsonFiles) { let minifiedObject try { minifiedObject = JSON.parse(minifiedString) - } catch(e) { + } catch (e) { console.error(`The minified file is not valid for: ${minifiedFile}`) - throw e; + throw e } return { minifiedString, minifiedObject } })