diff --git a/src/transform/frontmatter.ts b/src/transform/frontmatter.ts new file mode 100644 index 00000000..7948bed9 --- /dev/null +++ b/src/transform/frontmatter.ts @@ -0,0 +1,104 @@ +import {YAMLException, dump, load} from 'js-yaml'; +import cloneDeepWith from 'lodash/cloneDeepWith'; + +import {log} from './log'; + +export type FrontMatter = { + [key: string]: unknown; + metadata?: Record[]; +}; + +const SEP = '---'; + +/** + * Temporary workaround to enable parsing YAML metadata from potentially + * Liquid-aware source files + * @param content Input string which could contain Liquid-style substitution syntax (which clashes with YAML + * object syntax) + * @returns String with `{}` escaped, ready to be parsed with `js-yaml` + */ +const escapeLiquid = (content: string): string => + content.replace(/{{/g, '(({{').replace(/}}/g, '}}))'); + +/** + * Inverse of a workaround defined above. + * @see `escapeLiquidSubstitutionSyntax` + * @param escapedContent Input string with `{}` escaped with backslashes + * @returns Unescaped string + */ +const unescapeLiquid = (escapedContent: string): string => + escapedContent.replace(/\(\({{/g, '{{').replace(/}}\)\)/g, '}}'); + +const matchMetadata = (fileContent: string) => { + if (!fileContent.startsWith(SEP)) { + return null; + } + + const closeStart = fileContent.indexOf('\n' + SEP, SEP.length); + const closeEnd = fileContent.indexOf('\n', closeStart + 1); + + if (closeStart === -1) { + return null; + } + + return [fileContent.slice(SEP.length, closeStart).trim(), fileContent.slice(closeEnd + 1)]; +}; + +const duplicateKeysCompatibleLoad = (yaml: string, filePath: string | undefined) => { + try { + return load(yaml); + } catch (e) { + if (e instanceof YAMLException) { + const duplicateKeysDeprecationWarning = ` + In ${filePath ?? '(unknown)'}: Encountered a YAML parsing exception when processing file metadata: ${e.reason}. + It's highly possible the input file contains duplicate mapping keys. + Will retry processing with necessary compatibility flags. + Please note that this behaviour is DEPRECATED and WILL be removed in a future version + without further notice, so the build WILL fail when supplied with YAML-incompatible meta. + ` + .replace(/^\s+/gm, '') + .replace(/\n/g, ' ') + .trim(); + + log.warn(duplicateKeysDeprecationWarning); + + return load(yaml, {json: true}); + } + + throw e; + } +}; + +export const extractFrontMatter = ( + fileContent: string, + filePath?: string, +): [FrontMatter, string] => { + const matches = matchMetadata(fileContent); + + if (matches) { + const [metadata, strippedContent] = matches; + + return [ + cloneDeepWith( + duplicateKeysCompatibleLoad(escapeLiquid(metadata), filePath) as FrontMatter, + (v) => (typeof v === 'string' ? unescapeLiquid(v) : undefined), + ), + strippedContent, + ]; + } + + return [{}, fileContent]; +}; + +export const composeFrontMatter = (frontMatter: FrontMatter, strippedContent: string) => { + const dumped = dump(frontMatter, {lineWidth: -1}).trim(); + + // This empty object check is a bit naive + // The other option would be to check if all own fields are `undefined`, + // since we exploit passing in `undefined` to remove a field quite a bit + if (dumped === '{}') { + return strippedContent; + } + + return `${SEP}\n${dumped}\n${SEP}\n${strippedContent}`; +}; diff --git a/src/transform/frontmatter/common.ts b/src/transform/frontmatter/common.ts deleted file mode 100644 index 8679cdb1..00000000 --- a/src/transform/frontmatter/common.ts +++ /dev/null @@ -1,27 +0,0 @@ -export type FrontMatter = { - [key: string]: unknown; - metadata?: Record[]; -}; - -export const frontMatterFence = '---'; - -/** - * Temporary workaround to enable parsing YAML metadata from potentially - * Liquid-aware source files - * @param content Input string which could contain Liquid-style substitution syntax (which clashes with YAML - * object syntax) - * @returns String with `{}` escaped, ready to be parsed with `js-yaml` - */ -export const escapeLiquidSubstitutionSyntax = (content: string): string => - content.replace(/{{/g, '(({{').replace(/}}/g, '}}))'); - -/** - * Inverse of a workaround defined above. - * @see `escapeLiquidSubstitutionSyntax` - * @param escapedContent Input string with `{}` escaped with backslashes - * @returns Unescaped string - */ -export const unescapeLiquidSubstitutionSyntax = (escapedContent: string): string => - escapedContent.replace(/\(\({{/g, '{{').replace(/}}\)\)/g, '}}'); - -export const countLineAmount = (str: string) => str.split(/\r?\n/).length; diff --git a/src/transform/frontmatter/emplace.ts b/src/transform/frontmatter/emplace.ts deleted file mode 100644 index 28a5e246..00000000 --- a/src/transform/frontmatter/emplace.ts +++ /dev/null @@ -1,24 +0,0 @@ -import {dump} from 'js-yaml'; - -import {FrontMatter, frontMatterFence} from './common'; - -export const serializeFrontMatter = (frontMatter: FrontMatter) => { - const dumped = dump(frontMatter, {lineWidth: -1}).trim(); - - // This empty object check is a bit naive - // The other option would be to check if all own fields are `undefined`, - // since we exploit passing in `undefined` to remove a field quite a bit - if (dumped === '{}') { - return ''; - } - - return `${frontMatterFence}\n${dumped}\n${frontMatterFence}\n`; -}; - -export const emplaceSerializedFrontMatter = ( - frontMatterStrippedContent: string, - frontMatter: string, -) => `${frontMatter}${frontMatterStrippedContent}`; - -export const emplaceFrontMatter = (frontMatterStrippedContent: string, frontMatter: FrontMatter) => - emplaceSerializedFrontMatter(frontMatterStrippedContent, serializeFrontMatter(frontMatter)); diff --git a/src/transform/frontmatter/extract.ts b/src/transform/frontmatter/extract.ts deleted file mode 100644 index 1859e338..00000000 --- a/src/transform/frontmatter/extract.ts +++ /dev/null @@ -1,94 +0,0 @@ -import {YAMLException, load} from 'js-yaml'; - -import {log} from '../log'; - -import { - FrontMatter, - countLineAmount, - escapeLiquidSubstitutionSyntax, - frontMatterFence, - unescapeLiquidSubstitutionSyntax, -} from './common'; -import {transformFrontMatterValues} from './transformValues'; - -type ParseExistingMetadataReturn = { - frontMatter: FrontMatter; - frontMatterStrippedContent: string; - frontMatterLineCount: number; -}; - -const matchMetadata = (fileContent: string) => { - if (!fileContent.startsWith(frontMatterFence)) { - return null; - } - - // Search by format: - // --- - // metaName1: metaValue1 - // metaName2: meta value2 - // incorrectMetadata - // --- - const regexpMetadata = '(?<=-{3}\\r?\\n)((.*\\r?\\n)*?)(?=-{3}\\r?\\n)'; - // Search by format: - // --- - // main content 123 - const regexpFileContent = '-{3}\\r?\\n((.*[\r?\n]*)*)'; - - const regexpParseFileContent = new RegExp(`${regexpMetadata}${regexpFileContent}`, 'gm'); - - return regexpParseFileContent.exec(fileContent); -}; - -const duplicateKeysCompatibleLoad = (yaml: string, filePath: string | undefined) => { - try { - return load(yaml); - } catch (e) { - if (e instanceof YAMLException) { - const duplicateKeysDeprecationWarning = ` - In ${filePath ?? '(unknown)'}: Encountered a YAML parsing exception when processing file metadata: ${e.reason}. - It's highly possible the input file contains duplicate mapping keys. - Will retry processing with necessary compatibility flags. - Please note that this behaviour is DEPRECATED and WILL be removed in a future version - without further notice, so the build WILL fail when supplied with YAML-incompatible meta. - ` - .replace(/^\s+/gm, '') - .replace(/\n/g, ' ') - .trim(); - - log.warn(duplicateKeysDeprecationWarning); - - return load(yaml, {json: true}); - } - - throw e; - } -}; - -export const separateAndExtractFrontMatter = ( - fileContent: string, - filePath?: string, -): ParseExistingMetadataReturn => { - const matches = matchMetadata(fileContent); - - if (matches && matches.length > 0) { - const [, metadata, , metadataStrippedContent] = matches; - - return { - frontMatter: transformFrontMatterValues( - duplicateKeysCompatibleLoad( - escapeLiquidSubstitutionSyntax(metadata), - filePath, - ) as FrontMatter, - (v) => (typeof v === 'string' ? unescapeLiquidSubstitutionSyntax(v) : v), - ), - frontMatterStrippedContent: metadataStrippedContent, - frontMatterLineCount: countLineAmount(metadata), - }; - } - - return { - frontMatter: {}, - frontMatterStrippedContent: fileContent, - frontMatterLineCount: 0, - }; -}; diff --git a/src/transform/frontmatter/index.ts b/src/transform/frontmatter/index.ts deleted file mode 100644 index d2ea6608..00000000 --- a/src/transform/frontmatter/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './extract'; -export * from './emplace'; -export * from './transformValues'; -export {countLineAmount} from './common'; diff --git a/src/transform/frontmatter/transformValues.ts b/src/transform/frontmatter/transformValues.ts deleted file mode 100644 index 0185d951..00000000 --- a/src/transform/frontmatter/transformValues.ts +++ /dev/null @@ -1,26 +0,0 @@ -import {FrontMatter} from './common'; - -export const transformFrontMatterValues = ( - frontMatter: FrontMatter, - valueMapper: (v: unknown) => unknown, -): FrontMatter => { - const transformInner = (something: unknown): unknown => { - if (Array.isArray(something)) { - return something.map((el) => transformInner(el)); - } - - if (typeof something === 'object' && something !== null) { - return Object.fromEntries( - Object.entries(something).map(([k, v]) => [k, transformInner(v)]), - ); - } - - if (typeof something === 'function') { - return something; - } - - return valueMapper(something); - }; - - return transformInner(frontMatter) as FrontMatter; -}; diff --git a/src/transform/liquid/index.ts b/src/transform/liquid/index.ts index a2a4d545..a6e8070f 100644 --- a/src/transform/liquid/index.ts +++ b/src/transform/liquid/index.ts @@ -1,12 +1,8 @@ import type {Dictionary} from 'lodash'; -import { - countLineAmount, - emplaceSerializedFrontMatter, - separateAndExtractFrontMatter, - serializeFrontMatter, - transformFrontMatterValues, -} from '../frontmatter'; +import {cloneDeepWith} from 'lodash'; + +import {composeFrontMatter, extractFrontMatter} from '../frontmatter'; import applySubstitutions from './substitutions'; import {prepareSourceMap} from './sourceMap'; @@ -149,65 +145,68 @@ function liquidSnippet< return output as unknown as C; } -type TransformSourceMapOptions = { - emplacedResultOffset: number; - emplacedSourceOffset: number; -}; +function linesCount(content: string) { + let count = 1, + index = -1; + while ((index = content.indexOf('\n', index + 1)) > -1) { + count++; + } -function transformSourceMap( - sourceMap: Dictionary, - {emplacedResultOffset, emplacedSourceOffset}: TransformSourceMapOptions, -) { - return Object.fromEntries( - Object.entries(sourceMap).map(([lineInResult, lineInSource]) => [ - (Number(lineInResult) + emplacedResultOffset).toString(), - (Number(lineInSource) + emplacedSourceOffset).toString(), - ]), - ); + return count; } function liquidDocument< B extends boolean = false, C = B extends false ? string : {output: string; sourceMap: Dictionary}, >( - originInput: string, + input: string, vars: Record, path?: string, settings?: ArgvSettings & {withSourceMap?: B}, ): C { - const {frontMatter, frontMatterStrippedContent, frontMatterLineCount} = - separateAndExtractFrontMatter(originInput, path); + const [frontMatter, strippedContent] = extractFrontMatter(input, path); - const transformedFrontMatter = transformFrontMatterValues(frontMatter, (v) => - typeof v === 'string' - ? liquidSnippet(v, vars, path, {...settings, withSourceMap: false}) - : v, + const liquidedFrontMatter = cloneDeepWith(frontMatter, (value: unknown) => + typeof value === 'string' + ? liquidSnippet(value, vars, path, {...settings, withSourceMap: false}) + : undefined, ); - const transformedAndSerialized = serializeFrontMatter(transformedFrontMatter); - // -1 comes from the fact that the last line in serialized FM is the same as the first line in stripped content - const resultFrontMatterOffset = Math.max(0, countLineAmount(transformedAndSerialized) - 1); - const sourceFrontMatterOffset = Math.max(0, frontMatterLineCount - 1); + const liquidedResult = liquidSnippet(strippedContent, vars, path, settings); + const liquidedContent = + typeof liquidedResult === 'object' ? liquidedResult.output : liquidedResult; - const liquidProcessedContent = liquidSnippet(frontMatterStrippedContent, vars, path, settings); + const output = composeFrontMatter(liquidedFrontMatter, liquidedContent); + + if (typeof liquidedResult === 'object') { + const inputLinesCount = linesCount(input); + const outputLinesCount = linesCount(output); + const contentLinesCount = linesCount(strippedContent); + const contentLinesDiff = linesCount(liquidedContent) - contentLinesCount; + + const fullLinesDiff = outputLinesCount - inputLinesCount; + + // Always >= 0 + const sourceOffset = inputLinesCount - contentLinesCount; + // Content lines diff already counted in source map + const resultOffset = fullLinesDiff - contentLinesDiff; + + liquidedResult.sourceMap = Object.fromEntries( + Object.entries(liquidedResult.sourceMap).map(([lineInResult, lineInSource]) => [ + (Number(lineInResult) + resultOffset).toString(), + (Number(lineInSource) + sourceOffset).toString(), + ]), + ); + } // typeof check for better inference; the catch is that return of liquidSnippet can be an // object even with source maps off, see `substitutions.test.ts` - return (settings?.withSourceMap && typeof liquidProcessedContent === 'object' + return (settings?.withSourceMap && typeof liquidedResult === 'object' ? { - output: emplaceSerializedFrontMatter( - liquidProcessedContent.output, - transformedAndSerialized, - ), - sourceMap: transformSourceMap(liquidProcessedContent.sourceMap, { - emplacedResultOffset: resultFrontMatterOffset, - emplacedSourceOffset: sourceFrontMatterOffset, - }), + output, + sourceMap: liquidedResult.sourceMap, } - : emplaceSerializedFrontMatter( - liquidProcessedContent as string, - transformedAndSerialized, - )) as unknown as C; + : output) as unknown as C; } // both default and named exports for convenience diff --git a/test/liquid/frontmatter.test.ts b/test/liquid/frontmatter.test.ts index e2c371db..7a6be83b 100644 --- a/test/liquid/frontmatter.test.ts +++ b/test/liquid/frontmatter.test.ts @@ -1,6 +1,6 @@ import dedent from 'ts-dedent'; -import {emplaceFrontMatter, separateAndExtractFrontMatter} from '../../src/transform/frontmatter'; +import {composeFrontMatter, extractFrontMatter} from '../../src/transform/frontmatter'; import liquidDocument from '../../src/transform/liquid'; describe('front matter extract/emplace utility function pair', () => { @@ -36,14 +36,14 @@ describe('front matter extract/emplace utility function pair', () => { - + # Content. `, ])( `preserves the same amount of linebreaks between front matter block and content %#`, (input) => { - const {frontMatter, frontMatterStrippedContent} = separateAndExtractFrontMatter(input); - const emplaced = emplaceFrontMatter(frontMatterStrippedContent, frontMatter); + const [frontMatter, strippedContent] = extractFrontMatter(input); + const emplaced = composeFrontMatter(frontMatter, strippedContent); expect(emplaced).toEqual(input); }, @@ -59,7 +59,7 @@ describe('front matter extract/emplace utility function pair', () => { Test. `; - const {frontMatter} = separateAndExtractFrontMatter(content); + const [frontMatter] = extractFrontMatter(content); expect(frontMatter).toMatchObject({prop: '{{ wouldbreak }}'}); }); @@ -79,7 +79,7 @@ describe('Liquid substitutions in front matter (formerly metadata)', () => { const liquidProcessed = liquidDocument(content, {var: ''}, 'frontmatter.test.ts.md'); - const {frontMatter} = separateAndExtractFrontMatter(liquidProcessed); + const [frontMatter] = extractFrontMatter(liquidProcessed); expect(frontMatter).toEqual({ verbatim: '', diff --git a/test/liquid/transformValues.test.ts b/test/liquid/transformValues.test.ts deleted file mode 100644 index 20b11af6..00000000 --- a/test/liquid/transformValues.test.ts +++ /dev/null @@ -1,70 +0,0 @@ -import {transformFrontMatterValues} from '../../src/transform/frontmatter'; - -describe('transformFrontMatterValues utility function', () => { - it.each([ - {desc: 'strings', value: 'test'}, - {desc: 'numbers', value: 420.69}, - {desc: 'boolean', value: true}, - {desc: 'bigint', value: BigInt('0x1fffffffffffff')}, - {desc: 'symbol', value: Symbol('$$Test')}, - {desc: 'undefined', value: undefined}, - {desc: 'null', value: 'null'}, - ])('calls value mapper on primitive values inside the object ($desc)', ({value}) => { - const mapper = jest.fn((v: unknown) => v); - const noop = jest.fn(); - const obj = {}; - const arr = [] as []; - const inst = new (class {})(); - const fm = {prop: value, noop, obj, arr, inst}; - - transformFrontMatterValues(fm, mapper); - - expect(mapper).toBeCalledWith(value); - expect(mapper).not.toBeCalledWith(noop); - expect(mapper).not.toBeCalledWith(obj); - expect(mapper).not.toBeCalledWith(arr); - expect(mapper).not.toBeCalledWith(inst); - }); - - it('recursively traverses the object', () => { - const fm = { - prop: { - a: 7, - b: { - c: 9, - d: { - e: {f: 11}, - }, - }, - }, - }; - - const transformed = transformFrontMatterValues(fm, (v) => - typeof v === 'number' ? v + 2 : v, - ); - - expect(transformed).toEqual({prop: {a: 9, b: {c: 11, d: {e: {f: 13}}}}}); - }); - - it('recursively traverses arrays inside the object', () => { - const fm = { - arr: [1, 2, 3, {prop: 7, moreDepth: [5, {a: 11}]}], - }; - - const transformed = transformFrontMatterValues(fm, (v) => - typeof v === 'number' ? v + 2 : v, - ); - - expect(transformed).toEqual({arr: [3, 4, 5, {prop: 9, moreDepth: [7, {a: 13}]}]}); - }); - - it('can handle arrays as an entrypoint if need be', () => { - const fm = [1, 2, 3, {prop: 7, moreDepth: [5, {a: 11}]}]; - - const transformed = transformFrontMatterValues(fm as {}, (v) => - typeof v === 'number' ? v + 2 : v, - ); - - expect(transformed).toEqual([3, 4, 5, {prop: 9, moreDepth: [7, {a: 13}]}]); - }); -});