feat: implement ways to interact with collect via transform API

src/transform/index.ts

@@ -1,9 +1,9 @@
-import type {EnvType, OptionsType, OutputType} from './typings';
+import type {EnvType, OptionsType, OutputType, RootCollectorOptions} from './typings';
 
 import {bold} from 'chalk';
 
 import {log} from './log';
-import liquidSnippet from './liquid';
+import liquidDocument from './liquid';
 import initMarkdownIt from './md';
 
 function applyLiquid(input: string, options: OptionsType) {
@@ -17,7 +17,7 @@ function applyLiquid(input: string, options: OptionsType) {
 
     return disableLiquid || isLiquided
         ? input
-        : liquidSnippet(input, vars, path, {conditionsInCode});
+        : liquidDocument(input, vars, path, {conditionsInCode});
 }
 
 function handleError(error: unknown, path?: string): never {
@@ -33,8 +33,14 @@ function emitResult(html: string, env: EnvType): OutputType {
     };
 }
 
+type TransformFunction = {
+    (originInput: string, options?: OptionsType): OutputType;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    collect: (input: string, options: RootCollectorOptions<any>) => string;
+};
+
 // eslint-disable-next-line consistent-return
-function transform(originInput: string, options: OptionsType = {}) {
+const transform: TransformFunction = (originInput: string, options: OptionsType = {}) => {
     const input = applyLiquid(originInput, options);
     const {parse, compile, env} = initMarkdownIt(options);
 
@@ -43,11 +49,40 @@ function transform(originInput: string, options: OptionsType = {}) {
     } catch (error) {
         handleError(error, options.path);
     }
-}
+};
+
+transform.collect = (
+    input: string,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    {mdItInitOptions, pluginCollectOptions, parserPluginsOverride}: RootCollectorOptions<any>,
+) => {
+    const maybeLiquidedInput = applyLiquid(input, mdItInitOptions);
+    const {parse} = initMarkdownIt({
+        ...mdItInitOptions,
+        plugins: parserPluginsOverride ?? mdItInitOptions.plugins,
+    });
+
+    const plugins = mdItInitOptions.plugins ?? [];
+
+    try {
+        const tokenStream = parse(maybeLiquidedInput);
+
+        return plugins.reduce((collected, plugin) => {
+            const collectOutput = plugin.collect?.(collected, {
+                ...pluginCollectOptions,
+                tokenStream,
+            });
+
+            return collectOutput ?? collected;
+        }, maybeLiquidedInput);
+    } catch (error) {
+        handleError(error, mdItInitOptions.path);
+    }
+};
 
 export = transform;
 
-// eslint-disable-next-line @typescript-eslint/no-namespace, no-redeclare -- backward compatibility
+// eslint-disable-next-line @typescript-eslint/no-namespace, @typescript-eslint/no-redeclare -- backward compatibility
 namespace transform {
     export type Options = OptionsType;
     export type Output = OutputType;

src/transform/typings.ts

@@ -1,5 +1,5 @@
 import {LanguageFn} from 'highlight.js';
-import DefaultMarkdownIt from 'markdown-it';
+import DefaultMarkdownIt, {Token} from 'markdown-it';
 import DefaultStateCore from 'markdown-it/lib/rules_core/state_core';
 
 import {SanitizeOptions} from './sanitize';
@@ -47,7 +47,7 @@ export interface OptionsType {
     sanitizeOptions?: SanitizeOptions;
     needFlatListHeadings?: boolean;
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    plugins?: MarkdownItPluginCb<any>[];
+    plugins?: ExtendedPluginWithCollect<any, any>[];
     preprocessors?: MarkdownItPreprocessorCb[]; // Preprocessors should modify the input before passing it to MD
     highlightLangs?: HighlightLangMap;
     disableRules?: string[];
@@ -96,6 +96,27 @@ export type MarkdownItPluginCb<T extends {} = {}> = {
     (md: MarkdownIt, opts: T & MarkdownItPluginOpts): void;
 };
 
+export type IntrinsicCollectOptions = {
+    tokenStream: Token[];
+};
+
+export type ExtendedPluginWithCollect<
+    PluginRegularOptions extends {} = {},
+    PluginCollectOptions = {},
+> = MarkdownItPluginCb<PluginRegularOptions> & {
+    collect?: (
+        input: string,
+        options: PluginCollectOptions & IntrinsicCollectOptions,
+    ) => string | void;
+};
+
+export type RootCollectorOptions<PluginCollectOptions> = {
+    mdItInitOptions: OptionsType;
+    pluginCollectOptions: PluginCollectOptions;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    parserPluginsOverride?: ExtendedPluginWithCollect<any, any>[];
+};
+
 export type MarkdownItPreprocessorCb<T extends unknown = {}> = {
     (input: string, opts: T & Partial<MarkdownItPluginOpts>, md?: MarkdownIt): string;
 };

test/collect.test.ts

@@ -0,0 +1,218 @@
+import {Token} from 'markdown-it';
+
+import transform from '../src/transform';
+import {ExtendedPluginWithCollect, IntrinsicCollectOptions} from '../src/transform/typings';
+
+describe('transform.collect', () => {
+    it(`applies liquid if it wasn't disabled`, () => {
+        const input = `### Docs for {{ product }}`;
+        const vars = {product: 'Diplodoc Platform'};
+
+        const pluginWithCollect: ExtendedPluginWithCollect = jest.fn();
+        pluginWithCollect.collect = jest.fn();
+
+        const result = transform.collect(input, {
+            mdItInitOptions: {
+                plugins: [pluginWithCollect],
+                vars: vars,
+            },
+            pluginCollectOptions: {},
+        });
+
+        expect(pluginWithCollect.collect).toHaveBeenCalledTimes(1);
+        expect(pluginWithCollect.collect).toHaveBeenCalledWith(
+            '### Docs for Diplodoc Platform',
+            expect.objectContaining<IntrinsicCollectOptions>({
+                tokenStream: expect.arrayContaining<Token>([
+                    expect.objectContaining({
+                        children: expect.arrayContaining<Token>([
+                            expect.objectContaining({content: 'Docs for Diplodoc Platform'}),
+                        ]),
+                    }),
+                ]),
+            }),
+        );
+
+        expect(result).toBe('### Docs for Diplodoc Platform');
+    });
+
+    it(`doesn't apply liquid if it was explicitly disabled by passing isLiquided = true`, () => {
+        const input = `### Docs for {{ product }}`;
+        const vars = {product: 'Diplodoc Platform'};
+
+        const pluginWithCollect: ExtendedPluginWithCollect = jest.fn();
+        pluginWithCollect.collect = jest.fn();
+
+        const result = transform.collect(input, {
+            mdItInitOptions: {
+                plugins: [pluginWithCollect],
+                vars: vars,
+                isLiquided: true,
+            },
+            pluginCollectOptions: {},
+        });
+
+        expect(pluginWithCollect.collect).toHaveBeenCalledTimes(1);
+        expect(pluginWithCollect.collect).toHaveBeenCalledWith(
+            '### Docs for {{ product }}',
+            expect.objectContaining<IntrinsicCollectOptions>({
+                tokenStream: expect.arrayContaining<Token>([
+                    expect.objectContaining({
+                        children: expect.arrayContaining<Token>([
+                            expect.objectContaining({content: 'Docs for {{ product }}'}),
+                        ]),
+                    }),
+                ]),
+            }),
+        );
+
+        expect(result).toBe('### Docs for {{ product }}');
+    });
+
+    it(`doesn't apply liquid if it was explicitly disabled by passing disableLiquid = true`, () => {
+        const input = `### Docs for {{ product }}`;
+        const vars = {product: 'Diplodoc Platform'};
+
+        const pluginWithCollect: ExtendedPluginWithCollect = jest.fn();
+        pluginWithCollect.collect = jest.fn();
+
+        const result = transform.collect(input, {
+            mdItInitOptions: {
+                plugins: [pluginWithCollect],
+                vars: vars,
+                disableLiquid: true,
+            },
+            pluginCollectOptions: {},
+        });
+
+        expect(pluginWithCollect.collect).toHaveBeenCalledTimes(1);
+        expect(pluginWithCollect.collect).toHaveBeenCalledWith(
+            '### Docs for {{ product }}',
+            expect.objectContaining<IntrinsicCollectOptions>({
+                tokenStream: expect.arrayContaining<Token>([
+                    expect.objectContaining({
+                        children: expect.arrayContaining<Token>([
+                            expect.objectContaining({content: 'Docs for {{ product }}'}),
+                        ]),
+                    }),
+                ]),
+            }),
+        );
+
+        expect(result).toBe('### Docs for {{ product }}');
+    });
+
+    it('applies collect function in order as specified by the plugin array', () => {
+        const pluginFactory = (textToAppend: string) => {
+            const plugin: ExtendedPluginWithCollect = jest.fn();
+
+            plugin.collect = (input: string) => `${input}${textToAppend}`;
+
+            return plugin;
+        };
+
+        const result = transform.collect('', {
+            mdItInitOptions: {
+                plugins: [pluginFactory('foo'), pluginFactory('bar'), pluginFactory('baz')],
+            },
+            pluginCollectOptions: {},
+        });
+
+        expect(result).toBe('foobarbaz');
+    });
+
+    it('provides valid token stream to collect functions after running the input through mdit plugins', () => {
+        const mockPlugin: ExtendedPluginWithCollect = (md) => {
+            md.core.ruler.push('mock', (state) =>
+                state.tokens.push(new state.Token('mockToken', 'mock', 0)),
+            );
+        };
+
+        mockPlugin.collect = jest.fn();
+
+        transform.collect('', {
+            mdItInitOptions: {
+                plugins: [mockPlugin],
+            },
+            pluginCollectOptions: {},
+        });
+
+        expect(mockPlugin.collect).toHaveBeenCalledWith(
+            '',
+            expect.objectContaining<IntrinsicCollectOptions>({
+                tokenStream: expect.arrayContaining<Token>([
+                    expect.objectContaining({type: 'mockToken'}),
+                ]),
+            }),
+        );
+    });
+
+    it('prioritizes PluginOverrides when evaluating the token stream to pass to collect functions', () => {
+        const mockPluginFactory = (tokenType: string) => {
+            const plugin: ExtendedPluginWithCollect = (md) =>
+                md.core.ruler.push('mock', (state) =>
+                    state.tokens.push(new state.Token(tokenType, 'mock', 0)),
+                );
+
+            plugin.collect = jest.fn();
+
+            return plugin;
+        };
+
+        const originalPlugin = mockPluginFactory('original');
+        const overriddenPlugin = mockPluginFactory('overridden');
+
+        transform.collect('', {
+            mdItInitOptions: {
+                plugins: [originalPlugin],
+            },
+            pluginCollectOptions: {},
+            parserPluginsOverride: [overriddenPlugin],
+        });
+
+        expect(originalPlugin.collect).toHaveBeenCalledWith(
+            '',
+            expect.objectContaining<IntrinsicCollectOptions>({
+                tokenStream: expect.arrayContaining<Token>([
+                    expect.objectContaining({type: 'overridden'}),
+                ]),
+            }),
+        );
+
+        expect(originalPlugin.collect).not.toHaveBeenCalledWith(
+            '',
+            expect.objectContaining<IntrinsicCollectOptions>({
+                tokenStream: expect.arrayContaining<Token>([
+                    expect.objectContaining({type: 'original'}),
+                ]),
+            }),
+        );
+    });
+
+    it('never actually invokes collect functions that may exist in `parserPluginsOverride`', () => {
+        const mockPluginFactory = (tokenType: string) => {
+            const plugin: ExtendedPluginWithCollect = (md) =>
+                md.core.ruler.push('mock', (state) =>
+                    state.tokens.push(new state.Token(tokenType, 'mock', 0)),
+                );
+
+            plugin.collect = jest.fn(() => tokenType);
+
+            return plugin;
+        };
+
+        const originalPlugin = mockPluginFactory('original');
+        const overriddenPlugin = mockPluginFactory('overridden');
+
+        const result = transform.collect('', {
+            mdItInitOptions: {
+                plugins: [originalPlugin],
+            },
+            pluginCollectOptions: {},
+            parserPluginsOverride: [overriddenPlugin],
+        });
+
+        expect(result).toBe('original');
+        expect(overriddenPlugin.collect).not.toHaveBeenCalled();
+    });
+});