From 15931eb8949ae2618a167287ad72859acc3ab9c4 Mon Sep 17 00:00:00 2001 From: Maxim Karpov Date: Wed, 30 Oct 2024 17:23:08 +0300 Subject: [PATCH] feat: add async transform support --- src/cmd/build/index.ts | 104 ++++++++++++++++++----------- src/context/context.ts | 34 ++++++++++ src/context/fs.ts | 107 ++++++++++++++++++++++++++++++ src/models.ts | 16 ++++- src/resolvers/lintPage.ts | 52 +++++++++------ src/resolvers/md2html.ts | 64 ++++++++++-------- src/resolvers/md2md.ts | 62 +++++++++-------- src/services/leading.ts | 8 +-- src/services/plugins.ts | 9 ++- src/services/tocs.ts | 35 +++++----- src/steps/processAssets.ts | 85 +++++++++++++----------- src/steps/processExcludedFiles.ts | 2 +- src/steps/processLinter.ts | 46 +++++++------ src/steps/processMapFile.ts | 6 +- src/steps/processPages.ts | 101 +++++++++++++++++----------- src/steps/processServiceFiles.ts | 41 +++++++----- src/utils/common.ts | 11 +-- src/utils/file.ts | 64 ++++++++++++++---- src/workers/linter/index.ts | 27 +++++--- 19 files changed, 587 insertions(+), 287 deletions(-) create mode 100644 src/context/context.ts create mode 100644 src/context/fs.ts diff --git a/src/cmd/build/index.ts b/src/cmd/build/index.ts index 2c9f052e..540656e2 100644 --- a/src/cmd/build/index.ts +++ b/src/cmd/build/index.ts @@ -1,13 +1,14 @@ -import glob from 'glob'; import {Arguments, Argv} from 'yargs'; import {join, resolve} from 'path'; import shell from 'shelljs'; +import glob from 'glob'; import OpenapiIncluder from '@diplodoc/openapi-extension/includer'; -import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '../../constants'; -import {argvValidator} from '../../validator'; -import {ArgvService, Includers, SearchService} from '../../services'; +import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '~/constants'; +import {argvValidator} from '~/validator'; +import {Includer, YfmArgv} from '~/models'; +import {ArgvService, Includers, SearchService, TocService} from '~/services'; import { initLinterWorkers, processAssets, @@ -17,10 +18,12 @@ import { processLogs, processPages, processServiceFiles, -} from '../../steps'; -import {prepareMapFile} from '../../steps/processMapFile'; -import {copyFiles, logger} from '../../utils'; -import {upload as publishFilesToS3} from '../../commands/publish/upload'; +} from '~/steps'; +import {prepareMapFile} from '~/steps/processMapFile'; +import {copyFiles, logger} from '~/utils'; +import {upload as publishFilesToS3} from '~/commands/publish/upload'; +import {RevisionContext, makeRevisionContext} from '~/context/context'; +import {FsContextCli} from '~/context/fs'; export const build = { command: ['build', '$0'], @@ -43,6 +46,12 @@ function builder(argv: Argv) { type: 'string', group: 'Build options:', }) + .option('plugins', { + alias: 'p', + describe: 'Path to plugins js file', + type: 'string', + group: 'Build options:', + }) .option('varsPreset', { default: 'default', describe: 'Target vars preset of documentation ', @@ -175,7 +184,8 @@ function builder(argv: Argv) { ); } -async function handler(args: Arguments) { +async function handler(args: Arguments) { + const userInputFolder = resolve(args.input); const userOutputFolder = resolve(args.output); const tmpInputFolder = resolve(args.output, TMP_INPUT_FOLDER); const tmpOutputFolder = resolve(args.output, TMP_OUTPUT_FOLDER); @@ -185,14 +195,15 @@ async function handler(args: Arguments) { } try { + // Init singletone services ArgvService.init({ ...args, - rootInput: args.input, + rootInput: userInputFolder, input: tmpInputFolder, output: tmpOutputFolder, }); SearchService.init(); - Includers.init([OpenapiIncluder as any]); + Includers.init([OpenapiIncluder as Includer]); const { output: outputFolderPath, @@ -203,41 +214,64 @@ async function handler(args: Arguments) { addMapFile, } = ArgvService.getConfig(); - preparingTemporaryFolders(userOutputFolder); + const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER); - await processServiceFiles(); - processExcludedFiles(); + // Create build context that stores the information about the current build + const context = await makeRevisionContext( + userInputFolder, + userOutputFolder, + tmpInputFolder, + tmpOutputFolder, + outputBundlePath, + ); + const fs = new FsContextCli(context); + + // Creating temp .input & .output folder + await preparingTemporaryFolders(context); + + // Read and prepare Preset & Toc data + await processServiceFiles(context, fs); + + // Removes all content files that unspecified in toc files or ignored. + await processExcludedFiles(); + + // Write files.json if (addMapFile) { - prepareMapFile(); + await prepareMapFile(); } - const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER); + const navigationPaths = TocService.getNavigationPaths(); + // 1. Linting if (!lintDisabled) { /* Initialize workers in advance to avoid a timeout failure due to not receiving a message from them */ - await initLinterWorkers(); + await initLinterWorkers(navigationPaths); } const processes = [ - !lintDisabled && processLinter(), - !buildDisabled && processPages(outputBundlePath), + !lintDisabled && processLinter(context, navigationPaths), + !buildDisabled && processPages(fs, outputBundlePath, context), ].filter(Boolean) as Promise[]; await Promise.all(processes); + // 2. Building if (!buildDisabled) { - // process additional files - processAssets({ + // Process assets + await processAssets({ args, outputFormat, outputBundlePath, tmpOutputFolder, - userOutputFolder, + context, + fs, }); + // Process changelogs await processChangelogs(); + // Finish search service processing await SearchService.release(); // Copy all generated files to user' output folder @@ -246,6 +280,7 @@ async function handler(args: Arguments) { shell.cp('-r', join(tmpOutputFolder, '.*'), userOutputFolder); } + // Upload the files to S3 if (publish) { const DEFAULT_PREFIX = process.env.YFM_STORAGE_PREFIX ?? ''; const { @@ -273,31 +308,22 @@ async function handler(args: Arguments) { } catch (err) { logger.error('', err.message); } finally { + // Print logs processLogs(tmpInputFolder); shell.rm('-rf', tmpInputFolder, tmpOutputFolder); } } -function preparingTemporaryFolders(userOutputFolder: string) { - const args = ArgvService.getConfig(); - - shell.mkdir('-p', userOutputFolder); +// Creating temp .input & .output folder +async function preparingTemporaryFolders(context: RevisionContext) { + shell.mkdir('-p', context.userOutputFolder); // Create temporary input/output folders - shell.rm('-rf', args.input, args.output); - shell.mkdir(args.input, args.output); + shell.rm('-rf', context.tmpInputFolder, context.tmpOutputFolder); + shell.mkdir(context.tmpInputFolder, context.tmpOutputFolder); - copyFiles( - args.rootInput, - args.input, - glob.sync('**', { - cwd: args.rootInput, - nodir: true, - follow: true, - ignore: ['node_modules/**', '*/node_modules/**'], - }), - ); + await copyFiles(context.userInputFolder, context.tmpInputFolder, context.files); - shell.chmod('-R', 'u+w', args.input); + shell.chmod('-R', 'u+w', context.tmpInputFolder); } diff --git a/src/context/context.ts b/src/context/context.ts new file mode 100644 index 00000000..d6054202 --- /dev/null +++ b/src/context/context.ts @@ -0,0 +1,34 @@ +import {RevisionContext as RevisionContextTransfrom} from '@diplodoc/transform/lib/typings'; +import glob from 'glob'; + +export interface RevisionContext extends RevisionContextTransfrom { + userInputFolder: string; + userOutputFolder: string; + tmpInputFolder: string; + tmpOutputFolder: string; + outputBundlePath: string; +} + +export async function makeRevisionContext( + userInputFolder: string, + userOutputFolder: string, + tmpInputFolder: string, + tmpOutputFolder: string, + outputBundlePath: string, +): Promise { + const files = glob.sync('**', { + cwd: userInputFolder, + nodir: true, + follow: true, + ignore: ['node_modules/**', '*/node_modules/**'], + }); + + return { + userInputFolder, + userOutputFolder, + tmpInputFolder, + tmpOutputFolder, + outputBundlePath, + files, + }; +} diff --git a/src/context/fs.ts b/src/context/fs.ts new file mode 100644 index 00000000..24065e3c --- /dev/null +++ b/src/context/fs.ts @@ -0,0 +1,107 @@ +import {readFileSync, statSync, writeFileSync} from 'fs'; +import {readFile, stat, writeFile} from 'fs/promises'; +import {resolve} from 'path'; +import {FsContext} from '@diplodoc/transform/lib/typings'; +import {RevisionContext} from './context'; + +export function isFileExists(file: string) { + try { + const stats = statSync(file); + + return stats.isFile(); + } catch (e) { + return false; + } +} + +export async function isFileExistsAsync(file: string) { + try { + const stats = await stat(file); + + return stats.isFile(); + } catch (e) { + return false; + } +} + +export class FsContextCli implements FsContext { + private context: RevisionContext; + + constructor(context: RevisionContext) { + this.context = context; + } + + getPaths(path: string) { + const arr = [path]; + + const isFromTmpInputFolder = path.startsWith(resolve(this.context.tmpInputFolder) + '/'); + if (isFromTmpInputFolder) { + const assetPath = path.replace(resolve(this.context.tmpInputFolder) + '/', ''); + const originPath = resolve(this.context.userInputFolder, assetPath); + + arr.unshift(originPath); + } + + return arr; + } + + exist(path: string): boolean { + const paths = this.getPaths(path); + + for (const path of paths) { + if (isFileExists(path)) { + return true; + } + } + + return false; + } + + read(path: string): string { + const paths = this.getPaths(path); + + for (const path of paths) { + if (isFileExists(path)) { + return readFileSync(path, 'utf8'); + } + } + + throw Error(`File has not been found at: ${path}`); + } + + write(path: string, content: string): void { + writeFileSync(path, content, { + encoding: 'utf8', + }); + } + + async existAsync(path: string): Promise { + const paths = this.getPaths(path); + + for (const path of paths) { + if (await isFileExistsAsync(path)) { + return true; + } + } + + return false; + } + + async readAsync(path: string): Promise { + const paths = this.getPaths(path); + + for (const path of paths) { + if (await isFileExistsAsync(path)) { + return await readFile(path, 'utf8'); + } + } + + throw Error(`File has not been found at: ${path}`); + } + + async writeAsync(path: string, content: string): Promise { + await writeFile(path, content, { + encoding: 'utf8', + }); + } +} diff --git a/src/models.ts b/src/models.ts index 7b447965..ff52b4a1 100644 --- a/src/models.ts +++ b/src/models.ts @@ -6,6 +6,8 @@ import {LintConfig} from '@diplodoc/transform/lib/yfmlint'; import {IncludeMode, Lang, ResourceType, Stage} from './constants'; import {FileContributors, VCSConnector, VCSConnectorConfig} from './vcs-connector/connector-models'; +import {RevisionContext} from './context/context'; +import {FsContext} from '@diplodoc/transform/lib/typings'; export type VarsPreset = 'internal' | 'external'; @@ -26,7 +28,10 @@ export type NestedContributorsForPathFunction = ( nestedContributors: Contributors, ) => void; export type UserByLoginFunction = (login: string) => Promise; -export type CollectionOfPluginsFunction = (output: string, options: PluginOptions) => string; +export type CollectionOfPluginsFunction = ( + output: string, + options: PluginOptions, +) => Promise; export type GetModifiedTimeByPathFunction = (filepath: string) => number | undefined; /** @@ -58,6 +63,7 @@ interface YfmConfig { varsPreset: VarsPreset; ignore: string[]; outputFormat: string; + plugins: string; allowHTML: boolean; vars: Record; applyPresets: boolean; @@ -257,16 +263,20 @@ export interface PluginOptions { changelogs?: ChangelogItem[]; extractChangelogs?: boolean; included?: boolean; + context: RevisionContext; + fs?: FsContext; } export interface Plugin { - collect: (input: string, options: PluginOptions) => string | void; + collect: (input: string, options: PluginOptions) => Promise; } export interface ResolveMd2MdOptions { inputPath: string; outputPath: string; metadata: MetaDataOptions; + context: RevisionContext; + fs: FsContext; } export interface ResolverOptions { @@ -278,6 +288,8 @@ export interface ResolverOptions { outputPath: string; outputBundlePath: string; metadata?: MetaDataOptions; + context: RevisionContext; + fs: FsContext; } export interface PathData { diff --git a/src/resolvers/lintPage.ts b/src/resolvers/lintPage.ts index a98ed0b7..cd15cda0 100644 --- a/src/resolvers/lintPage.ts +++ b/src/resolvers/lintPage.ts @@ -10,23 +10,27 @@ import {isLocalUrl} from '@diplodoc/transform/lib/utils'; import {getLogLevel} from '@diplodoc/transform/lib/yfmlint/utils'; import {LINK_KEYS} from '@diplodoc/client/ssr'; -import {readFileSync} from 'fs'; import {bold} from 'chalk'; -import {ArgvService, PluginService} from '../services'; +import {FsContext} from '@diplodoc/transform/lib/typings'; +import {ArgvService, PluginService} from '~/services'; +import {RevisionContext} from '~/context/context'; +import {FsContextCli} from '~/context/fs'; import { checkPathExists, findAllValuesByKeys, getLinksWithExtension, getVarsPerFile, getVarsPerRelativeFile, -} from '../utils'; +} from '~/utils'; import {liquidMd2Html} from './md2html'; import {liquidMd2Md} from './md2md'; interface FileTransformOptions { path: string; root?: string; + context: RevisionContext; + fs: FsContext; } const FileLinter: Record = { @@ -38,22 +42,24 @@ export interface ResolverLintOptions { inputPath: string; fileExtension: string; onFinish?: () => void; + context: RevisionContext; } -export function lintPage(options: ResolverLintOptions) { - const {inputPath, fileExtension, onFinish} = options; +export async function lintPage(options: ResolverLintOptions) { + const {inputPath, fileExtension, onFinish, context} = options; const {input} = ArgvService.getConfig(); const resolvedPath: string = resolve(input, inputPath); + const fs = new FsContextCli(context); try { - const content: string = readFileSync(resolvedPath, 'utf8'); + const content: string = await fs.readAsync(resolvedPath); const lintFn: Function = FileLinter[fileExtension]; if (!lintFn) { return; } - lintFn(content, {path: inputPath}); + await lintFn(content, {path: inputPath, fs, context}); } catch (e) { const message = `No such file or has no access to ${bold(resolvedPath)}`; console.error(message, e); @@ -65,7 +71,7 @@ export function lintPage(options: ResolverLintOptions) { } } -function YamlFileLinter(content: string, lintOptions: FileTransformOptions): void { +async function YamlFileLinter(content: string, lintOptions: FileTransformOptions): Promise { const {input, lintConfig} = ArgvService.getConfig(); const {path: filePath} = lintOptions; const currentFilePath: string = resolve(input, filePath); @@ -76,21 +82,24 @@ function YamlFileLinter(content: string, lintOptions: FileTransformOptions): voi defaultLevel: log.LogLevels.ERROR, }); - const contentLinks = findAllValuesByKeys(load(content), LINK_KEYS); + const data = load(content) as object; + const contentLinks: string[] = findAllValuesByKeys(data, LINK_KEYS); const localLinks = contentLinks.filter( (link) => getLinksWithExtension(link) && isLocalUrl(link), ); - return localLinks.forEach( - (link) => - checkPathExists(link, currentFilePath) || - log[logLevel](`Link is unreachable: ${bold(link)} in ${bold(currentFilePath)}`), + await Promise.all( + localLinks.map( + async (link) => + (await checkPathExists(lintOptions.fs, link, currentFilePath)) || + log[logLevel](`Link is unreachable: ${bold(link)} in ${bold(currentFilePath)}`), + ), ); } -function MdFileLinter(content: string, lintOptions: FileTransformOptions): void { +async function MdFileLinter(content: string, lintOptions: FileTransformOptions): Promise { const {input, lintConfig, disableLiquid, outputFormat, ...options} = ArgvService.getConfig(); - const {path: filePath} = lintOptions; + const {path: filePath, fs} = lintOptions; const plugins = outputFormat === 'md' ? [] : PluginService.getPlugins(); const vars = getVarsPerFile(filePath); @@ -101,7 +110,7 @@ function MdFileLinter(content: string, lintOptions: FileTransformOptions): void /* Relative path from folder of .md file to root of user' output folder */ const assetsPublicPath = relative(dirname(path), root); - const lintMarkdown = function lintMarkdown(opts: LintMarkdownFunctionOptions) { + async function lintMarkdown(opts: LintMarkdownFunctionOptions) { const {input: localInput, path: localPath, sourceMap} = opts; const pluginOptions: PluginOptions = { @@ -114,9 +123,10 @@ function MdFileLinter(content: string, lintOptions: FileTransformOptions): void disableLiquid, log, getVarsPerFile: getVarsPerRelativeFile, + fs, }; - yfmlint({ + await yfmlint({ input: localInput, lintConfig, pluginOptions, @@ -125,22 +135,22 @@ function MdFileLinter(content: string, lintOptions: FileTransformOptions): void customLintRules: PluginService.getCustomLintRules(), sourceMap, }); - }; + } let sourceMap; if (!disableLiquid) { let liquidResult; if (outputFormat === 'md') { - liquidResult = liquidMd2Md(content, vars, path); + liquidResult = await liquidMd2Md(content, vars, path); } else { - liquidResult = liquidMd2Html(content, vars, path); + liquidResult = await liquidMd2Html(content, vars, path); } preparedContent = liquidResult.output; sourceMap = liquidResult.sourceMap; } - lintMarkdown({ + await lintMarkdown({ input: preparedContent, path, sourceMap, diff --git a/src/resolvers/md2html.ts b/src/resolvers/md2html.ts index 579ba40a..06307a37 100644 --- a/src/resolvers/md2html.ts +++ b/src/resolvers/md2html.ts @@ -1,21 +1,20 @@ -import type {DocInnerProps} from '@diplodoc/client'; - -import {readFileSync, writeFileSync} from 'fs'; import {basename, dirname, join, resolve, sep} from 'path'; -import {LINK_KEYS, preprocess} from '@diplodoc/client/ssr'; import {isString} from 'lodash'; +import yaml from 'js-yaml'; +import type {DocInnerProps} from '@diplodoc/client'; +import {LINK_KEYS, preprocess} from '@diplodoc/client/ssr'; import transform, {Output} from '@diplodoc/transform'; import liquid from '@diplodoc/transform/lib/liquid'; import log from '@diplodoc/transform/lib/log'; import {MarkdownItPluginCb} from '@diplodoc/transform/lib/plugins/typings'; import {getPublicPath, isFileExists} from '@diplodoc/transform/lib/utilsFS'; -import yaml from 'js-yaml'; +import {FsContext} from '@diplodoc/transform/lib/typings'; -import {Lang, PROCESSING_FINISHED} from '../constants'; -import {LeadingPage, ResolverOptions, YfmToc} from '../models'; -import {ArgvService, PluginService, SearchService, TocService} from '../services'; -import {getAssetsPublicPath, getAssetsRootPath, getVCSMetadata} from '../services/metadata'; +import {Lang, PROCESSING_FINISHED} from '~/constants'; +import {LeadingPage, ResolverOptions, YfmToc} from '~/models'; +import {ArgvService, PluginService, SearchService, TocService} from '~/services'; +import {getAssetsPublicPath, getAssetsRootPath, getVCSMetadata} from '~/services/metadata'; import { getLinksWithContentExtersion, getVarsPerFile, @@ -23,11 +22,15 @@ import { logger, modifyValuesByKeys, } from '../utils'; -import {generateStaticMarkup} from '../pages'; +import {RevisionContext} from '~/context/context'; +import {generateStaticMarkup} from '~/pages'; export interface FileTransformOptions { + lang: string; path: string; root?: string; + fs: FsContext; + context: RevisionContext; } const FileTransformer: Record = { @@ -39,14 +42,15 @@ const fixRelativePath = (relativeTo: string) => (path: string) => { return join(getAssetsPublicPath(relativeTo), path); }; -const getFileMeta = async ({fileExtension, metadata, inputPath}: ResolverOptions) => { +async function getFileMeta({fileExtension, metadata, inputPath, context, fs}: ResolverOptions) { const {input, allowCustomResources} = ArgvService.getConfig(); const resolvedPath: string = resolve(input, inputPath); - const content: string = readFileSync(resolvedPath, 'utf8'); + const content: string = await fs.readAsync(resolvedPath); const transformFn: Function = FileTransformer[fileExtension]; - const {result} = transformFn(content, {path: inputPath}); + + const {result} = await transformFn(content, {path: inputPath, context, fs}); const vars = getVarsPerFile(inputPath); const updatedMetadata = metadata?.isContributorsEnabled @@ -72,9 +76,9 @@ const getFileMeta = async ({fileExtension, metadata, inputPath}: ResolverOptions } return {...result, meta: fileMeta}; -}; +} -const getFileProps = async (options: ResolverOptions) => { +async function getFileProps(options: ResolverOptions) { const {inputPath, outputPath} = options; const pathToDir: string = dirname(inputPath); @@ -122,14 +126,14 @@ const getFileProps = async (options: ResolverOptions) => { }; return props; -}; +} export async function resolveMd2HTML(options: ResolverOptions): Promise { - const {outputPath, inputPath, deep, deepBase} = options; + const {outputPath, inputPath, deep, deepBase, fs} = options; const props = await getFileProps(options); const outputFileContent = generateStaticMarkup(props, deepBase, deep); - writeFileSync(outputPath, outputFileContent); + await fs.writeAsync(outputPath, outputFileContent); logger.info(inputPath, PROCESSING_FINISHED); return props; @@ -184,7 +188,10 @@ function getHref(path: string, href: string) { return href; } -function YamlFileTransformer(content: string, transformOptions: FileTransformOptions): Object { +async function YamlFileTransformer( + content: string, + transformOptions: FileTransformOptions, +): Promise { let data: LeadingPage | null = null; try { @@ -209,7 +216,7 @@ function YamlFileTransformer(content: string, transformOptions: FileTransformOpt const {path, lang} = transformOptions; const transformFn: Function = FileTransformer['.md']; - data = preprocess(data, {lang}, (lang, content) => { + data = await preprocess(data, {lang}, (lang, content) => { const {result} = transformFn(content, {path}); return result?.html; }); @@ -235,28 +242,31 @@ function YamlFileTransformer(content: string, transformOptions: FileTransformOpt }; } -export function liquidMd2Html(input: string, vars: Record, path: string) { +export async function liquidMd2Html(input: string, vars: Record, path: string) { const {conditionsInCode, useLegacyConditions} = ArgvService.getConfig(); - return liquid(input, vars, path, { + return await liquid(input, vars, path, { conditionsInCode, withSourceMap: true, useLegacyConditions, }); } -function MdFileTransformer(content: string, transformOptions: FileTransformOptions): Output { +async function MdFileTransformer( + content: string, + transformOptions: FileTransformOptions, +): Promise { const {input, ...options} = ArgvService.getConfig(); - const {path: filePath} = transformOptions; + const {path: filePath, context, fs} = transformOptions; const plugins = PluginService.getPlugins(); const vars = getVarsPerFile(filePath); const root = resolve(input); const path: string = resolve(input, filePath); - return transform(content, { + return await transform(content, { ...options, - plugins: plugins as MarkdownItPluginCb[], + plugins: plugins as MarkdownItPluginCb[], vars, root, path, @@ -265,5 +275,7 @@ function MdFileTransformer(content: string, transformOptions: FileTransformOptio getVarsPerFile: getVarsPerRelativeFile, getPublicPath, extractTitle: true, + context, + fs, }); } diff --git a/src/resolvers/md2md.ts b/src/resolvers/md2md.ts index 300913f3..7971cdd2 100644 --- a/src/resolvers/md2md.ts +++ b/src/resolvers/md2md.ts @@ -1,25 +1,24 @@ -import {readFileSync, writeFileSync} from 'fs'; import {basename, dirname, extname, join, resolve} from 'path'; import shell from 'shelljs'; import log from '@diplodoc/transform/lib/log'; import liquid from '@diplodoc/transform/lib/liquid'; +import {ChangelogItem} from '@diplodoc/transform/lib/plugins/changelog/types'; import {ArgvService, PluginService} from '../services'; import {getVarsPerFile, logger} from '../utils'; import {PluginOptions, ResolveMd2MdOptions} from '../models'; import {PROCESSING_FINISHED} from '../constants'; -import {ChangelogItem} from '@diplodoc/transform/lib/plugins/changelog/types'; import {enrichWithFrontMatter} from '../services/metadata'; export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise { - const {inputPath, outputPath, metadata: metadataOptions} = options; + const {inputPath, outputPath, metadata: metadataOptions, fs} = options; const {input, output, changelogs: changelogsSetting, included} = ArgvService.getConfig(); const resolvedInputPath = resolve(input, inputPath); const vars = getVarsPerFile(inputPath); const content = await enrichWithFrontMatter({ - fileContent: readFileSync(resolvedInputPath, 'utf8'), + fileContent: await fs.readAsync(resolvedInputPath), metadataOptions, resolvedFrontMatterVars: { systemVars: vars.__system as unknown, @@ -27,7 +26,21 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise }, }); - const {result, changelogs} = transformMd2Md(content, { + async function copyFile(targetPath: string, targetDestPath: string, options?: PluginOptions) { + shell.mkdir('-p', dirname(targetDestPath)); + + if (options) { + const sourceIncludeContent = await fs.readAsync(targetPath); + const {result} = await transformMd2Md(sourceIncludeContent, options); + + await fs.writeAsync(targetDestPath, result); + } else { + shell.cp(targetPath, targetDestPath); + } + } + + const {result, changelogs} = await transformMd2Md(content, { + ...options, path: resolvedInputPath, destPath: outputPath, root: resolve(input), @@ -35,16 +48,21 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise collectOfPlugins: PluginService.getCollectOfPlugins(), vars: vars, log, - copyFile, included, + copyFile, + context: options.context, + fs: options.fs, + deps: options.deps, }); - writeFileSync(outputPath, result); + await fs.writeAsync(outputPath, result); if (changelogsSetting && changelogs?.length) { const mdFilename = basename(outputPath, extname(outputPath)); const outputDir = dirname(outputPath); - changelogs.forEach((changes, index) => { + + let index = 0; + for (const changes of changelogs) { let changesName; const changesDate = changes.date as string | undefined; const changesIdx = changes.index as number | undefined; @@ -63,14 +81,16 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise const changesPath = join(outputDir, `__changes-${changesName}.json`); - writeFileSync( + await fs.writeAsync( changesPath, JSON.stringify({ ...changes, source: mdFilename, }), ); - }); + + index++; + } } logger.info(inputPath, PROCESSING_FINISHED); @@ -78,23 +98,11 @@ export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise return undefined; } -function copyFile(targetPath: string, targetDestPath: string, options?: PluginOptions) { - shell.mkdir('-p', dirname(targetDestPath)); - - if (options) { - const sourceIncludeContent = readFileSync(targetPath, 'utf8'); - const {result} = transformMd2Md(sourceIncludeContent, options); - writeFileSync(targetDestPath, result); - } else { - shell.cp(targetPath, targetDestPath); - } -} - -export function liquidMd2Md(input: string, vars: Record, path: string) { +export async function liquidMd2Md(input: string, vars: Record, path: string) { const {applyPresets, resolveConditions, conditionsInCode, useLegacyConditions} = ArgvService.getConfig(); - return liquid(input, vars, path, { + return await liquid(input, vars, path, { conditions: resolveConditions, substitutions: applyPresets, conditionsInCode, @@ -104,7 +112,7 @@ export function liquidMd2Md(input: string, vars: Record, path: }); } -function transformMd2Md(input: string, options: PluginOptions) { +async function transformMd2Md(input: string, options: PluginOptions) { const {disableLiquid, changelogs: changelogsSetting} = ArgvService.getConfig(); const {vars = {}, path, collectOfPlugins, log: pluginLog} = options; @@ -112,13 +120,13 @@ function transformMd2Md(input: string, options: PluginOptions) { const changelogs: ChangelogItem[] = []; if (!disableLiquid) { - const liquidResult = liquidMd2Md(input, vars, path); + const liquidResult = await liquidMd2Md(input, vars, path); output = liquidResult.output; } if (collectOfPlugins) { - output = collectOfPlugins(output, { + output = await collectOfPlugins(output, { ...options, vars, path, diff --git a/src/services/leading.ts b/src/services/leading.ts index d785a996..a8366c20 100644 --- a/src/services/leading.ts +++ b/src/services/leading.ts @@ -1,5 +1,5 @@ import {dirname, resolve} from 'path'; -import {readFileSync, writeFileSync} from 'fs'; +import {readFile, writeFile} from 'fs/promises'; import {dump, load} from 'js-yaml'; import log from '@diplodoc/transform/lib/log'; @@ -13,12 +13,12 @@ import { liquidFields, } from './utils'; -function filterFile(path: string) { +async function filterFile(path: string) { const {input: inputFolderPath, vars} = ArgvService.getConfig(); const pathToDir = dirname(path); const filePath = resolve(inputFolderPath, path); - const content = readFileSync(filePath, 'utf8'); + const content = await readFile(filePath, 'utf8'); const parsedIndex = load(content) as LeadingPage; const combinedVars = { @@ -74,7 +74,7 @@ function filterFile(path: string) { } }); - writeFileSync(filePath, dump(parsedIndex)); + await writeFile(filePath, dump(parsedIndex)); } catch (error) { log.error(`Error while filtering index file: ${path}. Error message: ${error}`); } diff --git a/src/services/plugins.ts b/src/services/plugins.ts index b941a0ab..410c026a 100644 --- a/src/services/plugins.ts +++ b/src/services/plugins.ts @@ -24,14 +24,13 @@ function makeCollectOfPlugins(): CollectionOfPluginsFunction { return typeof plugin.collect === 'function'; }); - return (output: string, options: PluginOptions) => { + return async (output: string, options: PluginOptions) => { let collectsOutput = output; - pluginsWithCollect.forEach((plugin: Plugin) => { - const collectOutput = plugin.collect(collectsOutput, options); - + for (const plugin of pluginsWithCollect) { + const collectOutput = await plugin.collect(collectsOutput, options); collectsOutput = typeof collectOutput === 'string' ? collectOutput : collectsOutput; - }); + } return collectsOutput; }; diff --git a/src/services/tocs.ts b/src/services/tocs.ts index 3d335dbc..dcdccfd4 100644 --- a/src/services/tocs.ts +++ b/src/services/tocs.ts @@ -1,10 +1,10 @@ import {dirname, extname, join, normalize, parse, relative, resolve, sep} from 'path'; -import {existsSync, readFileSync, writeFileSync} from 'fs'; import {dump, load} from 'js-yaml'; import shell from 'shelljs'; import walkSync from 'walk-sync'; import {liquidSnippet} from '@diplodoc/transform/lib/liquid'; import log from '@diplodoc/transform/lib/log'; +import {FsContext} from '@diplodoc/transform/lib/typings'; import {bold} from 'chalk'; import {ArgvService, PresetService} from './index'; @@ -22,13 +22,16 @@ export interface TocServiceData { includedTocPaths: Set; } +let fsContext: FsContext; const storage: TocServiceData['storage'] = new Map(); const tocs: TocServiceData['tocs'] = new Map(); let navigationPaths: TocServiceData['navigationPaths'] = []; const includedTocPaths: TocServiceData['includedTocPaths'] = new Set(); const tocFileCopyMap = new Map(); -async function init(tocFilePaths: string[]) { +async function init(fs: FsContext, tocFilePaths: string[]) { + fsContext = fs; + for (const path of tocFilePaths) { logger.proc(path); @@ -58,7 +61,7 @@ async function add(path: string) { } = ArgvService.getConfig(); const pathToDir = dirname(path); - const content = readFileSync(resolve(inputFolderPath, path), 'utf8'); + const content = await fsContext.readAsync(resolve(inputFolderPath, path)); const parsedToc = load(content) as YfmToc; // Should ignore toc with specified stage. @@ -114,7 +117,7 @@ async function add(path: string) { const outputPath = resolve(outputFolderPath, path); const outputToc = dump(parsedToc); shell.mkdir('-p', dirname(outputPath)); - writeFileSync(outputPath, outputToc); + await fsContext.writeAsync(outputPath, outputToc); } } @@ -258,7 +261,7 @@ function _normalizeHref(href: string): string { * @return * @private */ -function _copyTocDir(tocPath: string, destDir: string) { +async function _copyTocDir(tocPath: string, destDir: string) { const {input: inputFolderPath} = ArgvService.getConfig(); const {dir: tocDir} = parse(tocPath); @@ -268,7 +271,7 @@ function _copyTocDir(tocPath: string, destDir: string) { directories: false, }); - files.forEach((relPath) => { + for (const relPath of files) { const from = resolve(tocDir, relPath); const to = resolve(destDir, relPath); const fileExtension = extname(relPath); @@ -277,11 +280,11 @@ function _copyTocDir(tocPath: string, destDir: string) { shell.mkdir('-p', parse(to).dir); if (isMdFile) { - const fileContent = readFileSync(from, 'utf8'); + const fileContent = await fsContext.readAsync(from); const sourcePath = relative(inputFolderPath, from); const updatedFileContent = addSourcePath(fileContent, sourcePath); - writeFileSync(to, updatedFileContent); + await fsContext.writeAsync(to, updatedFileContent); } else { shell.cp(from, to); } @@ -289,7 +292,7 @@ function _copyTocDir(tocPath: string, destDir: string) { const relFrom = relative(inputFolderPath, from); const relTo = relative(inputFolderPath, to); tocFileCopyMap.set(relTo, relFrom); - }); + } } /** @@ -392,7 +395,7 @@ async function _replaceIncludes( const includeTocDir = dirname(includeTocPath); try { - const includeToc = load(readFileSync(includeTocPath, 'utf8')) as YfmToc; + const includeToc = load(await fsContext.readAsync(includeTocPath)) as YfmToc; // Should ignore included toc with tech-preview stage. if (includeToc.stage === Stage.TECH_PREVIEW) { @@ -400,7 +403,7 @@ async function _replaceIncludes( } if (mode === IncludeMode.MERGE || mode === IncludeMode.ROOT_MERGE) { - _copyTocDir(includeTocPath, tocDir); + await _copyTocDir(includeTocPath, tocDir); } /* Save the path to exclude toc from the output directory in the next step */ @@ -457,21 +460,23 @@ async function _replaceIncludes( return result; } -function getTocDir(pagePath: string): string { +async function getTocDir(pagePath: string, pageBasePath?: string): Promise { + pageBasePath = pageBasePath ?? pagePath; + const {input: inputFolderPath} = ArgvService.getConfig(); const tocDir = dirname(pagePath); const tocPath = resolve(tocDir, 'toc.yaml'); if (!tocDir.includes(inputFolderPath)) { - throw new Error('Error while finding toc dir'); + throw new Error(`Error while finding toc dir for "${pageBasePath}"`); } - if (existsSync(tocPath)) { + if (await fsContext.existAsync(tocPath)) { return tocDir; } - return getTocDir(tocDir); + return await getTocDir(tocDir, pageBasePath); } function setNavigationPaths(paths: TocServiceData['navigationPaths']) { diff --git a/src/steps/processAssets.ts b/src/steps/processAssets.ts index f7337228..c330b2dc 100644 --- a/src/steps/processAssets.ts +++ b/src/steps/processAssets.ts @@ -1,23 +1,23 @@ import walkSync from 'walk-sync'; import {load} from 'js-yaml'; -import {readFileSync} from 'fs'; import shell from 'shelljs'; import {join, resolve, sep} from 'path'; -import {ArgvService, TocService} from '../services'; -import {checkPathExists, copyFiles, findAllValuesByKeys} from '../utils'; - import {LINK_KEYS} from '@diplodoc/client/ssr'; import {isLocalUrl} from '@diplodoc/transform/lib/utils'; +import {resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; +import {FsContext} from '@diplodoc/transform/lib/typings'; import { ASSETS_FOLDER, LINT_CONFIG_FILENAME, REDIRECTS_FILENAME, YFM_CONFIG_FILENAME, -} from '../constants'; -import {Resources} from '../models'; -import {resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; +} from '~/constants'; +import {ArgvService, TocService} from '~/services'; +import {checkPathExists, copyFiles, findAllValuesByKeys} from '~/utils'; +import {Resources, YfmArgv} from '~/models'; +import {RevisionContext} from '~/context/context'; /** * @param {Array} args @@ -28,26 +28,29 @@ import {resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; */ type Props = { - args: string[]; + args: YfmArgv; outputBundlePath: string; outputFormat: string; tmpOutputFolder: string; + context: RevisionContext; + fs: FsContext; }; + /* * Processes assets files (everything except .md files) */ -export function processAssets({args, outputFormat, outputBundlePath, tmpOutputFolder}: Props) { - switch (outputFormat) { +export async function processAssets(props: Props) { + switch (props.outputFormat) { case 'html': - processAssetsHtmlRun({outputBundlePath}); + await processAssetsHtmlRun(props); break; case 'md': - processAssetsMdRun({args, tmpOutputFolder}); + await processAssetsMdRun(props); break; } } -function processAssetsHtmlRun({outputBundlePath}) { +async function processAssetsHtmlRun({outputBundlePath}: Props) { const {input: inputFolderPath, output: outputFolderPath} = ArgvService.getConfig(); const documentationAssetFilePath: string[] = walkSync(inputFolderPath, { @@ -56,17 +59,17 @@ function processAssetsHtmlRun({outputBundlePath}) { ignore: ['**/*.yaml', '**/*.md'], }); - copyFiles(inputFolderPath, outputFolderPath, documentationAssetFilePath); + await copyFiles(inputFolderPath, outputFolderPath, documentationAssetFilePath); const bundleAssetFilePath: string[] = walkSync(ASSETS_FOLDER, { directories: false, includeBasePath: false, }); - copyFiles(ASSETS_FOLDER, outputBundlePath, bundleAssetFilePath); + await copyFiles(ASSETS_FOLDER, outputBundlePath, bundleAssetFilePath); } -function processAssetsMdRun({args, tmpOutputFolder}) { +async function processAssetsMdRun({args, tmpOutputFolder, fs}: Props) { const {input: inputFolderPath, allowCustomResources, resources} = ArgvService.getConfig(); const pathToConfig = args.config || join(args.input, YFM_CONFIG_FILENAME); @@ -85,11 +88,11 @@ function processAssetsMdRun({args, tmpOutputFolder}) { resources[type as keyof Resources]?.forEach((path: string) => resourcePaths.push(path)), ); - //copy resources - copyFiles(args.input, tmpOutputFolder, resourcePaths); + // Сopy resources + await copyFiles(args.input, tmpOutputFolder, resourcePaths); } - const tocYamlFiles = TocService.getNavigationPaths().reduce((acc, file) => { + const tocYamlFiles = TocService.getNavigationPaths().reduce((acc, file) => { if (file.endsWith('.yaml')) { const resolvedPathToFile = resolve(inputFolderPath, file); @@ -98,32 +101,34 @@ function processAssetsMdRun({args, tmpOutputFolder}) { return acc; }, []); - tocYamlFiles.forEach((yamlFile) => { - const content = load(readFileSync(yamlFile, 'utf8')); + for (const yamlFile of tocYamlFiles) { + const text = await fs.readAsync(yamlFile); + const content = load(text) as object; if (!Object.prototype.hasOwnProperty.call(content, 'blocks')) { return; } const contentLinks = findAllValuesByKeys(content, LINK_KEYS); - const localMediaLinks = contentLinks.reduce( - (acc, link) => { - const linkHasMediaExt = new RegExp( - /^\S.*\.(svg|png|gif|jpg|jpeg|bmp|webp|ico)$/gm, - ).test(link); - - if (linkHasMediaExt && isLocalUrl(link) && checkPathExists(link, yamlFile)) { - const linkAbsolutePath = resolveRelativePath(yamlFile, link); - const linkRootPath = linkAbsolutePath.replace(`${inputFolderPath}${sep}`, ''); - - acc.push(linkRootPath); - } - return acc; - }, - - [], - ); + const localMediaLinks = []; + + for (const link of contentLinks) { + const linkHasMediaExt = new RegExp( + /^\S.*\.(svg|png|gif|jpg|jpeg|bmp|webp|ico)$/gm, + ).test(link); + + if ( + linkHasMediaExt && + isLocalUrl(link) && + (await checkPathExists(fs, link, yamlFile)) + ) { + const linkAbsolutePath = resolveRelativePath(yamlFile, link); + const linkRootPath = linkAbsolutePath.replace(`${inputFolderPath}${sep}`, ''); + + localMediaLinks.push(linkRootPath); + } + } - copyFiles(args.input, tmpOutputFolder, localMediaLinks); - }); + await copyFiles(args.input, tmpOutputFolder, localMediaLinks); + } } diff --git a/src/steps/processExcludedFiles.ts b/src/steps/processExcludedFiles.ts index f34f8420..ba795f7b 100644 --- a/src/steps/processExcludedFiles.ts +++ b/src/steps/processExcludedFiles.ts @@ -9,7 +9,7 @@ import {convertBackSlashToSlash} from '../utils'; * Removes all content files that unspecified in toc files or ignored. * @return {void} */ -export function processExcludedFiles() { +export async function processExcludedFiles() { const {input: inputFolderPath, output: outputFolderPath, ignore} = ArgvService.getConfig(); const allContentFiles: string[] = walkSync(inputFolderPath, { diff --git a/src/steps/processLinter.ts b/src/steps/processLinter.ts index 13c3c050..aa6007b5 100644 --- a/src/steps/processLinter.ts +++ b/src/steps/processLinter.ts @@ -2,24 +2,25 @@ import log from '@diplodoc/transform/lib/log'; import {Thread, Worker, spawn} from 'threads'; import {extname} from 'path'; -import {ArgvService, PluginService, PresetService, TocService} from '../services'; -import {ProcessLinterWorker} from '../workers/linter'; -import {logger} from '../utils'; -import {LINTING_FINISHED, MIN_CHUNK_SIZE, WORKERS_COUNT} from '../constants'; -import {lintPage} from '../resolvers'; -import {splitOnChunks} from '../utils/worker'; +import {LINTING_FINISHED, MIN_CHUNK_SIZE, WORKERS_COUNT} from '~/constants'; +import {ArgvService, PluginService, PresetService} from '~/services'; +import {ProcessLinterWorker} from '~/workers/linter'; +import {logger} from '~/utils'; +import {lintPage} from '~/resolvers'; +import {splitOnChunks} from '~/utils/worker'; +import {RevisionContext} from '~/context/context'; let processLinterWorkers: (ProcessLinterWorker & Thread)[]; -let navigationPathsChunks: string[][]; +let filesToProcessChunks: string[][]; -export async function processLinter(): Promise { +export async function processLinter( + context: RevisionContext, + filesToProcess: string[], +): Promise { const argvConfig = ArgvService.getConfig(); - const navigationPaths = TocService.getNavigationPaths(); - if (!processLinterWorkers) { - lintPagesFallback(navigationPaths); - + await lintPagesFallback(filesToProcess, context); return; } @@ -35,12 +36,13 @@ export async function processLinter(): Promise { /* Run processing the linter */ await Promise.all( processLinterWorkers.map((worker, i) => { - const navigationPathsChunk = navigationPathsChunks[i]; + const navigationPathsChunk = filesToProcessChunks[i]; return worker.run({ argvConfig, presetStorage, navigationPaths: navigationPathsChunk, + context, }); }), ); @@ -62,17 +64,16 @@ export async function processLinter(): Promise { ); } -export async function initLinterWorkers() { - const navigationPaths = TocService.getNavigationPaths(); - const chunkSize = getChunkSize(navigationPaths); +export async function initLinterWorkers(filesToProcess: string[]) { + const chunkSize = getChunkSize(filesToProcess); if (process.env.DISABLE_PARALLEL_BUILD || chunkSize < MIN_CHUNK_SIZE || WORKERS_COUNT <= 0) { return; } - navigationPathsChunks = splitOnChunks(navigationPaths, chunkSize).filter((arr) => arr.length); + filesToProcessChunks = splitOnChunks(filesToProcess, chunkSize).filter((arr) => arr.length); - const workersCount = navigationPathsChunks.length; + const workersCount = filesToProcessChunks.length; processLinterWorkers = await Promise.all( new Array(workersCount).fill(null).map(() => { @@ -86,16 +87,17 @@ function getChunkSize(arr: string[]) { return Math.ceil(arr.length / WORKERS_COUNT); } -function lintPagesFallback(navigationPaths: string[]) { +async function lintPagesFallback(filesToProcess: string[], context: RevisionContext) { PluginService.setPlugins(); - navigationPaths.forEach((pathToFile) => { - lintPage({ + for (const pathToFile of filesToProcess) { + await lintPage({ inputPath: pathToFile, fileExtension: extname(pathToFile), onFinish: () => { logger.info(pathToFile, LINTING_FINISHED); }, + context, }); - }); + } } diff --git a/src/steps/processMapFile.ts b/src/steps/processMapFile.ts index 5dc9c7d6..5f3c6216 100644 --- a/src/steps/processMapFile.ts +++ b/src/steps/processMapFile.ts @@ -1,4 +1,4 @@ -import {writeFileSync} from 'fs'; +import {writeFile} from 'fs/promises'; import {extname, join} from 'path'; import {ArgvService, TocService} from '../services'; @@ -12,7 +12,7 @@ type TocItem = { type TocItems = TocItem[]; -export function prepareMapFile(): void { +export async function prepareMapFile(): Promise { const {output: outputFolderPath} = ArgvService.getConfig(); const navigationPathsWithoutExtensions = TocService.getNavigationPaths().map((path) => { @@ -28,5 +28,5 @@ export function prepareMapFile(): void { const filesMapBuffer = Buffer.from(JSON.stringify(navigationPaths, null, '\t'), 'utf8'); const mapFile = join(outputFolderPath, 'files.json'); - writeFileSync(mapFile, filesMapBuffer); + await writeFile(mapFile, filesMapBuffer); } diff --git a/src/steps/processPages.ts b/src/steps/processPages.ts index abe7c05b..7bb860d1 100644 --- a/src/steps/processPages.ts +++ b/src/steps/processPages.ts @@ -1,6 +1,5 @@ import type {DocInnerProps} from '@diplodoc/client'; import {basename, dirname, extname, join, relative, resolve} from 'path'; -import {existsSync, readFileSync, writeFileSync} from 'fs'; import log from '@diplodoc/transform/lib/log'; import {asyncify, mapLimit} from 'async'; import {bold} from 'chalk'; @@ -8,13 +7,14 @@ import {dump, load} from 'js-yaml'; import shell from 'shelljs'; import dedent from 'ts-dedent'; +import {FsContext} from '@diplodoc/transform/lib/typings'; import { Lang, PAGE_PROCESS_CONCURRENCY, ResourceType, SINGLE_PAGE_DATA_FILENAME, SINGLE_PAGE_FILENAME, -} from '../constants'; +} from '~/constants'; import { LeadingPage, MetaDataOptions, @@ -22,20 +22,24 @@ import { Resources, SinglePageResult, YfmToc, -} from '../models'; -import {resolveMd2HTML, resolveMd2Md} from '../resolvers'; -import {ArgvService, LeadingService, PluginService, SearchService, TocService} from '../services'; -import {generateStaticMarkup} from '~/pages/document'; -import {generateStaticRedirect} from '~/pages/redirect'; -import {joinSinglePageResults, logger, transformToc, transformTocForSinglePage} from '../utils'; -import {getVCSConnector} from '../vcs-connector'; -import {VCSConnector} from '../vcs-connector/connector-models'; +} from '~/models'; +import {resolveMd2HTML, resolveMd2Md} from '~/resolvers'; +import {ArgvService, LeadingService, PluginService, SearchService, TocService} from '~/services'; +import {joinSinglePageResults, logger, transformToc, transformTocForSinglePage} from '~/utils'; +import {generateStaticMarkup, generateStaticRedirect} from '~/pages'; +import {getVCSConnector} from '~/vcs-connector'; +import {VCSConnector} from '~/vcs-connector/connector-models'; +import {RevisionContext} from '~/context/context'; const singlePageResults: Record = {}; const singlePagePaths: Record> = {}; // Processes files of documentation (like index.yaml, *.md) -export async function processPages(outputBundlePath: string): Promise { +export async function processPages( + fs: FsContext, + outputBundlePath: string, + context: RevisionContext, +): Promise { const { input: inputFolderPath, output: outputFolderPath, @@ -54,7 +58,7 @@ export async function processPages(outputBundlePath: string): Promise { navigationPaths, PAGE_PROCESS_CONCURRENCY, asyncify(async (pathToFile: string) => { - const pathData = getPathData( + const pathData = await getPathData( pathToFile, inputFolderPath, outputFolderPath, @@ -67,31 +71,33 @@ export async function processPages(outputBundlePath: string): Promise { const metaDataOptions = getMetaDataOptions(pathData, vcsConnector); await preparingPagesByOutputFormat( + fs, pathData, metaDataOptions, resolveConditions, singlePage, + context, ); }), ); if (singlePage) { - await saveSinglePages(); + await saveSinglePages(fs); } if (outputFormat === 'html') { - saveRedirectPage(outputFolderPath); - await saveTocData(); + await saveRedirectPage(fs, outputFolderPath); + await saveTocData(fs); } } -function getPathData( +async function getPathData( pathToFile: string, inputFolderPath: string, outputFolderPath: string, outputFormat: string, outputBundlePath: string, -): PathData { +): Promise { const pathToDir: string = dirname(pathToFile); const filename: string = basename(pathToFile); const fileExtension: string = extname(pathToFile); @@ -100,7 +106,8 @@ function getPathData( const outputFileName = `${fileBaseName}.${outputFormat}`; const outputPath = resolve(outputDir, outputFileName); const resolvedPathToFile = resolve(inputFolderPath, pathToFile); - const outputTocDir = TocService.getTocDir(resolvedPathToFile); + + const outputTocDir = await TocService.getTocDir(resolvedPathToFile); const pathData: PathData = { pathToFile, @@ -120,13 +127,13 @@ function getPathData( return pathData; } -async function saveTocData() { +async function saveTocData(fs: FsContext) { const tocs = TocService.getAllTocs(); const {output} = ArgvService.getConfig(); for (const [path, toc] of tocs) { const outputPath = join(output, path.replace(/\.yaml$/, '.js')); - writeFileSync( + await fs.writeAsync( outputPath, dedent` window.__DATA__.data.toc = ${JSON.stringify(transformToc(toc))}; @@ -136,7 +143,7 @@ async function saveTocData() { } } -async function saveSinglePages() { +async function saveSinglePages(fs: FsContext) { const { input: inputFolderPath, lang: configLang, @@ -189,8 +196,8 @@ async function saveSinglePages() { toc?.root?.deepBase || toc?.deepBase || 0, ); - writeFileSync(singlePageFn, singlePageContent); - writeFileSync(singlePageDataFn, JSON.stringify(pageData)); + await fs.writeAsync(singlePageFn, singlePageContent); + await fs.writeAsync(singlePageDataFn, JSON.stringify(pageData)); }), ); } catch (error) { @@ -198,7 +205,7 @@ async function saveSinglePages() { } } -function saveRedirectPage(outputDir: string): void { +async function saveRedirectPage(fs: FsContext, outputDir: string): Promise { const {lang, langs} = ArgvService.getConfig(); const redirectLang = lang || langs?.[0] || Lang.RU; @@ -207,9 +214,9 @@ function saveRedirectPage(outputDir: string): void { const redirectPagePath = join(outputDir, 'index.html'); const redirectLangPath = join(outputDir, redirectLangRelativePath); - if (!existsSync(redirectPagePath) && existsSync(redirectLangPath)) { + if (!(await fs.existAsync(redirectPagePath)) && (await fs.existAsync(redirectLangPath))) { const content = generateStaticRedirect(redirectLang, redirectLangRelativePath); - writeFileSync(redirectPagePath, content); + await fs.writeAsync(redirectPagePath, content); } } @@ -263,10 +270,12 @@ function getMetaDataOptions(pathData: PathData, vcsConnector?: VCSConnector): Me } async function preparingPagesByOutputFormat( + fs: FsContext, path: PathData, metaDataOptions: MetaDataOptions, resolveConditions: boolean, singlePage: boolean, + context: RevisionContext, ): Promise { const { filename, @@ -285,11 +294,11 @@ async function preparingPagesByOutputFormat( const isYamlFileExtension = fileExtension === '.yaml'; if (resolveConditions && fileBaseName === 'index' && isYamlFileExtension) { - LeadingService.filterFile(pathToFile); + await LeadingService.filterFile(pathToFile); } if (outputFormat === 'md' && isYamlFileExtension && allowCustomResources) { - processingYamlFile(path, metaDataOptions); + await processingYamlFile(fs, path, metaDataOptions); return; } @@ -297,16 +306,21 @@ async function preparingPagesByOutputFormat( (outputFormat === 'md' && isYamlFileExtension) || (outputFormat === 'html' && !isYamlFileExtension && fileExtension !== '.md') ) { - copyFileWithoutChanges(resolvedPathToFile, outputDir, filename); + await copyFileWithoutChanges(resolvedPathToFile, outputDir, filename); return; } switch (outputFormat) { case 'md': - await processingFileToMd(path, metaDataOptions); + await processingFileToMd(path, metaDataOptions, context, fs); return; case 'html': { - const resolvedFileProps = await processingFileToHtml(path, metaDataOptions); + const resolvedFileProps = await processingFileToHtml( + path, + metaDataOptions, + context, + fs, + ); SearchService.add(pathToFile, resolvedFileProps); @@ -323,45 +337,54 @@ async function preparingPagesByOutputFormat( log.error(message); } } -//@ts-ignore -function processingYamlFile(path: PathData, metaDataOptions: MetaDataOptions) { + +async function processingYamlFile(fs: FsContext, path: PathData, metaDataOptions: MetaDataOptions) { const {pathToFile, outputFolderPath, inputFolderPath} = path; const filePath = resolve(inputFolderPath, pathToFile); - const content = readFileSync(filePath, 'utf8'); + const content = await fs.readAsync(filePath); const parsedContent = load(content) as LeadingPage; if (metaDataOptions.resources) { parsedContent.meta = {...parsedContent.meta, ...metaDataOptions.resources}; } - writeFileSync(resolve(outputFolderPath, pathToFile), dump(parsedContent)); + await fs.writeAsync(resolve(outputFolderPath, pathToFile), dump(parsedContent)); } -function copyFileWithoutChanges( +async function copyFileWithoutChanges( resolvedPathToFile: string, outputDir: string, filename: string, -): void { +): Promise { const from = resolvedPathToFile; const to = resolve(outputDir, filename); shell.cp(from, to); } -async function processingFileToMd(path: PathData, metaDataOptions: MetaDataOptions): Promise { +async function processingFileToMd( + path: PathData, + metaDataOptions: MetaDataOptions, + context: RevisionContext, + fs: FsContext, +): Promise { const {outputPath, pathToFile} = path; await resolveMd2Md({ inputPath: pathToFile, outputPath, metadata: metaDataOptions, + context, + fs, }); } async function processingFileToHtml( path: PathData, metaDataOptions: MetaDataOptions, + context: RevisionContext, + fs: FsContext, ): Promise { const {outputBundlePath, filename, fileExtension, outputPath, pathToFile} = path; const {deepBase, deep} = TocService.getDeepForPath(pathToFile); @@ -375,5 +398,7 @@ async function processingFileToHtml( metadata: metaDataOptions, deep, deepBase, + context, + fs, }); } diff --git a/src/steps/processServiceFiles.ts b/src/steps/processServiceFiles.ts index 7c315264..ebe8c1d3 100644 --- a/src/steps/processServiceFiles.ts +++ b/src/steps/processServiceFiles.ts @@ -1,18 +1,19 @@ import {dirname, resolve} from 'path'; -import walkSync from 'walk-sync'; -import {readFileSync, writeFileSync} from 'fs'; import {dump, load} from 'js-yaml'; import log from '@diplodoc/transform/lib/log'; import {ArgvService, PresetService, TocService} from '../services'; -import {logger} from '../utils'; +import {logger, walkFolders} from '../utils'; import {DocPreset} from '../models'; import shell from 'shelljs'; +import {FsContext} from '@diplodoc/transform/lib/typings'; +import {RevisionContext} from '~/context/context'; -const getFilePathsByGlobals = (globs: string[]): string[] => { - const {input, ignore = []} = ArgvService.getConfig(); +const getFilePathsByGlobals = (globs: string[], context: RevisionContext): string[] => { + const {ignore = []} = ArgvService.getConfig(); - return walkSync(input, { + return walkFolders({ + folder: [context.tmpInputFolder, context.userInputFolder], directories: false, includeBasePath: false, globs, @@ -20,12 +21,12 @@ const getFilePathsByGlobals = (globs: string[]): string[] => { }); }; -export async function processServiceFiles(): Promise { - await preparingPresetFiles(); - await preparingTocFiles(); +export async function processServiceFiles(context: RevisionContext, fs: FsContext): Promise { + await preparingPresetFiles(context, fs); + await preparingTocFiles(context, fs); } -async function preparingPresetFiles() { +async function preparingPresetFiles(context: RevisionContext, fs: FsContext) { const { input: inputFolderPath, varsPreset = '', @@ -35,20 +36,20 @@ async function preparingPresetFiles() { } = ArgvService.getConfig(); try { - const presetsFilePaths = getFilePathsByGlobals(['**/presets.yaml']); + const presetsFilePaths = getFilePathsByGlobals(['**/presets.yaml'], context); for (const path of presetsFilePaths) { logger.proc(path); const pathToPresetFile = resolve(inputFolderPath, path); - const content = readFileSync(pathToPresetFile, 'utf8'); + const content = await fs.readAsync(pathToPresetFile); const parsedPreset = load(content) as DocPreset; PresetService.add(parsedPreset, path, varsPreset); if (outputFormat === 'md' && (!applyPresets || !resolveConditions)) { // Should save filtered presets.yaml only when --apply-presets=false or --resolve-conditions=false - saveFilteredPresets(path, parsedPreset); + await saveFilteredPresets(fs, path, parsedPreset); } } } catch (error) { @@ -57,7 +58,11 @@ async function preparingPresetFiles() { } } -function saveFilteredPresets(path: string, parsedPreset: DocPreset): void { +async function saveFilteredPresets( + fs: FsContext, + path: string, + parsedPreset: DocPreset, +): Promise { const {output: outputFolderPath, varsPreset = ''} = ArgvService.getConfig(); const outputPath = resolve(outputFolderPath, path); @@ -74,13 +79,13 @@ function saveFilteredPresets(path: string, parsedPreset: DocPreset): void { }); shell.mkdir('-p', dirname(outputPath)); - writeFileSync(outputPath, outputPreset); + await fs.writeAsync(outputPath, outputPreset); } -async function preparingTocFiles(): Promise { +async function preparingTocFiles(context: RevisionContext, fs: FsContext): Promise { try { - const tocFilePaths = getFilePathsByGlobals(['**/toc.yaml']); - await TocService.init(tocFilePaths); + const tocFilePaths = getFilePathsByGlobals(['**/toc.yaml'], context); + await TocService.init(fs, tocFilePaths); } catch (error) { log.error(`Preparing toc.yaml files failed. Error: ${error}`); throw error; diff --git a/src/utils/common.ts b/src/utils/common.ts index c2c3782c..6e29c189 100644 --- a/src/utils/common.ts +++ b/src/utils/common.ts @@ -1,5 +1,6 @@ import {cloneDeepWith, flatMapDeep, isArray, isObject, isString} from 'lodash'; -import {isFileExists, resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; +import {resolveRelativePath} from '@diplodoc/transform/lib/utilsFS'; +import {FsContext} from '@diplodoc/transform/lib/typings'; export function findAllValuesByKeys(obj: object, keysToFind: string[]) { return flatMapDeep(obj, (value: string | string[], key: string) => { @@ -21,10 +22,10 @@ export function findAllValuesByKeys(obj: object, keysToFind: string[]) { export function modifyValuesByKeys( originalObj: object, keysToFind: string[], - modifyFn: (value: string) => string, + modifyFn: (value: string) => string | undefined, ) { // Clone the object deeply with a customizer function that modifies matching keys - return cloneDeepWith(originalObj, function (value: unknown, key) { + return cloneDeepWith(originalObj, (value: unknown, key) => { if (keysToFind.includes(key as string) && isString(value)) { return modifyFn(value); } @@ -45,8 +46,8 @@ export function getLinksWithExtension(link: string) { return oneLineWithExtension.test(link); } -export function checkPathExists(path: string, parentFilePath: string) { +export async function checkPathExists(fs: FsContext, path: string, parentFilePath: string) { const includePath = resolveRelativePath(parentFilePath, path); - return isFileExists(includePath); + return await fs.existAsync(includePath); } diff --git a/src/utils/file.ts b/src/utils/file.ts index c599f84f..50cdd526 100644 --- a/src/utils/file.ts +++ b/src/utils/file.ts @@ -1,26 +1,66 @@ import {dirname, resolve} from 'path'; +import {copyFile} from 'node:fs/promises'; import shell from 'shelljs'; +import walkSync from 'walk-sync'; import {logger} from './logger'; -export function copyFiles( +export async function copyFiles( inputFolderPath: string, outputFolderPath: string, files: string[], -): void { +) { + if (files.length === 0) { + return; + } + const dirs = new Set(); - files.forEach((pathToAsset) => { - const outputDir = resolve(outputFolderPath, dirname(pathToAsset)); - const from = resolve(inputFolderPath, pathToAsset); - const to = resolve(outputFolderPath, pathToAsset); + for (const pathToAsset of files) { + try { + const from = resolve(inputFolderPath, pathToAsset); + const to = resolve(outputFolderPath, pathToAsset); + const outputDir = resolve(outputFolderPath, dirname(pathToAsset)); + + if (!dirs.has(outputDir)) { + dirs.add(outputDir); + shell.mkdir('-p', outputDir); + } + + await copyFile(from, to); - if (!dirs.has(outputDir)) { - dirs.add(outputDir); - shell.mkdir('-p', outputDir); + logger.copy(pathToAsset); + } catch (error) { + logger.error(pathToAsset, error.message); } + } +} + +export function walkFolders({ + folder, + globs, + ignore, + directories, + includeBasePath, +}: { + folder?: string | string[]; + globs?: string[]; + ignore?: string[]; + directories?: boolean; + includeBasePath?: boolean; +}) { + if (!Array.isArray(folder) && folder) { + folder = [folder]; + } - shell.cp(from, to); + const dirs = [...(folder || [])].filter(Boolean) as string[]; + const files = dirs.map((folder) => + walkSync(folder as string, { + directories, + includeBasePath, + globs, + ignore, + }), + ); - logger.copy(pathToAsset); - }); + return [...new Set(files.flat())]; } diff --git a/src/workers/linter/index.ts b/src/workers/linter/index.ts index 969ee487..633c3e7f 100644 --- a/src/workers/linter/index.ts +++ b/src/workers/linter/index.ts @@ -3,11 +3,12 @@ import {extname} from 'path'; import {Observable, Subject} from 'threads/observable'; import {expose} from 'threads'; -import {ArgvService, PluginService, PresetService, TocService} from '../../services'; -import {TocServiceData} from '../../services/tocs'; -import {PresetStorage} from '../../services/preset'; -import {YfmArgv} from '../../models'; -import {lintPage} from '../../resolvers'; +import {ArgvService, PluginService, PresetService, TocService} from '~/services'; +import {TocServiceData} from '~/services/tocs'; +import {PresetStorage} from '~/services/preset'; +import {YfmArgv} from '~/models'; +import {lintPage} from '~/resolvers'; +import {RevisionContext} from '~/context/context'; let processedPages = new Subject(); @@ -15,23 +16,31 @@ interface ProcessLinterWorkerOptions { argvConfig: YfmArgv; navigationPaths: TocServiceData['navigationPaths']; presetStorage: PresetStorage; + context: RevisionContext; } -async function run({argvConfig, presetStorage, navigationPaths}: ProcessLinterWorkerOptions) { +async function run({ + argvConfig, + presetStorage, + navigationPaths, + context, +}: ProcessLinterWorkerOptions) { ArgvService.set(argvConfig); PresetService.setPresetStorage(presetStorage); TocService.setNavigationPaths(navigationPaths); PluginService.setPlugins(); - TocService.getNavigationPaths().forEach((pathToFile) => { - lintPage({ + for (const pathToFile of TocService.getNavigationPaths()) { + await lintPage({ inputPath: pathToFile, fileExtension: extname(pathToFile), + context, + // eslint-disable-next-line @typescript-eslint/no-loop-func onFinish: () => { processedPages.next(pathToFile); }, }); - }); + } } async function finish() {