diff --git a/src/cmd/build/index.ts b/src/cmd/build/index.ts index c8ab36938..70a7a0bed 100644 --- a/src/cmd/build/index.ts +++ b/src/cmd/build/index.ts @@ -13,7 +13,8 @@ import {join, resolve} from 'path'; import {ArgvService, Includers} from '../../services'; import OpenapiIncluder from '@diplodoc/openapi-extension/includer'; import { - initLinterWorkers, initProcessWorkers, + initLinterWorkers, + initProcessWorkers, processAssets, processExcludedFiles, processLinter, diff --git a/src/resolvers/lintPage.ts b/src/resolvers/lintPage.ts index d8bd4f94a..5b2023cd3 100644 --- a/src/resolvers/lintPage.ts +++ b/src/resolvers/lintPage.ts @@ -1,10 +1,10 @@ import {dirname, relative, resolve} from 'path'; -import log from '@doc-tools/transform/lib/log'; +import log from '@diplodoc/transform/lib/log'; import { default as yfmlint, LintMarkdownFunctionOptions, PluginOptions, -} from '@doc-tools/transform/lib/yfmlint'; +} from '@diplodoc/transform/lib/yfmlint'; import {bold} from 'chalk'; import {ArgvService, PluginService} from '../services'; @@ -54,13 +54,7 @@ export async function lintPage(options: ResolverLintOptions) { } function MdFileLinter(content: string, lintOptions: FileTransformOptions): void { - const { - input, - lintConfig, - disableLiquid, - outputFormat, - ...options - } = ArgvService.getConfig(); + const {input, lintConfig, disableLiquid, outputFormat, ...options} = ArgvService.getConfig(); const {path: filePath} = lintOptions; const plugins = outputFormat === 'md' ? [] : PluginService.getPlugins(); diff --git a/src/resolvers/processPage.ts b/src/resolvers/processPage.ts index b622108ff..ce82acea8 100644 --- a/src/resolvers/processPage.ts +++ b/src/resolvers/processPage.ts @@ -6,7 +6,7 @@ import {VCSConnector} from '../vcs-connector/connector-models'; import {LeadingPage, MetaDataOptions, PathData, Resources, SinglePageResult} from '../models'; import {DocInnerProps} from '@diplodoc/client'; import {bold} from 'chalk'; -import log from '@doc-tools/transform/lib/log'; +import log from '@diplodoc/transform/lib/log'; import * as fs from 'fs'; import {dump, load} from 'js-yaml'; import {resolveMd2Md} from './md2md'; @@ -37,7 +37,13 @@ export async function processPage(options: ProcessPageOptions) { const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER); - const pathData = getPathData(pathToFile, inputFolderPath, outputFolderPath, outputFormat, outputBundlePath); + const pathData = getPathData( + pathToFile, + inputFolderPath, + outputFolderPath, + outputFormat, + outputBundlePath, + ); logger.proc(pathToFile); @@ -46,7 +52,6 @@ export async function processPage(options: ProcessPageOptions) { await preparingPagesByOutputFormat(pathData, metaDataOptions, resolveConditions, singlePage); } - function getPathData( pathToFile: string, inputFolderPath: string, @@ -105,7 +110,10 @@ function savePageResultForSinglePage(pageProps: DocInnerProps, pathData: PathDat }); } -function getMetaDataOptions(pathData: PathData, inputFolderPathLength: number, vcsConnector?: VCSConnector, +function getMetaDataOptions( + pathData: PathData, + inputFolderPathLength: number, + vcsConnector?: VCSConnector, ): MetaDataOptions { const {contributors, addSystemMeta, resources, allowCustomResources} = ArgvService.getConfig(); @@ -120,7 +128,6 @@ function getMetaDataOptions(pathData: PathData, inputFolderPathLength: number, v addSystemMeta, }; - if (allowCustomResources && resources) { const allowedResources = Object.entries(resources).reduce((acc: Resources, [key, val]) => { if (Object.keys(ResourceType).includes(key)) { @@ -132,7 +139,6 @@ function getMetaDataOptions(pathData: PathData, inputFolderPathLength: number, v metaDataOptions.resources = allowedResources; } - return metaDataOptions; } @@ -167,8 +173,10 @@ async function preparingPagesByOutputFormat( return; } - if (outputFormat === 'md' && isYamlFileExtension || - outputFormat === 'html' && !isYamlFileExtension && fileExtension !== '.md') { + if ( + (outputFormat === 'md' && isYamlFileExtension) || + (outputFormat === 'html' && !isYamlFileExtension && fileExtension !== '.md') + ) { await copyFileWithoutChanges(resolvedPathToFile, outputDir, filename); return; } @@ -207,7 +215,11 @@ async function processingYamlFile(path: PathData, metaDataOptions: MetaDataOptio await fs.promises.writeFile(resolve(outputFolderPath, pathToFile), dump(parsedContent)); } -async function copyFileWithoutChanges(resolvedPathToFile: string, outputDir: string, filename: string) { +async function copyFileWithoutChanges( + resolvedPathToFile: string, + outputDir: string, + filename: string, +) { const from = resolvedPathToFile; const to = resolve(outputDir, filename); @@ -224,14 +236,11 @@ async function processingFileToMd(path: PathData, metaDataOptions: MetaDataOptio }); } -async function processingFileToHtml(path: PathData, metaDataOptions: MetaDataOptions): Promise { - const { - outputBundlePath, - filename, - fileExtension, - outputPath, - pathToFile, - } = path; +async function processingFileToHtml( + path: PathData, + metaDataOptions: MetaDataOptions, +): Promise { + const {outputBundlePath, filename, fileExtension, outputPath, pathToFile} = path; return resolveMd2HTML({ inputPath: pathToFile, diff --git a/src/services/leading.ts b/src/services/leading.ts index 0f21d6b89..2b4bc377c 100644 --- a/src/services/leading.ts +++ b/src/services/leading.ts @@ -1,17 +1,20 @@ import {dirname, resolve} from 'path'; import {dump, load} from 'js-yaml'; -import log from '@doc-tools/transform/lib/log'; +import log from '@diplodoc/transform/lib/log'; import {ArgvService, PresetService} from './index'; import {LeadingPage} from '../models'; -import {filterTextItems, filterFiles, firstFilterTextItems, liquidFields, liquidField} from './utils'; +import { + filterTextItems, + filterFiles, + firstFilterTextItems, + liquidFields, + liquidField, +} from './utils'; import * as fs from 'fs'; async function filterFile(path: string) { - const { - input: inputFolderPath, - vars, - } = ArgvService.getConfig(); + const {input: inputFolderPath, vars} = ArgvService.getConfig(); const pathToDir = dirname(path); const filePath = resolve(inputFolderPath, path); @@ -25,26 +28,20 @@ async function filterFile(path: string) { /* Should remove all links with false expressions */ try { - const title = firstFilterTextItems( - parsedIndex.title, - combinedVars, - {resolveConditions: true}, - ); + const title = firstFilterTextItems(parsedIndex.title, combinedVars, { + resolveConditions: true, + }); parsedIndex.title = liquidField(title, combinedVars, path); - const description = filterTextItems( - parsedIndex.description, - combinedVars, - {resolveConditions: true}, - ); + const description = filterTextItems(parsedIndex.description, combinedVars, { + resolveConditions: true, + }); parsedIndex.description = liquidFields(description, combinedVars, path); if (parsedIndex.meta?.title) { - const metaTitle = firstFilterTextItems( - parsedIndex.meta.title, - combinedVars, - {resolveConditions: true}, - ); + const metaTitle = firstFilterTextItems(parsedIndex.meta.title, combinedVars, { + resolveConditions: true, + }); parsedIndex.meta.title = liquidField(metaTitle, combinedVars, path); } @@ -58,15 +55,15 @@ async function filterFile(path: string) { } if (parsedIndex.nav) { - const navTitle = firstFilterTextItems( - parsedIndex.nav.title, - combinedVars, - {resolveConditions: true}, - ); + const navTitle = firstFilterTextItems(parsedIndex.nav.title, combinedVars, { + resolveConditions: true, + }); parsedIndex.nav.title = liquidField(navTitle, combinedVars, path); } - parsedIndex.links = filterFiles(parsedIndex.links, 'links', combinedVars, {resolveConditions: true}); + parsedIndex.links = filterFiles(parsedIndex.links, 'links', combinedVars, { + resolveConditions: true, + }); parsedIndex.links.forEach((link) => { if (link.title) { diff --git a/src/services/tocs.ts b/src/services/tocs.ts index 99eb78cd2..1c54276b4 100644 --- a/src/services/tocs.ts +++ b/src/services/tocs.ts @@ -409,7 +409,11 @@ function dumpData() { }; } -function loadData({storageKeyValue, includedTocPathsArr, navigationPaths: navigationPathsLocal}: TocServiceDataDump) { +function loadData({ + storageKeyValue, + includedTocPathsArr, + navigationPaths: navigationPathsLocal, +}: TocServiceDataDump) { navigationPaths.splice(0); navigationPaths.push(...navigationPathsLocal); diff --git a/src/steps/processPages.ts b/src/steps/processPages.ts index 35b391f4e..8803f3f46 100644 --- a/src/steps/processPages.ts +++ b/src/steps/processPages.ts @@ -1,10 +1,15 @@ import {join, relative, resolve} from 'path'; import {writeFileSync} from 'fs'; -import log from '@doc-tools/transform/lib/log'; +import log from '@diplodoc/transform/lib/log'; import {ArgvService, PluginService, PresetService, TocService} from '../services'; -import {generateStaticMarkup, joinSinglePageResults, logger, transformTocForSinglePage} from '../utils'; +import { + generateStaticMarkup, + joinSinglePageResults, + logger, + transformTocForSinglePage, +} from '../utils'; import {SinglePageResult, YfmToc} from '../models'; import { BUNDLE_FOLDER, @@ -28,16 +33,13 @@ const singlePageResults: Record = {}; const singlePagePaths: Record> = {}; let processTransformWorkers: (ProcessTransformWorker & Thread)[]; -let navigationPathsChunks: (string[])[]; +let navigationPathsChunks: string[][]; // Processes files of documentation (like index.yaml, *.md) export async function processPages(vcsConnector?: VCSConnector): Promise { const argvConfig = ArgvService.getConfig(); - const { - output: outputFolderPath, - singlePage, - } = argvConfig; + const {output: outputFolderPath, singlePage} = argvConfig; const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER); @@ -84,25 +86,33 @@ export async function processPages(vcsConnector?: VCSConnector): Promise { /* Unsubscribe from workers */ await Promise.all( processTransformWorkers.map((worker) => { - return worker.finish().then(({logs, singlePagePathsSerialized, singlePageResults: singlePageResultsLocal}) => { - Object.entries(singlePageResultsLocal).forEach(([key, values]) => { - let arr = singlePageResults[key]; - if (!arr) { - arr = singlePageResults[key] = []; - } - arr.push(...values); - }); - - Object.entries(singlePagePathsSerialized).forEach(([key, values]) => { - let setMap = singlePagePaths[key]; - if (!setMap) { - setMap = new Set(); - singlePagePaths[key] = setMap; - } - values.forEach((p) => setMap.add(p)); - }); - log.add(logs); - }); + return worker + .finish() + .then( + ({ + logs, + singlePagePathsSerialized, + singlePageResults: singlePageResultsLocal, + }) => { + Object.entries(singlePageResultsLocal).forEach(([key, values]) => { + let arr = singlePageResults[key]; + if (!arr) { + arr = singlePageResults[key] = []; + } + arr.push(...values); + }); + + Object.entries(singlePagePathsSerialized).forEach(([key, values]) => { + let setMap = singlePagePaths[key]; + if (!setMap) { + setMap = new Set(); + singlePagePaths[key] = setMap; + } + values.forEach((p) => setMap.add(p)); + }); + log.add(logs); + }, + ); }), ); @@ -128,44 +138,50 @@ async function saveSinglePages(outputBundlePath: string) { } = ArgvService.getConfig(); try { - await Promise.all(Object.keys(singlePageResults).map(async (tocDir) => { - if (!singlePageResults[tocDir].length) { - return; - } - - const singlePageBody = joinSinglePageResults(singlePageResults[tocDir], inputFolderPath, tocDir); - const tocPath = join(relative(inputFolderPath, tocDir), 'toc.yaml'); - const toc: YfmToc|null = TocService.getForPath(tocPath) || null; - const preparedToc = transformTocForSinglePage(toc, { - root: inputFolderPath, - currentPath: join(tocDir, SINGLE_PAGE_FILENAME), - }) as YfmToc; - - const pageData = { - data: { - leading: false as const, - html: singlePageBody, - headings: [], - meta: resources || {}, - toc: preparedToc, - }, - router: { - pathname: SINGLE_PAGE_FILENAME, - }, - lang: lang || Lang.RU, - } as unknown as Parameters[0]; - - const outputTocDir = resolve(outputFolderPath, relative(inputFolderPath, tocDir)); - const relativeOutputBundlePath = relative(outputTocDir, outputBundlePath); - - // Save the full single page for viewing locally - const singlePageFn = join(tocDir, SINGLE_PAGE_FILENAME); - const singlePageDataFn = join(tocDir, SINGLE_PAGE_DATA_FILENAME); - const singlePageContent = generateStaticMarkup(pageData, relativeOutputBundlePath); - - writeFileSync(singlePageFn, singlePageContent); - writeFileSync(singlePageDataFn, JSON.stringify(pageData)); - })); + await Promise.all( + Object.keys(singlePageResults).map(async (tocDir) => { + if (!singlePageResults[tocDir].length) { + return; + } + + const singlePageBody = joinSinglePageResults( + singlePageResults[tocDir], + inputFolderPath, + tocDir, + ); + const tocPath = join(relative(inputFolderPath, tocDir), 'toc.yaml'); + const toc: YfmToc | null = TocService.getForPath(tocPath) || null; + const preparedToc = transformTocForSinglePage(toc, { + root: inputFolderPath, + currentPath: join(tocDir, SINGLE_PAGE_FILENAME), + }) as YfmToc; + + const pageData = { + data: { + leading: false as const, + html: singlePageBody, + headings: [], + meta: resources || {}, + toc: preparedToc, + }, + router: { + pathname: SINGLE_PAGE_FILENAME, + }, + lang: lang || Lang.RU, + } as unknown as Parameters[0]; + + const outputTocDir = resolve(outputFolderPath, relative(inputFolderPath, tocDir)); + const relativeOutputBundlePath = relative(outputTocDir, outputBundlePath); + + // Save the full single page for viewing locally + const singlePageFn = join(tocDir, SINGLE_PAGE_FILENAME); + const singlePageDataFn = join(tocDir, SINGLE_PAGE_DATA_FILENAME); + const singlePageContent = generateStaticMarkup(pageData, relativeOutputBundlePath); + + writeFileSync(singlePageFn, singlePageContent); + writeFileSync(singlePageDataFn, JSON.stringify(pageData)); + }), + ); } catch (error) { console.log(error); } @@ -179,28 +195,36 @@ export async function initProcessWorkers() { return; } - navigationPathsChunks = splitOnChunks(navigationPaths, chunkSize) - .filter((arr) => arr.length); + navigationPathsChunks = splitOnChunks(navigationPaths, chunkSize).filter((arr) => arr.length); const workersCount = navigationPathsChunks.length; - processTransformWorkers = await Promise.all((new Array(workersCount)).fill(null).map(() => { - // TODO: get transform path from env - return spawn(new Worker('./transform'), {timeout: 60000}); - })); + processTransformWorkers = await Promise.all( + new Array(workersCount).fill(null).map(() => { + // TODO: get transform path from env + return spawn(new Worker('./transform'), {timeout: 60000}); + }), + ); } -async function processPagesFallback(vcsConnector: VCSConnector | undefined, navigationPaths: string[]) { +async function processPagesFallback( + vcsConnector: VCSConnector | undefined, + navigationPaths: string[], +) { PluginService.setPlugins(); const concurrency = 500; - await mapLimit(navigationPaths, concurrency, asyncify(async (pathToFile: string) => { - await processPage({ - pathToFile, - vcsConnector, - singlePageResults, - singlePagePaths, - }); - })); + await mapLimit( + navigationPaths, + concurrency, + asyncify(async (pathToFile: string) => { + await processPage({ + pathToFile, + vcsConnector, + singlePageResults, + singlePagePaths, + }); + }), + ); } diff --git a/src/vcs-connector/github.ts b/src/vcs-connector/github.ts index f1f0974f0..7148ca8d6 100644 --- a/src/vcs-connector/github.ts +++ b/src/vcs-connector/github.ts @@ -5,16 +5,13 @@ import {minimatch} from 'minimatch'; import github from './client/github'; import {ArgvService} from '../services'; -import { - CommitInfo, - Contributor, - Contributors, -} from '../models'; +import {CommitInfo, Contributor, Contributors} from '../models'; import { FileContributors, GitHubConnectorFields, SourceType, - VCSConnector, VCSConnectorDump, + VCSConnector, + VCSConnectorDump, } from './connector-models'; import { ALL_CONTRIBUTORS_RECEIVED, @@ -34,10 +31,14 @@ const userLoginGithubUserCache = new Map(); function getGitHubVCSConnector(): VCSConnector | undefined { const {contributors} = ArgvService.getConfig(); - if (!contributors) { return undefined; } + if (!contributors) { + return undefined; + } const httpClientByToken = getHttpClientByToken(); - if (!httpClientByToken) { return undefined; } + if (!httpClientByToken) { + return undefined; + } return { init: async () => await getAllContributorsTocFiles(httpClientByToken), @@ -114,7 +115,14 @@ async function getAllContributorsTocFiles(httpClientByToken: Octokit): Promise { +async function matchContributionsForEachPath( + repoLogs: string[], + httpClientByToken: Octokit, +): Promise { for (const repoLog of repoLogs) { if (!repoLog) { continue; @@ -167,7 +178,10 @@ async function matchContributionsForEachPath(repoLogs: string[], httpClientByTok if (hasContributorData === undefined) { logger.info('Contributors: Getting data for', email); - contributorDataByHash = await getContributorDataByHashCommit(httpClientByToken, hashCommit); + contributorDataByHash = await getContributorDataByHashCommit( + httpClientByToken, + hashCommit, + ); if (contributorDataByHash) { const paths = dataArray.splice(1); @@ -203,7 +217,9 @@ async function matchAuthorsForEachPath(authorRepoLogs: string[], httpClientByTok } } -async function getContributorDataByHashCommit(httpClientByToken: Octokit, hashCommit: string, +async function getContributorDataByHashCommit( + httpClientByToken: Octokit, + hashCommit: string, ): Promise { const repoCommit = await github.getRepoCommitByHash(httpClientByToken, hashCommit); @@ -228,7 +244,11 @@ async function getContributorDataByHashCommit(httpClientByToken: Octokit, hashCo }; } -async function getAuthorByPaths(commitInfo: CommitInfo, paths: string[], httpClientByToken: Octokit) { +async function getAuthorByPaths( + commitInfo: CommitInfo, + paths: string[], + httpClientByToken: Octokit, +) { for (const path of paths) { if (!path) { continue; @@ -294,7 +314,11 @@ async function getUserByLogin(octokit: Octokit, userLogin: string): Promise { const normalizePath = normalize(addSlashPrefix(path)); diff --git a/src/workers/linter/index.ts b/src/workers/linter/index.ts index 4f9d83fa8..3f3c481bc 100644 --- a/src/workers/linter/index.ts +++ b/src/workers/linter/index.ts @@ -1,4 +1,4 @@ -import log from '@doc-tools/transform/lib/log'; +import log from '@diplodoc/transform/lib/log'; import {extname} from 'path'; import {Observable, Subject} from 'threads/observable'; import {expose} from 'threads'; @@ -18,11 +18,7 @@ interface ProcessLinterWorkerOptions { presetStorageDump: PresetStorageDump; } -async function run({ - argvConfig, - navigationPaths, - presetStorageDump, -}: ProcessLinterWorkerOptions) { +async function run({argvConfig, navigationPaths, presetStorageDump}: ProcessLinterWorkerOptions) { ArgvService.set(argvConfig); PresetService.load(presetStorageDump); TocService.setNavigationPaths(navigationPaths); @@ -31,15 +27,19 @@ async function run({ const concurrency = 500; - await mapLimit(navigationPaths, concurrency, asyncify(async (pathToFile: string) => { - await lintPage({ - inputPath: pathToFile, - fileExtension: extname(pathToFile), - onFinish: () => { - processedPages.next(pathToFile); - }, - }); - })); + await mapLimit( + navigationPaths, + concurrency, + asyncify(async (pathToFile: string) => { + await lintPage({ + inputPath: pathToFile, + fileExtension: extname(pathToFile), + onFinish: () => { + processedPages.next(pathToFile); + }, + }); + }), + ); } async function finish() { diff --git a/src/workers/transform/index.ts b/src/workers/transform/index.ts index 7475e16ed..375403f18 100644 --- a/src/workers/transform/index.ts +++ b/src/workers/transform/index.ts @@ -1,4 +1,4 @@ -import log from '@doc-tools/transform/lib/log'; +import log from '@diplodoc/transform/lib/log'; import {Observable, Subject} from 'threads/observable'; import {expose} from 'threads'; @@ -41,23 +41,29 @@ async function run({ const vcsConnector = createVCSConnector(); if (vcsConnectorDump && vcsConnector) { vcsConnector.load(vcsConnectorDump); - vcsConnector.getUserByLogin = mainBridge.createFn('vcsConnector.getUserByLogin'); + vcsConnector.getUserByLogin = mainBridge.createFn( + 'vcsConnector.getUserByLogin', + ); } PluginService.setPlugins(); const concurrency = 500; - await mapLimit(navigationPaths, concurrency, asyncify(async (pathToFile: string) => { - await processPage({ - pathToFile, - vcsConnector, - singlePageResults, - singlePagePaths, - }).finally(() => { - processedPages.next({type: 'path', path: pathToFile}); - }); - })); + await mapLimit( + navigationPaths, + concurrency, + asyncify(async (pathToFile: string) => { + await processPage({ + pathToFile, + vcsConnector, + singlePageResults, + singlePagePaths, + }).finally(() => { + processedPages.next({type: 'path', path: pathToFile}); + }); + }), + ); } async function finish() { diff --git a/src/workers/transform/mainBridge.ts b/src/workers/transform/mainBridge.ts index bcaaf3b25..99bc375f3 100644 --- a/src/workers/transform/mainBridge.ts +++ b/src/workers/transform/mainBridge.ts @@ -3,10 +3,17 @@ type Callback = (payload: ReceivePayload) => void; type ReplyFn = (id: number, payload: ReceivePayload) => void; export type SendPayload = {type: 'call'; id: number; method: string; args: unknown[]}; -export type ReceivePayload = {result?: unknown; error?: {name: string; message: string; stack?: string}}; +export type ReceivePayload = { + result?: unknown; + error?: {name: string; message: string; stack?: string}; +}; export class MainBridge { - static handleCall = async (worker: {reply: ReplyFn}, data: SendPayload, scope: Record) => { + static handleCall = async ( + worker: {reply: ReplyFn}, + data: SendPayload, + scope: Record, + ) => { const {id, method, args} = data; let result; let error;