From d41ec353f364fce761f3b444b1dc850d7c64a414 Mon Sep 17 00:00:00 2001 From: Li0liQ Date: Mon, 29 Mar 2021 18:38:53 +1100 Subject: [PATCH] Do not cache file contents in memory, version bump --- CHANGELOG.md | 4 ++++ package.json | 2 +- src/file-processor/index.ts | 9 ++++---- src/file-processor/parse-imports.test.ts | 16 +++++++------- src/utils/index.ts | 6 +++++- src/utils/test.ts | 27 +++++++++++++++++++++++- 6 files changed, 49 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68f5d4d1..85279ea2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Master +## 0.4.7 - Decrease memory consumption + +- Do not cache file contents in memory for faster ast parsing; this will significantly decrease memory requirements for larger codebases + ## 0.4.6 - Faster dependency resolution - Bump dependencies diff --git a/package.json b/package.json index 10b8145b..7735f71a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stricter", - "version": "0.4.6", + "version": "0.4.7", "description": "A project-wide js-linting tool", "files": [ "LICENSE", diff --git a/src/file-processor/index.ts b/src/file-processor/index.ts index 34ef36d4..e6bdc8c4 100644 --- a/src/file-processor/index.ts +++ b/src/file-processor/index.ts @@ -38,7 +38,8 @@ const readFileData = ( logger: Logger, ): FileData => { const source = readFile(filePath); - const ast = parsedExtensionsRe.test(filePath) ? () => parse(source, filePath) : undefined; + const isParsedExtension = parsedExtensionsRe.test(filePath); + const getAst = isParsedExtension ? () => parse(filePath) : undefined; let dependencies: string[] | undefined; const hash = getHash(source); @@ -47,10 +48,10 @@ const readFileData = ( if (cachedValue && cachedValue.hash === hash) { dependencies = cachedValue.dependencies; } else { - if (ast) { + if (isParsedExtension) { let parsedAst: any; try { - parsedAst = ast(); + parsedAst = parse(filePath, source); } catch (e) { logger.error(`Unable to parse ${filePath}`); throw e; @@ -62,7 +63,7 @@ const readFileData = ( const result = { source, - ast, + ast: getAst, dependencies, }; diff --git a/src/file-processor/parse-imports.test.ts b/src/file-processor/parse-imports.test.ts index 44861e0d..eb5a3d45 100644 --- a/src/file-processor/parse-imports.test.ts +++ b/src/file-processor/parse-imports.test.ts @@ -5,10 +5,10 @@ describe('resolveImport', () => { it('should count require', () => { const result = parseImports( parse( + 'filePath', ` const test1 = require('test-file1'); `, - 'filePath', ), ); @@ -19,10 +19,10 @@ describe('resolveImport', () => { it('should count es6 dynamic import', () => { const result = parseImports( parse( + 'filePath', ` const test1 = import('test-file1'); `, - 'filePath', ), ); @@ -33,13 +33,13 @@ describe('resolveImport', () => { it('should count es6 imports', () => { const result = parseImports( parse( + 'filePath', ` import { default as test1 } from 'test-file1'; import { test2, test3 } from 'test-file2'; import test4 from 'test-file3'; import * as test5 from 'test-file4'; `, - 'filePath', ), ); @@ -55,12 +55,12 @@ describe('resolveImport', () => { it('should count es6 reexports', () => { const result = parseImports( parse( + 'filePath', ` export { default as test1 } from 'test-file1'; export { test2, test3 } from 'test-file2'; export * from 'test-file3'; `, - 'filePath', ), ); @@ -71,11 +71,11 @@ describe('resolveImport', () => { it('should not count exports', () => { const result = parseImports( parse( + 'filePath', ` export default () => {}; export const test = () => {}; `, - 'filePath', ), ); @@ -86,11 +86,11 @@ describe('resolveImport', () => { it('should ignore dynamic imports', () => { const result = parseImports( parse( + 'filePath', ` const test1 = require(foo); const test2 = import(foo); `, - 'filePath', ), ); @@ -101,10 +101,10 @@ describe('resolveImport', () => { it('should count TS import equals declaration', () => { const result = parseImports( parse( + 'filePath.tsx', ` import test1 = require('test-file1'); `, - 'filePath.tsx', ), ); @@ -115,10 +115,10 @@ describe('resolveImport', () => { it('should count jest.requireActual', () => { const result = parseImports( parse( + 'filePath', ` const test1 = jest.requireActual('test-file1'); `, - 'filePath', ), ); diff --git a/src/utils/index.ts b/src/utils/index.ts index 9894e9a2..bb1c1dc0 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -97,7 +97,11 @@ const defaultPlugins: parser.ParserPlugin[] = [ 'throwExpressions', ] as parser.ParserPlugin[]; -export const parse = (source: string, filePath: string): any => { +export const parse = (filePath: string, source?: string): any => { + if (!source) { + source = readFile(filePath); + } + const plugins = [...defaultPlugins]; const fileType = /\.([jt])s(x?)$/.exec(filePath); diff --git a/src/utils/test.ts b/src/utils/test.ts index 8edad4ca..dd3550a4 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -4,6 +4,10 @@ jest.mock('fs'); jest.mock('path'); jest.mock('@babel/parser'); +afterEach(() => { + jest.resetAllMocks(); +}); + describe('readFile', () => { it('reads file in utf8', () => { const fileName = 'test'; @@ -161,7 +165,28 @@ describe('parse', () => { parseMock.mockImplementation((i: string) => i); const src = 'test'; - const result = parse(src, 'filePath'); + const result = parse('filePath', src); + + expect(parseMock.mock.calls.length).toBe(1); + expect(parseMock.mock.calls[0][0]).toBe(src); + expect(result).toBe(src); + }); + + it('reads file if not provided', () => { + const src = 'test'; + const fileName = 'filePath'; + + const { parse: parseMock } = require('@babel/parser'); + parseMock.mockImplementation((i: string) => i); + + const { readFileSync } = require('fs'); + readFileSync.mockReturnValueOnce(src); + + const result = parse(fileName); + + expect(readFileSync.mock.calls.length).toBe(1); + expect(readFileSync.mock.calls[0][0]).toBe(fileName); + expect(readFileSync.mock.calls[0][1]).toBe('utf8'); expect(parseMock.mock.calls.length).toBe(1); expect(parseMock.mock.calls[0][0]).toBe(src);