perf(schema-compiler): Improve yaml compilation speed

packages/cubejs-backend-shared/src/FileRepository.ts

@@ -6,6 +6,7 @@ export interface FileContent {
   content: string;
   readOnly?: boolean;
   isModule?: boolean;
+  convertedToJs?: boolean;
 }
 
 export interface SchemaFileRepository {

packages/cubejs-backend-shared/src/PerfTracker.ts

@@ -0,0 +1,88 @@
+import { performance, PerformanceObserver } from 'perf_hooks';
+
+interface PerfMetric {
+  count: number;
+  totalTime: number;
+  avgTime: number;
+}
+
+interface PerfStats {
+  [key: string]: PerfMetric;
+}
+
+class PerfTracker {
+  private metrics: PerfStats = {};
+
+  private globalMetric: string | null = null;
+
+  public constructor() {
+    const obs = new PerformanceObserver((items) => {
+      for (const entry of items.getEntries()) {
+        const { name } = entry;
+        if (!this.metrics[name]) {
+          this.metrics[name] = { count: 0, totalTime: 0, avgTime: 0 };
+        }
+        const m = this.metrics[name];
+        m.count++;
+        m.totalTime += entry.duration;
+        m.avgTime = m.totalTime / m.count;
+      }
+    });
+    obs.observe({ entryTypes: ['measure'] });
+  }
+
+  public start(name: string, global: boolean = false): { end: () => void } {
+    const uid = `${name}-${performance.now()}`;
+    const startMark = `${uid}-start`;
+    const endMark = `${uid}-end`;
+    performance.mark(startMark);
+
+    if (global && !this.globalMetric) {
+      this.globalMetric = name;
+    }
+
+    let ended = false;
+
+    return {
+      end: () => {
+        if (ended) return;
+        performance.mark(endMark);
+        performance.measure(name, startMark, endMark);
+        ended = true;
+      }
+    };
+  }
+
+  public printReport() {
+    console.log('\n🚀 PERFORMANCE REPORT 🚀\n');
+    console.log('═'.repeat(90));
+
+    const sorted = Object.entries(this.metrics)
+      .sort(([, a], [, b]) => b.totalTime - a.totalTime);
+
+    if (!sorted.length) {
+      console.log('No performance data collected.');
+      return;
+    }
+
+    let totalTime: number = 0;
+
+    if (this.globalMetric) {
+      totalTime = this.metrics[this.globalMetric]?.totalTime;
+    } else {
+      totalTime = sorted.reduce((sum, [, m]) => sum + m.totalTime, 0);
+    }
+
+    console.log(`⏱️  TOTAL TIME: ${totalTime.toFixed(2)}ms\n`);
+
+    sorted.forEach(([name, m]) => {
+      const pct = totalTime > 0 ? (m.totalTime / totalTime * 100) : 0;
+      console.log(`  ${name.padEnd(40)} │ ${m.totalTime.toFixed(2).padStart(8)}ms │ ${m.avgTime.toFixed(2).padStart(7)}ms avg │ ${pct.toFixed(1).padStart(5)}% │ ${m.count.toString().padStart(4)} calls`);
+    });
+
+    console.log('═'.repeat(90));
+    console.log('🎯 End of Performance Report\n');
+  }
+}
+
+export const perfTracker = new PerfTracker();

packages/cubejs-backend-shared/src/index.ts

@@ -21,3 +21,4 @@ export * from './process';
 export * from './platform';
 export * from './FileRepository';
 export * from './decorators';
+export * from './PerfTracker';

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.ts

@@ -88,6 +88,8 @@ type CompileCubeFilesCompilers = {
   contextCompilers?: CompilerInterface[];
 };
 
+export type CompileContext = any;
+
 export class DataSchemaCompiler {
   private readonly repository: SchemaFileRepository;
 
@@ -123,7 +125,7 @@ export class DataSchemaCompiler {
 
   private readonly compilerCache: CompilerCache;
 
-  private readonly compileContext: any;
+  private readonly compileContext: CompileContext;
 
   private errorReportOptions: ErrorReporterOptions | undefined;
 
@@ -172,14 +174,13 @@ export class DataSchemaCompiler {
     this.standalone = options.standalone || false;
     this.nativeInstance = options.nativeInstance;
     this.yamlCompiler = options.yamlCompiler;
-    this.yamlCompiler.dataSchemaCompiler = this;
     this.pythonContext = null;
     this.workerPool = null;
     this.compilerId = options.compilerId || 'default';
     this.compiledScriptCache = options.compiledScriptCache;
   }
 
-  public compileObjects(compileServices, objects, errorsReport: ErrorReporter) {
+  public compileObjects(compileServices: CompilerInterface[], objects, errorsReport: ErrorReporter) {
     try {
       return compileServices
         .map((compileService) => (() => compileService.compile(objects, errorsReport)))
@@ -193,7 +194,7 @@ export class DataSchemaCompiler {
     }
   }
 
-  protected async loadPythonContext(files, nsFileName) {
+  protected async loadPythonContext(files: FileContent[], nsFileName: string): Promise<PythonCtx> {
     const ns = files.find((f) => f.fileName === nsFileName);
     if (ns) {
       return this.nativeInstance.loadPythonContext(
@@ -216,7 +217,23 @@ export class DataSchemaCompiler {
     this.pythonContext = await this.loadPythonContext(files, 'globals.py');
     this.yamlCompiler.initFromPythonContext(this.pythonContext);
 
-    const toCompile = files.filter((f) => !this.filesToCompile || !this.filesToCompile.length || this.filesToCompile.indexOf(f.fileName) !== -1);
+    // As we mutate files data, we need a copy, not a refs.
+    // FileContent is a plain object with primitives, so it's enough for a shallow copy.
+    let toCompile = (this.filesToCompile?.length
+      ? files.filter(f => this.filesToCompile.includes(f.fileName))
+      : files).filter(f => f.fileName.endsWith('.js')
+        // We don't transpile/compile other files (like .py and so on)
+        || f.fileName.endsWith('.yml')
+        || f.fileName.endsWith('.yaml')
+        || f.fileName.endsWith('.jinja')).map(f => ({ ...f }));
+
+    const jinjaTemplatedFiles = toCompile.filter((file) => file.fileName.endsWith('.jinja') ||
+      (file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml')) && file.content.match(JINJA_SYNTAX));
+
+    if (jinjaTemplatedFiles.length > 0) {
+      // Preload Jinja templates to the engine
+      this.loadJinjaTemplates(jinjaTemplatedFiles);
+    }
 
     const errorsReport = new ErrorReporter(null, [], this.errorReportOptions);
     this.errorsReporter = errorsReport;
@@ -271,10 +288,10 @@ export class DataSchemaCompiler {
 
         await this.transpileJsFile(dummyFile, errorsReport, { cubeNames, cubeSymbols, transpilerNames, contextSymbols: CONTEXT_SYMBOLS, compilerId, stage });
 
-        const nonJsFilesTasks = toCompile.filter(file => !file.fileName.endsWith('.js'))
+        const nonJsFilesTasks = toCompile.filter(file => !file.fileName.endsWith('.js') && !file.convertedToJs)
           .map(f => this.transpileFile(f, errorsReport, { transpilerNames, compilerId }));
 
-        const jsFiles = toCompile.filter(file => file.fileName.endsWith('.js'));
+        const jsFiles = toCompile.filter(file => file.fileName.endsWith('.js') || file.convertedToJs);
         let JsFilesTasks = [];
 
         if (jsFiles.length > 0) {
@@ -373,7 +390,7 @@ export class DataSchemaCompiler {
         } else {
           const foundFile = this.resolveModuleFile(file, extensionName, transpiledFiles, errorsReport);
           if (!foundFile && this.allowNodeRequire) {
-            if (extensionName.indexOf('.') === 0) {
+            if (extensionName.startsWith('.')) {
               extensionName = path.resolve(this.repository.localPath(), extensionName);
             }
             // eslint-disable-next-line global-require,import/no-dynamic-require
@@ -387,7 +404,6 @@ export class DataSchemaCompiler {
             foundFile,
             errorsReport,
             compiledFiles,
-            [],
             { doSyntaxCheck: true }
           );
           exports[foundFile.fileName] = exports[foundFile.fileName] || {};
@@ -426,6 +442,7 @@ export class DataSchemaCompiler {
         compiledFiles = {};
         asyncModules = [];
         transpiledFiles = [];
+        toCompile = [];
 
         if (transpilationNative) {
           // Clean up cache
@@ -462,30 +479,59 @@ export class DataSchemaCompiler {
     return this.compilePromise;
   }
 
+  private loadJinjaTemplates(files: FileContent[]): void {
+    if (NATIVE_IS_SUPPORTED !== true) {
+      throw new Error(
+        `Native extension is required to process jinja files. ${NATIVE_IS_SUPPORTED.reason}. Read more: ` +
+        'https://github.com/cube-js/cube/blob/master/packages/cubejs-backend-native/README.md#supported-architectures-and-platforms'
+      );
+    }
+
+    const jinjaEngine = this.yamlCompiler.getJinjaEngine();
+
+    files.forEach((file) => {
+      jinjaEngine.loadTemplate(file.fileName, file.content);
+    });
+  }
+
   private async transpileFile(
     file: FileContent,
     errorsReport: ErrorReporter,
     options: TranspileOptions = {}
   ): Promise<(FileContent | undefined)> {
-    if (file.fileName.endsWith('.jinja') ||
+    if (file.fileName.endsWith('.js') || file.convertedToJs) {
+      return this.transpileJsFile(file, errorsReport, options);
+    } else if (file.fileName.endsWith('.jinja') ||
       (file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml'))
-      // TODO do Jinja syntax check with jinja compiler
       && file.content.match(JINJA_SYNTAX)
     ) {
-      if (NATIVE_IS_SUPPORTED !== true) {
-        throw new Error(
-          `Native extension is required to process jinja files. ${NATIVE_IS_SUPPORTED.reason}. Read more: ` +
-          'https://github.com/cube-js/cube/blob/master/packages/cubejs-backend-native/README.md#supported-architectures-and-platforms'
-        );
+      const transpiledFile = await this.yamlCompiler.compileYamlWithJinjaFile(
+        file,
+        errorsReport,
+        this.standalone ? {} : this.cloneCompileContextWithGetterAlias(this.compileContext),
+        this.pythonContext!
+      );
+      if (transpiledFile) {
+        // We update the jinja/yaml file content to the transpiled js content
+        // and raise related flag so it will go JS transpilation flow afterward
+        // avoiding costly YAML/Python parsing again.
+        file.content = transpiledFile.content;
+        file.convertedToJs = true;
       }
 
-      this.yamlCompiler.getJinjaEngine().loadTemplate(file.fileName, file.content);
-
-      return file;
+      return transpiledFile;
     } else if (file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml')) {
-      return file;
-    } else if (file.fileName.endsWith('.js')) {
-      return this.transpileJsFile(file, errorsReport, options);
+      const transpiledFile = this.yamlCompiler.transpileYamlFile(file, errorsReport);
+
+      if (transpiledFile) {
+        // We update the yaml file content to the transpiled js content
+        // and raise related flag so it will go JS transpilation flow afterward
+        // avoiding costly YAML/Python parsing again.
+        file.content = transpiledFile.content;
+        file.convertedToJs = true;
+      }
+
+      return transpiledFile;
     } else {
       return file;
     }
@@ -628,16 +674,15 @@ export class DataSchemaCompiler {
     compiledFiles: Record<string, boolean>,
     asyncModules: CallableFunction[],
     compilers: CompileCubeFilesCompilers,
-    toCompile: FileContent[],
+    transpiledFiles: FileContent[],
     errorsReport: ErrorReporter
   ) {
-    toCompile
+    transpiledFiles
       .forEach((file) => {
         this.compileFile(
           file,
           errorsReport,
           compiledFiles,
-          asyncModules
         );
       });
     await asyncModules.reduce((a: Promise<void>, b: CallableFunction) => a.then(() => b()), Promise.resolve());
@@ -653,7 +698,6 @@ export class DataSchemaCompiler {
     file: FileContent,
     errorsReport: ErrorReporter,
     compiledFiles: Record<string, boolean>,
-    asyncModules: CallableFunction[],
     { doSyntaxCheck } = { doSyntaxCheck: false }
   ) {
     if (compiledFiles[file.fileName]) {
@@ -662,23 +706,7 @@ export class DataSchemaCompiler {
 
     compiledFiles[file.fileName] = true;
 
-    if (file.fileName.endsWith('.js')) {
-      this.compileJsFile(file, errorsReport, { doSyntaxCheck });
-    } else if (file.fileName.endsWith('.yml.jinja') || file.fileName.endsWith('.yaml.jinja') ||
-      (
-        file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml')
-        // TODO do Jinja syntax check with jinja compiler
-      ) && file.content.match(JINJA_SYNTAX)
-    ) {
-      asyncModules.push(() => this.yamlCompiler.compileYamlWithJinjaFile(
-        file,
-        errorsReport,
-        this.standalone ? {} : this.cloneCompileContextWithGetterAlias(this.compileContext),
-        this.pythonContext!
-      ));
-    } else if (file.fileName.endsWith('.yml') || file.fileName.endsWith('.yaml')) {
-      this.yamlCompiler.compileYamlFile(file, errorsReport);
-    }
+    this.compileJsFile(file, errorsReport, { doSyntaxCheck });
   }
 
   private getJsScript(file: FileContent): vm.Script {

packages/cubejs-schema-compiler/src/compiler/ViewCompilationGate.ts

@@ -6,10 +6,10 @@ export class ViewCompilationGate {
   }
 
   public compile(cubes: any[]) {
-    // When developing Data Access Policies feature, we've came across a
+    // When developing Data Access Policies feature, we've come across a
     // limitation that Cube members can't be referenced in access policies defined on Views,
     // because views aren't (yet) compiled at the time of access policy evaluation.
-    // To workaround this limitation and additional compilation pass is necessary,
+    // To work around this limitation and additional compilation pass is necessary,
     // however it comes with a significant performance penalty.
     // This gate check whether the data model contains views with access policies,
     // and only then allows the additional compilation pass.