work case-insensitive only with snowflake (#1018)

* Revert "use lower case names in catalog" This reverts commit 6e5a65d. * work case-insensitive with snowflake
fivetran · Jun 17, 2023 · 99a3cf3 · 99a3cf3
1 parent aa5630a
commit 99a3cf3
Show file tree

Hide file tree

Showing 11 changed files with 82 additions and 75 deletions.
diff --git a/e2e/projects/snowflake/models/users.sql b/e2e/projects/snowflake/models/users.sql
@@ -5,4 +5,4 @@
   )
 }}
 
-select * from `singular-vector-135519`.dbt_ls_e2e_dataset.users
+select * from dbt_ls_e2e_dataset.users
diff --git a/server/src/BigQueryZetaSqlWrapper.ts b/server/src/BigQueryZetaSqlWrapper.ts
@@ -0,0 +1,8 @@
+import { TableDefinition } from './TableDefinition';
+import { ZetaSqlWrapper } from './ZetaSqlWrapper';
+
+export class BigQueryZetaSqlWrapper extends ZetaSqlWrapper {
+  override createTableDefinition(namePath: string[]): TableDefinition {
+    return new TableDefinition(namePath);
+  }
+}
diff --git a/server/src/DestinationContext.ts b/server/src/DestinationContext.ts
@@ -1,14 +1,16 @@
 import { Err, err, ok, Result } from 'neverthrow';
 import { Emitter, Event } from 'vscode-languageserver';
+import { BigQueryZetaSqlWrapper } from './BigQueryZetaSqlWrapper';
 import { DagNode } from './dag/DagNode';
 import { DbtProfileSuccess } from './DbtProfileCreator';
 import { DbtRepository } from './DbtRepository';
 import { DestinationDefinition } from './DestinationDefinition';
 import { AnalyzeResult, AnalyzeTrackerFunc, ModelsAnalyzeResult, ProjectAnalyzer } from './ProjectAnalyzer';
+import { SnowflakeZetaSqlWrapper } from './SnowflakeZetaSqlWrapper';
 import { SqlHeaderAnalyzer } from './SqlHeaderAnalyzer';
 import { SupportedDestinations, ZetaSqlApi } from './ZetaSqlApi';
 import { ZetaSqlParser } from './ZetaSqlParser';
-import { KnownColumn, ZetaSqlWrapper } from './ZetaSqlWrapper';
+import { KnownColumn } from './ZetaSqlWrapper';
 
 export class DestinationContext {
   private static readonly ZETASQL_SUPPORTED_PLATFORMS = ['darwin', 'linux', 'win32'];
@@ -53,12 +55,14 @@ export class DestinationContext {
 
         const destination: SupportedDestinations = profileResult.type?.toLowerCase().trim() === 'snowflake' ? 'snowflake' : 'bigquery';
         const zetaSqlApi = new ZetaSqlApi(destination);
-        this.projectAnalyzer = new ProjectAnalyzer(
-          dbtRepository,
-          projectName,
-          destinationClient,
-          new ZetaSqlWrapper(destinationClient, zetaSqlApi, new ZetaSqlParser(zetaSqlApi), new SqlHeaderAnalyzer(zetaSqlApi)),
-        );
+        const zetaSqlParser = new ZetaSqlParser(zetaSqlApi);
+        const sqlHeaderAnalyzer = new SqlHeaderAnalyzer(zetaSqlApi);
+        const zetaSqlWrapper =
+          destination === 'bigquery'
+            ? new BigQueryZetaSqlWrapper(destinationClient, zetaSqlApi, zetaSqlParser, sqlHeaderAnalyzer)
+            : new SnowflakeZetaSqlWrapper(destinationClient, zetaSqlApi, zetaSqlParser, sqlHeaderAnalyzer);
+
+        this.projectAnalyzer = new ProjectAnalyzer(dbtRepository, projectName, destinationClient, zetaSqlWrapper);
         await this.projectAnalyzer.initialize();
       } catch (e) {
         const message = e instanceof Error ? e.message : JSON.stringify(e);

diff --git a/server/src/ProjectAnalyzer.ts b/server/src/ProjectAnalyzer.ts
@@ -236,7 +236,7 @@ export class ProjectAnalyzer {
 
     const ast = await this.zetaSqlWrapper.getAstOrError(compiledSql, catalogWithTempUdfs);
     if (ast.isOk() && model) {
-      const table = ProjectAnalyzer.createTableDefinition(model);
+      const table = this.createTableDefinition(model);
       this.fillTableWithAnalyzeResponse(table, ast.value);
       this.zetaSqlWrapper.registerTable(table);
     }
@@ -247,8 +247,8 @@ export class ProjectAnalyzer {
     };
   }
 
-  private static createTableDefinition(model: ManifestModel): TableDefinition {
-    return new TableDefinition([model.database, model.schema, model.alias ?? model.name]);
+  private createTableDefinition(model: ManifestModel): TableDefinition {
+    return this.zetaSqlWrapper.createTableDefinition([model.database, model.schema, model.alias ?? model.name]);
   }
 
   private fillTableWithAnalyzeResponse(table: TableDefinition, analyzeOutput: AnalyzeResponse__Output): void {
@@ -267,7 +267,7 @@ export class ProjectAnalyzer {
         .find(n => n.getValue().uniqueId === node && n.getValue().config?.materialized === 'ephemeral')
         ?.getValue();
       if (dependsOnEphemeralModel) {
-        const table = ProjectAnalyzer.createTableDefinition(dependsOnEphemeralModel);
+        const table = this.createTableDefinition(dependsOnEphemeralModel);
         if (!this.zetaSqlWrapper.isTableRegistered(table)) {
           await this.analyzeModelCached(dependsOnEphemeralModel, tableFetcher, undefined, visitedModels);
         }

diff --git a/server/src/SnowflakeZetaSqlWrapper.ts b/server/src/SnowflakeZetaSqlWrapper.ts
@@ -0,0 +1,8 @@
+import { TableDefinition } from './TableDefinition';
+import { ZetaSqlWrapper } from './ZetaSqlWrapper';
+
+export class SnowflakeZetaSqlWrapper extends ZetaSqlWrapper {
+  override createTableDefinition(namePath: string[]): TableDefinition {
+    return new TableDefinition(namePath.map(n => n.toLowerCase()));
+  }
+}
diff --git a/server/src/TableDefinition.ts b/server/src/TableDefinition.ts
@@ -47,65 +47,55 @@ export class TableDefinition {
         this.projectName = this.namePath.length >= 3 ? this.namePath[0] : undefined;
       }
     }
-    return this.projectName?.toLowerCase();
+    return this.projectName;
   }
 
   getProjectCatalogName(): string | undefined {
     return this.catalogCount === undefined ? this.getProjectName() : undefined;
   }
 
   getDatasetCatalogName(): string | undefined {
-    let name = undefined;
     switch (this.catalogCount) {
       case 1: {
-        name = undefined;
-        break;
+        return undefined;
       }
       case 2: {
-        name = `${this.namePath[0]}.${this.namePath[1]}`;
-        break;
+        return `${this.namePath[0]}.${this.namePath[1]}`;
       }
       default: {
-        name = this.getDataSetName();
-        break;
+        return this.getDataSetName();
       }
     }
-    return name?.toLowerCase();
   }
 
   getTableNameInZetaSql(): string {
-    let name = undefined;
     switch (this.catalogCount) {
       case 1: {
-        name = this.namePath.join('.');
-        break;
+        return this.namePath.join('.');
       }
       case 2: {
-        name = this.namePath[2];
-        break;
+        return this.namePath[2];
       }
       default: {
-        name = this.getTableName();
-        break;
+        return this.getTableName();
       }
     }
-    return name.toLowerCase();
   }
 
   getDataSetName(): string | undefined {
     if (!this.dataSetName) {
-      this.dataSetName = (
-        this.containsInformationSchema() ? this.namePath[this.informationSchemaIndex - 1] : this.namePath.at(this.datasetIndex)
-      )?.toLocaleLowerCase();
+      this.dataSetName = this.containsInformationSchema()
+        ? this.namePath[this.informationSchemaIndex - 1]?.toLocaleLowerCase()
+        : this.namePath[this.datasetIndex];
     }
     return this.dataSetName;
   }
 
   getTableName(): string {
     if (!this.tableName) {
-      this.tableName = (
-        this.containsInformationSchema() ? this.namePath[this.informationSchemaIndex + 1] : this.namePath[this.datasetIndex + 1]
-      ).toLocaleLowerCase();
+      this.tableName = this.containsInformationSchema()
+        ? this.namePath[this.informationSchemaIndex + 1].toLocaleLowerCase()
+        : this.namePath[this.datasetIndex + 1];
     }
     return this.tableName;
   }

diff --git a/server/src/ZetaSqlWrapper.ts b/server/src/ZetaSqlWrapper.ts
@@ -22,7 +22,7 @@ export interface KnownColumn {
   type: string;
 }
 
-export class ZetaSqlWrapper {
+export abstract class ZetaSqlWrapper {
   static readonly PARTITION_TIME = '_PARTITIONTIME';
   static readonly PARTITION_DATE = '_PARTITIONDATE';
 
@@ -40,6 +40,8 @@ export class ZetaSqlWrapper {
     this.catalog = this.getDefaultCatalog();
   }
 
+  abstract createTableDefinition(namePath: string[]): TableDefinition;
+
   async initializeZetaSql(): Promise<void> {
     await this.zetaSqlApi.initialize();
   }
@@ -79,7 +81,7 @@ export class ZetaSqlWrapper {
   async findTableNames(sql: string): Promise<TableDefinition[]> {
     try {
       const extractResult = await this.extractTableNamesFromStatement(sql);
-      return extractResult.tableName.map(t => new TableDefinition(t.tableNameSegment));
+      return extractResult.tableName.map(t => this.createTableDefinition(t.tableNameSegment));
     } catch (e) {
       console.log(e instanceof Error ? e.message : e);
     }
@@ -263,10 +265,10 @@ export class ZetaSqlWrapper {
   private ensureUdfOwnerCatalogExists(udf: Udf): SimpleCatalogProto {
     let udfOwner = this.catalog;
     if (udf.nameParts.length > 2) {
-      const projectCatalog = ZetaSqlWrapper.addChildCatalog(this.catalog, udf.nameParts[0].toLowerCase());
-      udfOwner = ZetaSqlWrapper.addChildCatalog(projectCatalog, udf.nameParts[1].toLowerCase());
+      const projectCatalog = ZetaSqlWrapper.addChildCatalog(this.catalog, udf.nameParts[0]);
+      udfOwner = ZetaSqlWrapper.addChildCatalog(projectCatalog, udf.nameParts[1]);
     } else if (udf.nameParts.length === 2) {
-      udfOwner = ZetaSqlWrapper.addChildCatalog(this.catalog, udf.nameParts[0].toLowerCase());
+      udfOwner = ZetaSqlWrapper.addChildCatalog(this.catalog, udf.nameParts[0]);
     }
     return udfOwner;
   }

diff --git a/server/src/snowflake/SnowflakeClient.ts b/server/src/snowflake/SnowflakeClient.ts
@@ -90,7 +90,8 @@ export class SnowflakeClient implements DbtDestinationClient {
           typeKind: SnowflakeClient.toTypeKind(a),
         },
       }));
-      results.push({ nameParts, returnType: { typeKind }, arguments: args });
+      // In Snowflake names are case-insensitive so we use lowercase
+      results.push({ nameParts: nameParts.map(n => n.toLowerCase()), returnType: { typeKind }, arguments: args });
     });
 
     return runWithTimeout(

diff --git a/server/src/test/AnalyzeTable.spec.ts b/server/src/test/AnalyzeTable.spec.ts
@@ -88,6 +88,7 @@ describe('ProjectAnalyzer analyzeModelsTree', () => {
         }),
       ),
     );
+    when(mockZetaSqlWrapper.createTableDefinition(anything())).thenReturn(new TableDefinition(MAIN_TABLE_NAME_PATH));
     spiedProjectAnalyzer = spy(projectAnalyzer);
   });
 

diff --git a/server/src/test/TableDefinition.spec.ts b/server/src/test/TableDefinition.spec.ts
@@ -5,94 +5,86 @@ import { TableDefinition } from '../TableDefinition';
 describe('TableDefinition', () => {
   it('getProjectName should return project name', () => {
     getProjectNameShouldReturnProjectName(['project', 'data_set', 'table'], 'project');
-    getProjectNameShouldReturnProjectName(['ProJecT', 'data_set', 'table'], 'project');
+    getProjectNameShouldReturnProjectName(['PROJECT', 'data_set', 'table'], 'PROJECT');
     getProjectNameShouldReturnProjectName(['project.data_set.table'], 'project');
-    getProjectNameShouldReturnProjectName(['ProJecT.data_set.table'], 'project');
+    getProjectNameShouldReturnProjectName(['PROJECT.data_set.table'], 'PROJECT');
     getProjectNameShouldReturnProjectName(['project.data_set', 'table'], 'project');
-    getProjectNameShouldReturnProjectName(['ProJecT.data_set', 'table'], 'project');
+    getProjectNameShouldReturnProjectName(['PROJECT.data_set', 'table'], 'PROJECT');
     getProjectNameShouldReturnProjectName(['data_set', 'table'], undefined);
 
     getProjectNameShouldReturnProjectName(['project', 'data_set', 'INFORMATION_SCHEMA', 'COLUMNS'], 'project');
-    getProjectNameShouldReturnProjectName(['ProJecT', 'data_set', 'INFORMATION_SCHEMA', 'COLUMNS'], 'project');
+    getProjectNameShouldReturnProjectName(['PROJECT', 'data_set', 'INFORMATION_SCHEMA', 'COLUMNS'], 'PROJECT');
     getProjectNameShouldReturnProjectName(['data_set', 'INFORMATION_SCHEMA', 'COLUMNS'], undefined);
     getProjectNameShouldReturnProjectName(['INFORMATION_SCHEMA', 'COLUMNS'], undefined);
   });
 
   it('getProjectCatalogName should return project catalog name', () => {
     getProjectCatalogNameShouldReturnProjectCatalogName(['project', 'data_set', 'table'], 'project');
-    getProjectCatalogNameShouldReturnProjectCatalogName(['ProJecT', 'data_set', 'table'], 'project');
+    getProjectCatalogNameShouldReturnProjectCatalogName(['PROJECT', 'data_set', 'table'], 'PROJECT');
     getProjectCatalogNameShouldReturnProjectCatalogName(['project.data_set.table'], undefined);
     getProjectCatalogNameShouldReturnProjectCatalogName(['project.data_set', 'table'], undefined);
     getProjectCatalogNameShouldReturnProjectCatalogName(['data_set', 'table'], undefined);
 
     getProjectCatalogNameShouldReturnProjectCatalogName(['project', 'data_set', 'INFORMATION_SCHEMA', 'COLUMNS'], 'project');
-    getProjectCatalogNameShouldReturnProjectCatalogName(['ProJecT', 'data_set', 'INFORMATION_SCHEMA', 'COLUMNS'], 'project');
+    getProjectCatalogNameShouldReturnProjectCatalogName(['PROJECT', 'data_set', 'INFORMATION_SCHEMA', 'COLUMNS'], 'PROJECT');
     getProjectCatalogNameShouldReturnProjectCatalogName(['data_set', 'INFORMATION_SCHEMA', 'COLUMNS'], undefined);
     getProjectCatalogNameShouldReturnProjectCatalogName(['INFORMATION_SCHEMA', 'COLUMNS'], undefined);
   });
 
   it('getDataSetName should return data set name', () => {
     getDataSetNameShouldReturnDataSetName(['project', 'data_set', 'table'], 'data_set');
-    getDataSetNameShouldReturnDataSetName(['project', 'DaTa_seT', 'table'], 'data_set');
+    getDataSetNameShouldReturnDataSetName(['project', 'DATA_SET', 'table'], 'DATA_SET');
     getDataSetNameShouldReturnDataSetName(['project.data_set.table'], 'data_set');
-    getDataSetNameShouldReturnDataSetName(['project.DaTa_seT.table'], 'data_set');
+    getDataSetNameShouldReturnDataSetName(['project.DATA_SET.table'], 'DATA_SET');
     getDataSetNameShouldReturnDataSetName(['project.data_set', 'table'], 'data_set');
-    getDataSetNameShouldReturnDataSetName(['project.DaTa_seT', 'table'], 'data_set');
+    getDataSetNameShouldReturnDataSetName(['project.DATA_SET', 'table'], 'DATA_SET');
     getDataSetNameShouldReturnDataSetName(['data_set', 'table'], 'data_set');
-    getDataSetNameShouldReturnDataSetName(['DaTa_seT', 'table'], 'data_set');
+    getDataSetNameShouldReturnDataSetName(['DATA_SET', 'table'], 'DATA_SET');
 
     getDataSetNameShouldReturnDataSetName(['project', 'data_set', 'INFORMATION_SCHEMA', 'COLUMNS'], 'data_set');
-    getDataSetNameShouldReturnDataSetName(['project', 'DaTa_seT', 'INFORMATION_SCHEMA', 'COLUMNS'], 'data_set');
     getDataSetNameShouldReturnDataSetName(['data_set', 'INFORMATION_SCHEMA', 'COLUMNS'], 'data_set');
-    getDataSetNameShouldReturnDataSetName(['DaTa_seT', 'INFORMATION_SCHEMA', 'COLUMNS'], 'data_set');
     getDataSetNameShouldReturnDataSetName(['INFORMATION_SCHEMA', 'COLUMNS'], undefined);
   });
 
   it('getDataSetCatalogName should return data set catalog name', () => {
     getDataSetCatalogNameShouldReturnDataSetCatalogName(['project', 'data_set', 'table'], 'data_set');
-    getDataSetCatalogNameShouldReturnDataSetCatalogName(['project', 'DaTa_seT', 'table'], 'data_set');
+    getDataSetCatalogNameShouldReturnDataSetCatalogName(['project', 'DATA_SET', 'table'], 'DATA_SET');
     getDataSetCatalogNameShouldReturnDataSetCatalogName(['project.data_set.table'], undefined);
     getDataSetCatalogNameShouldReturnDataSetCatalogName(['project.data_set', 'table'], 'project.data_set');
-    getDataSetCatalogNameShouldReturnDataSetCatalogName(['project.DaTa_seT', 'table'], 'project.data_set');
+    getDataSetCatalogNameShouldReturnDataSetCatalogName(['project.DATA_SET', 'table'], 'project.DATA_SET');
     getDataSetCatalogNameShouldReturnDataSetCatalogName(['data_set', 'table'], 'data_set');
-    getDataSetCatalogNameShouldReturnDataSetCatalogName(['DaTa_seT', 'table'], 'data_set');
+    getDataSetCatalogNameShouldReturnDataSetCatalogName(['DATA_SET', 'table'], 'DATA_SET');
 
     getDataSetCatalogNameShouldReturnDataSetCatalogName(['project', 'data_set', 'INFORMATION_SCHEMA', 'COLUMNS'], 'data_set');
-    getDataSetCatalogNameShouldReturnDataSetCatalogName(['project', 'DaTa_seT', 'INFORMATION_SCHEMA', 'COLUMNS'], 'data_set');
     getDataSetCatalogNameShouldReturnDataSetCatalogName(['data_set', 'INFORMATION_SCHEMA', 'COLUMNS'], 'data_set');
-    getDataSetCatalogNameShouldReturnDataSetCatalogName(['DaTa_seT', 'INFORMATION_SCHEMA', 'COLUMNS'], 'data_set');
     getDataSetCatalogNameShouldReturnDataSetCatalogName(['INFORMATION_SCHEMA', 'COLUMNS'], undefined);
   });
 
   it('getTableName should return table name', () => {
     geTableNameShouldReturnTableName(['project', 'data_set', 'table'], 'table');
-    geTableNameShouldReturnTableName(['project', 'data_set', 'TABLE'], 'table');
+    geTableNameShouldReturnTableName(['project', 'data_set', 'TABLE'], 'TABLE');
     geTableNameShouldReturnTableName(['project.data_set.table'], 'table');
-    geTableNameShouldReturnTableName(['project.data_set.TABLE'], 'table');
+    geTableNameShouldReturnTableName(['project.data_set.TABLE'], 'TABLE');
     geTableNameShouldReturnTableName(['project.data_set', 'table'], 'table');
-    geTableNameShouldReturnTableName(['project.data_set', 'TABLE'], 'table');
+    geTableNameShouldReturnTableName(['project.data_set', 'TABLE'], 'TABLE');
     geTableNameShouldReturnTableName(['data_set', 'table'], 'table');
-    geTableNameShouldReturnTableName(['data_set', 'TABLE'], 'table');
+    geTableNameShouldReturnTableName(['data_set', 'TABLE'], 'TABLE');
 
-    geTableNameShouldReturnTableName(['project', 'data_set', 'INFORMATION_SCHEMA', 'COLuMnS'], 'columns');
-    geTableNameShouldReturnTableName(['project', 'data_set', 'INFORMATION_SCHEMA', 'COLUMNS'], 'columns');
-    geTableNameShouldReturnTableName(['data_set', 'INFORMATION_SCHEMA', 'COLUMNS'], 'columns');
+    geTableNameShouldReturnTableName(['project', 'data_set', 'INFORMATION_SCHEMA', 'columns'], 'columns');
     geTableNameShouldReturnTableName(['data_set', 'INFORMATION_SCHEMA', 'columns'], 'columns');
   });
 
   it('getTableNameInZetaSql should return table name in ZetaSQL', () => {
     getTableNameInZetaSqlShouldReturnTableNameInZetaSql(['project', 'data_set', 'table'], 'table');
-    getTableNameInZetaSqlShouldReturnTableNameInZetaSql(['project', 'data_set', 'TABLE'], 'table');
+    getTableNameInZetaSqlShouldReturnTableNameInZetaSql(['project', 'data_set', 'TABLE'], 'TABLE');
     getTableNameInZetaSqlShouldReturnTableNameInZetaSql(['project.data_set.table'], 'project.data_set.table');
-    getTableNameInZetaSqlShouldReturnTableNameInZetaSql(['project.data_set.TABLE'], 'project.data_set.table');
+    getTableNameInZetaSqlShouldReturnTableNameInZetaSql(['project.data_set.TABLE'], 'project.data_set.TABLE');
     getTableNameInZetaSqlShouldReturnTableNameInZetaSql(['project.data_set', 'table'], 'table');
-    getTableNameInZetaSqlShouldReturnTableNameInZetaSql(['project.data_set', 'TABLE'], 'table');
+    getTableNameInZetaSqlShouldReturnTableNameInZetaSql(['project.data_set', 'TABLE'], 'TABLE');
     getTableNameInZetaSqlShouldReturnTableNameInZetaSql(['data_set', 'table'], 'table');
-    getTableNameInZetaSqlShouldReturnTableNameInZetaSql(['data_set', 'TABLE'], 'table');
+    getTableNameInZetaSqlShouldReturnTableNameInZetaSql(['data_set', 'TABLE'], 'TABLE');
 
-    getTableNameInZetaSqlShouldReturnTableNameInZetaSql(['project', 'data_set', 'INFORMATION_SCHEMA', 'COLUMNS'], 'columns');
     getTableNameInZetaSqlShouldReturnTableNameInZetaSql(['project', 'data_set', 'INFORMATION_SCHEMA', 'columns'], 'columns');
-    getTableNameInZetaSqlShouldReturnTableNameInZetaSql(['data_set', 'INFORMATION_SCHEMA', 'COLUMNS'], 'columns');
     getTableNameInZetaSqlShouldReturnTableNameInZetaSql(['data_set', 'INFORMATION_SCHEMA', 'columns'], 'columns');
   });
 });
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,4 +5,4 @@ @@
       )
     }}
-    select * from `singular-vector-135519`.dbt_ls_e2e_dataset.users
+    select * from dbt_ls_e2e_dataset.users