Merge pull request #3 from tago-io/fix/newBuckets

APPDEV-2 Fix/new buckets

.gitignore

@@ -51,3 +51,5 @@ yarn.lock
 # ignore local files
 *.LOCAL.*
 *.local.*
+.tago-lock.tago.lock
+.tagoio

package.json

@@ -24,29 +24,33 @@
     }
   },
   "devDependencies": {
-    "@tago-io/builder": "2.4.6",
-    "@types/jest": "27.0.3",
-    "@types/luxon": "^2.0.7",
-    "@types/node": "^16.11.9",
+    "@tago-io/builder": "3.1.3",
+    "@types/async": "3.2.16",
+    "@types/jest": "29.2.3",
+    "@types/lodash": "4.14.191",
+    "@types/luxon": "^3.1.0",
+    "@types/node": "^18.11.9",
     "@types/socket.io-client": "3.0.0",
-    "@typescript-eslint/eslint-plugin": "^5.4.0",
-    "@typescript-eslint/parser": "^5.4.0",
-    "eslint": "8.2.0",
-    "eslint-config-prettier": "^8.3.0",
-    "eslint-import-resolver-typescript": "^2.5.0",
-    "eslint-plugin-import": "^2.25.3",
-    "eslint-plugin-jest": "^25.2.4",
-    "eslint-plugin-prettier": "^4.0.0",
-    "jest": "27.3.1",
-    "prettier": "2.4.1",
-    "ts-jest": "27.0.7",
-    "ts-node": "^10.4.0",
-    "ts-node-dev": "1.1.8",
-    "typescript": "^4.5.2"
+    "@typescript-eslint/eslint-plugin": "^5.45.0",
+    "@typescript-eslint/parser": "^5.45.0",
+    "eslint": "8.28.0",
+    "eslint-config-prettier": "^8.5.0",
+    "eslint-import-resolver-typescript": "^3.5.2",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-jest": "^27.1.6",
+    "eslint-plugin-prettier": "^4.2.1",
+    "jest": "29.3.1",
+    "prettier": "2.8.0",
+    "ts-jest": "29.0.3",
+    "ts-node": "^10.9.1",
+    "ts-node-dev": "2.0.0",
+    "typescript": "^4.9.3"
   },
   "dependencies": {
-    "@tago-io/sdk": "^10.7.1",
-    "axios": "0.24.0",
-    "luxon": "^2.1.1"
+    "@tago-io/sdk": "^10.10.0",
+    "async": "3.2.4",
+    "axios": "1.2.0",
+    "lodash": "4.17.21",
+    "luxon": "^3.1.1"
   }
 }

src/clean.ts

@@ -16,7 +16,6 @@ async function startImport() {
         console.info("Cleaning Devices");
         const device_list = await import_account.devices.list({ amount: 999, fields: ["id", "bucket"] });
         await Promise.all(device_list.map(({ id }) => import_account.devices.delete(id)));
-        await Promise.all(device_list.map(({ bucket }) => import_account.buckets.delete(bucket)));
         break;
       case "dictionaries":
         console.info("Cleaning Dashboards");

src/cleanAnalysis.ts

@@ -1,9 +1,10 @@
 import { Account, Analysis, Utils } from "@tago-io/sdk";
 import { Data } from "@tago-io/sdk/out/common/common.types";
 import { TagoContext } from "@tago-io/sdk/out/modules/Analysis/analysis.types";
+
 import { EntityType, IExport } from "./exportTypes";
 import auditLogSetup from "./lib/auditLogSetup";
-import validation from "./lib/validation";
+import { initializeValidation } from "./lib/validation";
 
 const IMPORT_ORDER: EntityType = ["devices", "dictionaries", "accessManagement", "run_buttons", "analysis", "actions", "dictionaries", "dashboards"];
 
@@ -40,7 +41,7 @@ async function startCleaner(context: TagoContext, scope: Data[]) {
   }
   const main_account = new Account({ token: environment.account_token });
   const config_dev = await Utils.getDevice(main_account, scope[0].device);
-  const validate = validation("cleaner_validation", config_dev);
+  const validate = initializeValidation("cleaner_validation", config_dev);
 
   const target_token = scope.find((x) => x.variable === "cleaner_target_token");
   const entities = scope.find((x) => x.variable === "cleaner_entity_list" && x.metadata?.sentValues);

src/config.ts

@@ -5,23 +5,19 @@ const config: IExport = {
   export_tag: "export_id",
 
   // Entities that will be copied from the application.
-  // entities: ["dictionaries"],
-  entities: ["devices", "analysis", "dashboards", "accessManagement", "run_buttons", "actions", "dictionaries"],
+  entities: ["dashboards", "dictionaries"],
+  // entities: ["devices", "analysis", "dashboards", "accessManagement", "run_buttons", "actions", "dictionaries"],
   // entities: ["accessManagement"],
   // data: ["list_devtype_id"],
 
   // Account that entities will be copied from.
   export: {
-    // token: "0912bcff-51f1-46a1-b56c-94fd63bf6eb4", // Development
-    token: "", // Development
+    token: "1a5b22bd-30c6-47f3-ab76-939952dec310", // TEST
   },
 
   // Account where the entities will be pasted to.
   import: {
-    // token: "683d440e-4bf4-4950-aa60-3be553964fd9", // Sales
-    // token: "b8e6bd99-ca5d-48c3-90cc-7ec37608ea1b", // prod
-    // token: "5c0aab39-27cf-4a51-8094-cc0555eaa50b", // X-Talia
-    token: "7ec20119-e7e4-4605-a913-51c74dd78614", // X-Talia
+    token: "04051baf-4bce-4e49-a78c-e4527f3c7324", // PROD
   },
 };
 

src/exportTypes.ts

@@ -11,7 +11,7 @@ interface IExportHolder {
 }
 
 type Entity = "dashboards" | "devices" | "analysis";
-type EntityType = Array<"dashboards" | "devices" | "run_buttons" | "analysis" | "actions" | "dictionaries" | "accessManagement">;
+type EntityType = Array<"dashboards" | "devices" | "run_buttons" | "analysis" | "actions" | "dictionaries" | "accessManagement" | "connector">;
 interface IExport {
   export_tag: string;
   entities: EntityType;

src/lib/replaceObj.ts

@@ -1,7 +1,7 @@
 function replaceObj(original: any, replacer: { [key: string]: any }) {
   let string_obj = JSON.stringify(original);
   for (const key in replacer) {
-    string_obj = string_obj.replace(new RegExp(key, "g"), replacer[key]);
+    string_obj = string_obj.replaceAll(new RegExp(key, "g"), replacer[key]);
   }
 
   return JSON.parse(string_obj);

src/lib/validation.ts

@@ -1,5 +1,13 @@
+import { DateTime } from "luxon";
+
 import { Device } from "@tago-io/sdk";
 
+type validation_type = "success" | "danger" | "warning" | string;
+interface IValidateOptions {
+  show_markdown?: boolean;
+  user_id?: string;
+}
+
 /**
  * Setup function to send validation data to widgets.
  *
@@ -8,23 +16,41 @@ import { Device } from "@tago-io/sdk";
  * @param device device associated to the variable in the widget
  * @param show_markdown enable/disable markdown
  */
-type validation_type = "success" | "danger" | "warning" | string;
-
-export default function validation(validation_var: string, device: Device, show_markdown?: boolean) {
-  return function _(message: string, type: validation_type) {
+function initializeValidation(validation_var: string, device: Device, opts?: IValidateOptions) {
+  let i = 0;
+  return async function _(message: string, type: validation_type) {
     if (!message || !type) {
       throw "Missing message or type";
     }
-    device.sendData({
-      variable: validation_var,
-      value: message,
-      metadata: {
-        type: ["success", "danger", "warning"].includes(type) ? type : null,
-        color: !["success", "danger", "warning"].includes(type) ? type : null,
-        show_markdown,
-      },
-    });
+
+    i += 1;
+    // clean the bucket
+    await device
+      .deleteData({
+        variables: validation_var,
+        qty: 999,
+        end_date: DateTime.now().minus({ minutes: 1 }).toJSDate(),
+      })
+      .catch(console.log);
+
+    await device
+      .sendData({
+        variable: validation_var,
+        value: message,
+        time: DateTime.now()
+          .plus({ milliseconds: i * 200 })
+          .toJSDate(), //increment time by i
+        metadata: {
+          type: ["success", "danger", "warning"].includes(type) ? type : null,
+          color: !["success", "danger", "warning"].includes(type) ? type : undefined,
+          show_markdown: !!opts?.show_markdown,
+          user_id: opts?.user_id,
+        },
+      })
+      .catch(console.error);
 
     return message;
   };
 }
+
+export { initializeValidation };

src/services/accessExport.ts

@@ -1,4 +1,5 @@
 import { Account } from "@tago-io/sdk";
+
 import { IExportHolder } from "../exportTypes";
 import replaceObj from "../lib/replaceObj";
 
@@ -11,9 +12,9 @@ async function accessExport(account: Account, import_account: Account, export_ho
   for (const { id: access_id, tags: access_tags, name } of list) {
     console.info(`Exporting access rule ${name}`);
     const access = await account.accessManagement.info(access_id);
-    const export_id = access.tags.find((tag) => tag.key === "export_id")?.value;
+    const export_id = access_tags?.find((tag) => tag.key === "export_id")?.value;
 
-    let { id: target_id } = import_list.find((access) => access.tags.find((tag) => tag.key === "export_id" && tag.value == export_id)) || { id: null };
+    let { id: target_id } = import_list.find((access) => access.tags?.find((tag) => tag.key === "export_id" && tag.value == export_id)) || { id: null };
 
     const new_access = replaceObj(access, { ...export_holder.devices, ...export_holder.dashboards });
     if (!target_id) {
@@ -27,4 +28,4 @@ async function accessExport(account: Account, import_account: Account, export_ho
   return export_holder;
 }
 
-export default accessExport;
+export { accessExport };

src/services/actionsExport.ts

@@ -1,18 +1,22 @@
 import { Account } from "@tago-io/sdk";
+
 import { IExportHolder } from "../exportTypes";
 import filterExport from "../lib/filterExport";
 import replaceObj from "../lib/replaceObj";
 
 async function actionsExport(account: Account, import_account: Account, export_holder: IExportHolder) {
   console.info("Exporting actions: started");
 
-  const list = await account.actions.list({ amount: 99, fields: ["id", "name", "tags"], filter: { tags: [{ key: "export_id" }] } });
-  const import_list = await import_account.actions.list({ amount: 99, fields: ["id", "tags"], filter: { tags: [{ key: "export_id" }] } });
+  const list = await account.actions.list({ amount: 900, fields: ["id", "name", "tags"], filter: { tags: [{ key: "export_id" }] } });
+  const import_list = await import_account.actions.list({ amount: 900, fields: ["id", "tags"], filter: { tags: [{ key: "export_id" }] } });
 
   for (const { id: action_id, tags: action_tags, name } of list) {
-    console.info(`Exporting action ${name}`);
     const action = await account.actions.info(action_id);
     const export_id = action.tags.find((tag) => tag.key === "export_id")?.value;
+    if (!export_id) {
+      continue;
+    }
+    console.info(`Exporting action ${name}`);
 
     let { id: target_id } = import_list.find((action) => action.tags.find((tag) => tag.key === "export_id" && tag.value == export_id)) || { id: null };
 

src/services/analysisExport.ts

@@ -1,21 +1,22 @@
 import zlib from "zlib";
 import { Account } from "@tago-io/sdk";
 import axios from "axios";
+
 import { IExportHolder } from "../exportTypes";
 import replaceObj from "../lib/replaceObj";
 
 async function analysisExport(account: Account, import_account: Account, export_holder: IExportHolder) {
   console.info("Exporting analysis: started");
 
-  const list = await account.analysis.list({ amount: 99, fields: ["id", "name", "tags"], filter: { tags: [{ key: "export_id" }] } });
+  const list = await account.analysis.list({ amount: 99, fields: ["id", "name", "tags"], filter: { tags: [{ key: "export_id" }] } }).then((r) => r.reverse());
   const import_list = await import_account.analysis.list({ amount: 99, fields: ["id", "tags"], filter: { tags: [{ key: "export_id" }] } });
 
   for (const { id: analysis_id, name } of list) {
-    console.info(`Exporting dashboard ${name}...`);
+    console.info(`Exporting analysis ${name}...`);
     const analysis = await account.analysis.info(analysis_id);
-    const export_id = analysis.tags.find((tag) => tag.key === "export_id")?.value;
+    const export_id = analysis.tags?.find((tag) => tag.key === "export_id")?.value;
 
-    let { id: target_id } = import_list.find((analysis) => analysis.tags.find((tag) => tag.key === "export_id" && tag.value == export_id)) || { id: null };
+    let { id: target_id } = import_list.find((analysis) => analysis.tags?.find((tag) => tag.key === "export_id" && tag.value == export_id)) || { id: null };
 
     const new_analysis = replaceObj(analysis, { ...export_holder.devices, ...export_holder.tokens });
     if (!target_id) {
@@ -44,4 +45,4 @@ async function analysisExport(account: Account, import_account: Account, export_
   return export_holder;
 }
 
-export default analysisExport;
+export { analysisExport };

src/services/collectIDs.test.ts

@@ -0,0 +1,39 @@
+import { IExportHolder } from "../exportTypes";
+import { getExportHolder } from "./collectIDs";
+
+describe("Collect ID", () => {
+  test("Get Export Holder - Devices", () => {
+    const list = [
+      { id: "Test1", token: "1234-1234-1234-1234", tags: [{ key: "export_id", value: "config_dev" }] },
+      { id: "Test2", token: "1235-1235-1235-1235", tags: [{ key: "export_id", value: "other_dev" }] },
+    ];
+
+    const import_list = [
+      { id: "1Test", token: "4321-4321-4321-4321", tags: [{ key: "export_id", value: "config_dev" }] },
+      { id: "2Test", token: "5321-5321-5321-5321", tags: [{ key: "export_id", value: "other_dev" }] },
+    ];
+
+    const exportHolder: IExportHolder = { devices: {}, analysis: {}, dashboards: {}, tokens: {} };
+
+    getExportHolder(list, import_list, "devices", exportHolder);
+
+    expect(exportHolder.devices).toStrictEqual({ Test1: "1Test", Test2: "2Test" });
+    expect(exportHolder.tokens).toStrictEqual({ "1234-1234-1234-1234": "4321-4321-4321-4321", "1235-1235-1235-1235": "5321-5321-5321-5321" });
+  });
+
+  test("Get Export Holder - Devices Token not Found", () => {
+    const list = [
+      { id: "Test1", token: "1234-1234-1234-1234", tags: [{ key: "export_id", value: "config_dev" }] },
+      { id: "Test2", token: "1235-1235-1235-1235", tags: [{ key: "export_id", value: "other_dev" }] },
+    ];
+
+    const import_list = [
+      { id: "1Test", name: "1Test", tags: [{ key: "export_id", value: "config_dev" }] },
+      { id: "2Test", token: "5321-5321-5321-5321", tags: [{ key: "export_id", value: "other_dev" }] },
+    ];
+
+    const exportHolder: IExportHolder = { devices: {}, analysis: {}, dashboards: {}, tokens: {} };
+
+    expect(() => Promise.reject(getExportHolder(list, import_list, "devices", exportHolder))).toThrow("Device Token not found: 1Test [1Test]");
+  });
+});