From cdefa36464fc7383abad988207f8f0e74a61dd81 Mon Sep 17 00:00:00 2001 From: ciatph Date: Wed, 28 Aug 2024 00:07:07 +0800 Subject: [PATCH 1/4] chore: follow-up total municipalities count test, #68 --- README.md | 3 +- .../createMunicipalityInstance.js | 3 +- .../municipalities/municipalitiesCount.js | 31 +++++++--- app/__tests__/provinces/updateInstances.js | 2 +- app/src/classes/excel/index.js | 62 +++++++++++++++---- 5 files changed, 77 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 5eb640d..b803ff8 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,8 @@ The following dependencies are used for this project. Feel free to use other dep - npm v8.5.0 4. Excel file - ph-municipalities uses Excel files in the `/app/data` directory as data source. - - At minimum, the excel file should have a **column** that contains municipality and province names following the pattern `"municipalityName (provinceName)"` + - At minimum, the Excel file should have a **column** that contains municipality and province names following the pattern `"municipalityName (provinceName)"` + - (Optional) The Excel file should have a row on the same **column** as above containing the text `"Project Areas"` plus two (2) blank rows before rows containing municipality and province names to enable strict testing and validation of the number of parsed data rows - Checkout the excel file format on the `/app/data/day1.xlsx` sample file for more information 5. (Optional) Download URL for a remote excel file. - See the `EXCEL_FILE_URL` variable on the [Installation](#installation) section. diff --git a/app/__tests__/municipalities/createMunicipalityInstance.js b/app/__tests__/municipalities/createMunicipalityInstance.js index b7eae32..55ab750 100644 --- a/app/__tests__/municipalities/createMunicipalityInstance.js +++ b/app/__tests__/municipalities/createMunicipalityInstance.js @@ -49,7 +49,8 @@ const createMunicipalityInstance = (excelFile) => { logger.log( `[INFO]: Parsed municipalities from config: ${config.countMunicipalities}\n` + - `loaded municipalities from Excel file: ${excel.countMunicipalities}\n`, { + `[INFO]: Parsed municipalities from Excel file: ${excel.countMunicipalities}\n` + + `[INFO]: Total data rows from Excel file: ${excelFile.data.length}, SheetJS (Excel) header rows count: ${excelFile.options.dataRowStart}\n`, { color: ColorLog.COLORS.TEXT.CYAN }) diff --git a/app/__tests__/municipalities/municipalitiesCount.js b/app/__tests__/municipalities/municipalitiesCount.js index 7605a88..385d13e 100644 --- a/app/__tests__/municipalities/municipalitiesCount.js +++ b/app/__tests__/municipalities/municipalitiesCount.js @@ -36,6 +36,8 @@ describe('Municipalities total count match', () => { hasMissingInConfig } = createMunicipalityInstance(excelFile) + let totalMunicipalitiesConfig = config.countMunicipalities + // Process missing PAGASA seasonal config provinces/municipalities if (hasMissingInConfig) { const fromConfig = excel.provinces.filter(item => !config.provinces.has(item)) @@ -44,6 +46,9 @@ describe('Municipalities total count match', () => { excelFile.listMunicipalities({ provinces: fromConfig }) ).flat()?.length ?? 0 + // Add missing Excel municipalities count + totalMunicipalitiesConfig += countMissingConfig + logger.log( `[WARNING]: ${fromConfig.length} PROVINCE(S) PRESENT in the 10-day Excel file\n` + `but MISSING in the PAGASA seasonal config: ${arrayToString(fromConfig)}, ${countMissingConfig} municipalities`, { @@ -68,21 +73,29 @@ describe('Municipalities total count match', () => { 'or extend and override the ExcelFile or ExcelFactory classes in your scripts.', { color: ColorLog.COLORS.TEXT.RED }) + + let passMsg = '[20240826]: Allow the test to succeed here since there is little information about updated\n' + passMsg += 'PAGASA seasonal & 10-day province/municipalities naming conventions for the other regions, and they\n' + passMsg += 'may change anytime without prior notice. Take note of the INFOS/WARNINGS and\n' + passMsg += 'extend/override the class methods on custom scripts to accommodate custom settings as necessary' + expect(logger.log(passMsg, { color: ColorLog.COLORS.TEXT.YELLOW })).toBe(undefined) } else { logger.log('[MUNICIPALITIES]: Municipalities counts match in config and Excel', { color: ColorLog.COLORS.TEXT.GREEN }) } - let passMsg = '[20240826]: Allow the test to succeed here since there is little information about updated\n' - passMsg += 'PAGASA seasonal & 10-day province/municipalities naming conventions for the other regions, and they\n' - passMsg += 'may change anytime without prior notice. Take note of the INFOS/WARNINGS and\n' - passMsg += 'extend/override the class methods on custom scripts to accommodate custom settings as necessary' - expect(logger.log(passMsg, { color: ColorLog.COLORS.TEXT.YELLOW })).toBe(undefined) - - /* Uncomment true "tests" for testing - expect(countExcelMunicipalities).toBe(countConfigMunicipalities) - expect(allExcelProvinces.length).toBe(allProvinces.size) + /* Uncomment true "tests" for municipalities count match testing + expect(excel.countMunicipalities).toBe(config.countMunicipalities) + expect(excel.provinces.length).toBe(config.provinces.size) */ + + if (excelFile.options.dataRowStart > 0) { + // Parsed/loaded municipalities in the Excel file using the (manual-encoded) PAGASA seasonal config + // including provinces missing in the config should be equal to the raw loaded data count + expect(totalMunicipalitiesConfig + excelFile.options.dataRowStart).toBe(excelFile.data.length) + } else { + throw new Error('Invalid 10-day Excel file format: Missing "Project Areas" text') + } }) }) diff --git a/app/__tests__/provinces/updateInstances.js b/app/__tests__/provinces/updateInstances.js index b9d5285..d376639 100644 --- a/app/__tests__/provinces/updateInstances.js +++ b/app/__tests__/provinces/updateInstances.js @@ -49,7 +49,7 @@ const updateInstances = ({ // Provinces names do not match in 10-Day Excel file and the (PAGASA seasonal) config file if (fromExcel.length > 0 || fromConfig.length > 0) { let msg = `[INFO]: Original provinces count are: ${allProvinces.length} (PAGASA seasonal config) vs. ${allExcelProvinces.length} (10-Day Excel file)\n` - msg += '[INFO]: Removed incosistent provinces in the config and Excel file during check (see yellow WARNINGs)\n' + msg += '[INFO]: Removed incosistent provinces in the config and Excel file only during checking/testing (see yellow WARNINGs)\n' msg += `[INFO]: Modified provinces count are: ${uniqueProvinces.size} (PAGASA seasonal config) vs. ${uniqueExcelProvinces.size} (10-Day Excel file)\n\n` msg += '[NOTE]: If these you believe these INFOs are incorrect, feel free to reach out or extend and override\n' msg += 'the ExcelFile or ExcelFactory classes in your scripts to customize this behaviour and other settings.' diff --git a/app/src/classes/excel/index.js b/app/src/classes/excel/index.js index 4000ce4..17e94fc 100644 --- a/app/src/classes/excel/index.js +++ b/app/src/classes/excel/index.js @@ -21,9 +21,24 @@ class ExcelFile { /** Full file path to excel file on local storage */ #pathToFile = null - /** Region information from the /config/regions.json or other config file */ + /** Region information from the /app/config/regions.json or other config file */ #settings = null + /** Other app settings and configurations */ + #options = { + /** + * SheetJS array index number translated from the Excel headers row count + * before elements containing "municipalityName (provinceName)" data + */ + dataRowStart: 0, + + /** Internal excel file column name read by sheetjs. + * This column contains strings following the pattern + * "municipalityName (provinceName)" + */ + SHEETJS_COL: process.env.SHEETJS_COLUMN || '__EMPTY' + } + /** Excel workbook object parsed by sheetjs */ #workbook = null @@ -40,12 +55,6 @@ class ExcelFile { */ #datalist = [] - /** Internal excel file column name read by sheetjs. - * This column contains strings following the pattern - * "municipalityName (provinceName)" - */ - #SHEETJS_COL = process.env.SHEETJS_COLUMN || '__EMPTY' - /** Event emitter for listening to custom events */ events = new EventEmitter() @@ -66,7 +75,7 @@ class ExcelFile { * @param {Bool} [params.fastload] - (Optional) Start loading and parsing the local excel file on class initialization if the "url" param is not provided. * - If `false` or not provided, call the `.init()` method later on a more convenient time. */ - constructor ({ url, pathToFile, fastload = true, settings = null } = {}) { + constructor ({ url, pathToFile, fastload = true, settings = null, options = null } = {}) { if (url === '' || pathToFile === '') { throw new Error('Missing remote file url or local file path.') } @@ -79,6 +88,8 @@ class ExcelFile { throw new Error('pathToFile should contain an excel file name ending in .xlsx') } + this.setOptions(options) + // Set the local Excel file path this.#pathToFile = pathToFile @@ -141,10 +152,10 @@ class ExcelFile { this.#data = XLSX.utils.sheet_to_json(this.#workbook.Sheets[this.#sheets[0]]) // Extract the municipality and province names - this.#datalist = this.#data.reduce((acc, row) => { - if (row[this.#SHEETJS_COL] !== undefined && this.followsStringPattern(row[this.#SHEETJS_COL])) { - const municipality = this.getMunicipalityName(row[this.#SHEETJS_COL]) - const province = this.getProvinceName(row[this.#SHEETJS_COL]) + this.#datalist = this.#data.reduce((acc, row, index) => { + if (row[this.#options.SHEETJS_COL] !== undefined && this.followsStringPattern(row[this.#options.SHEETJS_COL])) { + const municipality = this.getMunicipalityName(row[this.#options.SHEETJS_COL]) + const province = this.getProvinceName(row[this.#options.SHEETJS_COL]) if (province !== null) { acc.push({ @@ -152,6 +163,13 @@ class ExcelFile { province }) } + } else { + // Find the SheetJS array index of rows containing data + // Note: this relies on the structure of the default Excel file in /app/data/day1.xlsx or similar + if (row[this.#options.SHEETJS_COL] === 'Project Areas') { + const OFFSET_FROM_FLAG = 2 + this.#options.dataRowStart = index + OFFSET_FROM_FLAG + } } return acc @@ -205,6 +223,21 @@ class ExcelFile { return /[a-zA-Z,.] *\([^)]*\) */.test(str) } + /** + * Sets the local this.#options settings + * @param {Object} options - Miscellaneous app settings defined in this.#options + * @returns + */ + setOptions (options) { + if (!options) return false + + for (const key in this.#options) { + if (options[key] !== undefined) { + this.#options[key] = options[key] + } + } + } + /** * Checks if a string contains special characters * @param {String} str - String to check @@ -301,6 +334,11 @@ class ExcelFile { return this.#settings } + // Retuls the local options object + get options () { + return this.#options + } + // Returns the full path to the 10-day weather forecast Excel file get pathToFile () { return this.#pathToFile From 80d2c8aa10b7bf3c8ca45e7f80572171ac03a6ec Mon Sep 17 00:00:00 2001 From: ciatph Date: Wed, 28 Aug 2024 01:18:09 +0800 Subject: [PATCH 2/4] chore: display 10-day excel file info log --- .../municipalities/municipalitiesCount.js | 2 +- app/__tests__/provinces/createInstances.js | 12 ++++++--- app/__tests__/provinces/updateInstances.js | 2 +- app/src/classes/excel/index.js | 26 +++++++++++++++++-- app/src/lib/utils.js | 24 +++++++++++++++-- 5 files changed, 56 insertions(+), 10 deletions(-) diff --git a/app/__tests__/municipalities/municipalitiesCount.js b/app/__tests__/municipalities/municipalitiesCount.js index 385d13e..732b56f 100644 --- a/app/__tests__/municipalities/municipalitiesCount.js +++ b/app/__tests__/municipalities/municipalitiesCount.js @@ -70,7 +70,7 @@ describe('Municipalities total count match', () => { if (hasMissingInConfig || hasMissingInExcel) { logger.log( '[INFO]: If you believe these RED warning(s) are incorrect, feel free to reach out\n' + - 'or extend and override the ExcelFile or ExcelFactory classes in your scripts.', { + 'or you may extend and override the ExcelFile or ExcelFactory classes in your scripts.', { color: ColorLog.COLORS.TEXT.RED }) diff --git a/app/__tests__/provinces/createInstances.js b/app/__tests__/provinces/createInstances.js index 2b29162..a0bb48b 100644 --- a/app/__tests__/provinces/createInstances.js +++ b/app/__tests__/provinces/createInstances.js @@ -37,10 +37,14 @@ const createInstances = (excelFile) => { // Action: remove these provinces from provinces count equality check const fromExcel = allExcelProvinces.filter(item => !allProvinces.includes(item)) - // Log other information - logger.log(`[INFO]: Loaded ${excelFile.datalist.length} data rows`, { - color: ColorLog.COLORS.TEXT.GREEN - }) + // Log other info + const dataSource = excelFile?.url ?? 'default local 10-Day Excel file' + + let message = `[INFO]: Loaded ${excelFile.data.length} Excel rows\n` + message += `[INFO]: Parsed ${excelFile.datalist.length} data rows\n` + message += `[INFO]: from ${dataSource}\n` + message += `[INFO]: ${excelFile.metadata.forecastDate}` + logger.log(message, { color: ColorLog.COLORS.TEXT.GREEN }) return { allExcelProvinces, diff --git a/app/__tests__/provinces/updateInstances.js b/app/__tests__/provinces/updateInstances.js index d376639..3bb8e29 100644 --- a/app/__tests__/provinces/updateInstances.js +++ b/app/__tests__/provinces/updateInstances.js @@ -51,7 +51,7 @@ const updateInstances = ({ let msg = `[INFO]: Original provinces count are: ${allProvinces.length} (PAGASA seasonal config) vs. ${allExcelProvinces.length} (10-Day Excel file)\n` msg += '[INFO]: Removed incosistent provinces in the config and Excel file only during checking/testing (see yellow WARNINGs)\n' msg += `[INFO]: Modified provinces count are: ${uniqueProvinces.size} (PAGASA seasonal config) vs. ${uniqueExcelProvinces.size} (10-Day Excel file)\n\n` - msg += '[NOTE]: If these you believe these INFOs are incorrect, feel free to reach out or extend and override\n' + msg += '[NOTE]: If these you believe these INFOs are incorrect, feel free to reach out or you may extend and override\n' msg += 'the ExcelFile or ExcelFactory classes in your scripts to customize this behaviour and other settings.' logger.log(msg, { diff --git a/app/src/classes/excel/index.js b/app/src/classes/excel/index.js index 17e94fc..80244fe 100644 --- a/app/src/classes/excel/index.js +++ b/app/src/classes/excel/index.js @@ -9,6 +9,8 @@ const Schema = require('../schema') const regionSchema = require('../../lib/schemas/regionSchema') const defaultRegionsConfig = require('../../../config/regions.json') +const { capitalizeText } = require('../../lib/utils') + /** * Load, process and parse an Excel File containing a list of PH municipalities. * The Excel File should contain a column with string pattern: @@ -24,6 +26,12 @@ class ExcelFile { /** Region information from the /app/config/regions.json or other config file */ #settings = null + /** 10-day Excel file information */ + #metadata = { + // Weather forecast date + forecastDate: null + } + /** Other app settings and configurations */ #options = { /** @@ -170,6 +178,15 @@ class ExcelFile { const OFFSET_FROM_FLAG = 2 this.#options.dataRowStart = index + OFFSET_FROM_FLAG } + + if (this.#metadata.forecastDate === null) { + const contentAsKeys = Object.keys(row ?? '') + const content = contentAsKeys.filter(item => item.includes('FORECAST DATE')) + + this.#metadata.forecastDate = content.length > 0 + ? capitalizeText(content[0]) + : 'Forecast Date: n/a' + } } return acc @@ -334,11 +351,16 @@ class ExcelFile { return this.#settings } - // Retuls the local options object + // Returns the local options object get options () { return this.#options } + // Returns the loaded Excel file's metadata + get metadata () { + return this.#metadata + } + // Returns the full path to the 10-day weather forecast Excel file get pathToFile () { return this.#pathToFile @@ -464,7 +486,7 @@ class ExcelFile { const keys = [...Object.keys(this.#settings.data[0])] if ( - !keys.includes(key) || !typeof key === 'string' + !keys.includes(key) || typeof key !== 'string' ) { return [] } diff --git a/app/src/lib/utils.js b/app/src/lib/utils.js index 376c5b6..cf2a6bb 100644 --- a/app/src/lib/utils.js +++ b/app/src/lib/utils.js @@ -9,9 +9,29 @@ const isObject = (item) => { item.constructor === Object } -const arrayToString = (array) => array.toString().split(',').join(', ') +/** + * Converts an Array of strings (text) into a single comma-separated string + * @param {String[]} arrayOfText - Array containing String items + * @returns {String} Comma-separated text + */ +const arrayToString = (arrayOfText) => arrayOfText.toString().split(',').join(', ') + +/** + * Capitalizes the first letter of words in a text + * @param {String} text - String text + * @returns {String} Capitalized text + */ +const capitalizeText = (text) => { + if (typeof text !== 'string') return null + + return text + .split(' ') + .map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase()) + .join(' ') +} module.exports = { isObject, - arrayToString + arrayToString, + capitalizeText } From 58734ab5aff486023a756b510b0013ab9d70bace Mon Sep 17 00:00:00 2001 From: ciatph Date: Wed, 28 Aug 2024 01:23:09 +0800 Subject: [PATCH 3/4] chore: bump version patch v1.3.1 --- app/package-lock.json | 4 ++-- app/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/package-lock.json b/app/package-lock.json index 9f49b7e..a3f30da 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -1,12 +1,12 @@ { "name": "ph-municipalities", - "version": "1.3.0", + "version": "1.3.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "ph-municipalities", - "version": "1.3.0", + "version": "1.3.1", "license": "ISC", "dependencies": { "dotenv": "^16.0.1", diff --git a/app/package.json b/app/package.json index 397cc38..7dd295e 100644 --- a/app/package.json +++ b/app/package.json @@ -1,6 +1,6 @@ { "name": "ph-municipalities", - "version": "1.3.0", + "version": "1.3.1", "description": "List and write the `municipalities` of Philippines provinces or regions into JSON files", "main": "index.js", "engines": { From 98a89ea7597d0dddd70a79d3a5a7806017ff283a Mon Sep 17 00:00:00 2001 From: ciatph Date: Wed, 28 Aug 2024 01:28:29 +0800 Subject: [PATCH 4/4] chore: string pattern should end in space or ')' --- app/src/classes/excel/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/src/classes/excel/index.js b/app/src/classes/excel/index.js index 80244fe..61c17d8 100644 --- a/app/src/classes/excel/index.js +++ b/app/src/classes/excel/index.js @@ -237,7 +237,7 @@ class ExcelFile { * @returns {Bool} true | false */ followsStringPattern (str) { - return /[a-zA-Z,.] *\([^)]*\) */.test(str) + return /[a-zA-Z,.] *\([^)]*\) *$/.test(str) } /** @@ -325,6 +325,8 @@ class ExcelFile { * @returns {null} Returns null if "provinceName" is not found */ getProvinceName (str) { + if (!str) return null + const match = str.match(/\(([^)]+)\)/) return (match !== null) ? match[1]