Merge pull request #86 from ciatph/dev

v1.3.1

README.md

@@ -40,7 +40,8 @@ The following dependencies are used for this project. Feel free to use other dep
    - npm v8.5.0
 4. Excel file
    - ph-municipalities uses Excel files in the `/app/data` directory as data source.
-   - At minimum, the excel file should have a **column** that contains municipality and province names following the pattern `"municipalityName (provinceName)"`
+   - At minimum, the Excel file should have a **column** that contains municipality and province names following the pattern `"municipalityName (provinceName)"`
+   - (Optional) The Excel file should have a row on the same **column** as above containing the text `"Project Areas"` plus two (2) blank rows before rows containing municipality and province names to enable strict testing and validation of the number of parsed data rows
    - Checkout the excel file format on the `/app/data/day1.xlsx` sample file for more information
 5. (Optional) Download URL for a remote excel file.
    - See the `EXCEL_FILE_URL` variable on the [Installation](#installation) section.

app/__tests__/municipalities/createMunicipalityInstance.js

@@ -49,7 +49,8 @@ const createMunicipalityInstance = (excelFile) => {
 
     logger.log(
       `[INFO]: Parsed municipalities from config: ${config.countMunicipalities}\n` +
-      `loaded municipalities from Excel file: ${excel.countMunicipalities}\n`, {
+      `[INFO]: Parsed municipalities from Excel file: ${excel.countMunicipalities}\n` +
+      `[INFO]: Total data rows from Excel file: ${excelFile.data.length}, SheetJS (Excel) header rows count: ${excelFile.options.dataRowStart}\n`, {
         color: ColorLog.COLORS.TEXT.CYAN
       })
 

app/__tests__/municipalities/municipalitiesCount.js

@@ -36,6 +36,8 @@ describe('Municipalities total count match', () => {
       hasMissingInConfig
     } = createMunicipalityInstance(excelFile)
 
+    let totalMunicipalitiesConfig = config.countMunicipalities
+
     // Process missing PAGASA seasonal config provinces/municipalities
     if (hasMissingInConfig) {
       const fromConfig = excel.provinces.filter(item => !config.provinces.has(item))
@@ -44,6 +46,9 @@ describe('Municipalities total count match', () => {
         excelFile.listMunicipalities({ provinces: fromConfig })
       ).flat()?.length ?? 0
 
+      // Add missing Excel municipalities count
+      totalMunicipalitiesConfig += countMissingConfig
+
       logger.log(
         `[WARNING]: ${fromConfig.length} PROVINCE(S) PRESENT in the 10-day Excel file\n` +
         `but MISSING in the PAGASA seasonal config: ${arrayToString(fromConfig)}, ${countMissingConfig} municipalities`, {
@@ -65,24 +70,32 @@ describe('Municipalities total count match', () => {
     if (hasMissingInConfig || hasMissingInExcel) {
       logger.log(
         '[INFO]: If you believe these RED warning(s) are incorrect, feel free to reach out\n' +
-        'or extend and override the ExcelFile or ExcelFactory classes in your scripts.', {
+        'or you may extend and override the ExcelFile or ExcelFactory classes in your scripts.', {
           color: ColorLog.COLORS.TEXT.RED
         })
+
+      let passMsg = '[20240826]: Allow the test to succeed here since there is little information about updated\n'
+      passMsg += 'PAGASA seasonal & 10-day province/municipalities naming conventions for the other regions, and they\n'
+      passMsg += 'may change anytime without prior notice. Take note of the INFOS/WARNINGS and\n'
+      passMsg += 'extend/override the class methods on custom scripts to accommodate custom settings as necessary'
+      expect(logger.log(passMsg, { color: ColorLog.COLORS.TEXT.YELLOW })).toBe(undefined)
     } else {
       logger.log('[MUNICIPALITIES]: Municipalities counts match in config and Excel', {
         color: ColorLog.COLORS.TEXT.GREEN
       })
     }
 
-    let passMsg = '[20240826]: Allow the test to succeed here since there is little information about updated\n'
-    passMsg += 'PAGASA seasonal & 10-day province/municipalities naming conventions for the other regions, and they\n'
-    passMsg += 'may change anytime without prior notice. Take note of the INFOS/WARNINGS and\n'
-    passMsg += 'extend/override the class methods on custom scripts to accommodate custom settings as necessary'
-    expect(logger.log(passMsg, { color: ColorLog.COLORS.TEXT.YELLOW })).toBe(undefined)
-
-    /* Uncomment true "tests" for testing
-    expect(countExcelMunicipalities).toBe(countConfigMunicipalities)
-    expect(allExcelProvinces.length).toBe(allProvinces.size)
+    /* Uncomment true "tests" for municipalities count match testing
+    expect(excel.countMunicipalities).toBe(config.countMunicipalities)
+    expect(excel.provinces.length).toBe(config.provinces.size)
     */
+
+    if (excelFile.options.dataRowStart > 0) {
+    // Parsed/loaded municipalities in the Excel file using the (manual-encoded) PAGASA seasonal config
+    // including provinces missing in the config should be equal to the raw loaded data count
+      expect(totalMunicipalitiesConfig + excelFile.options.dataRowStart).toBe(excelFile.data.length)
+    } else {
+      throw new Error('Invalid 10-day Excel file format: Missing "Project Areas" text')
+    }
   })
 })

app/__tests__/provinces/createInstances.js

@@ -37,10 +37,14 @@ const createInstances = (excelFile) => {
     // Action: remove these provinces from provinces count equality check
     const fromExcel = allExcelProvinces.filter(item => !allProvinces.includes(item))
 
-    // Log other information
-    logger.log(`[INFO]: Loaded ${excelFile.datalist.length} data rows`, {
-      color: ColorLog.COLORS.TEXT.GREEN
-    })
+    // Log other info
+    const dataSource = excelFile?.url ?? 'default local 10-Day Excel file'
+
+    let message = `[INFO]: Loaded ${excelFile.data.length} Excel rows\n`
+    message += `[INFO]: Parsed ${excelFile.datalist.length} data rows\n`
+    message += `[INFO]: from ${dataSource}\n`
+    message += `[INFO]: ${excelFile.metadata.forecastDate}`
+    logger.log(message, { color: ColorLog.COLORS.TEXT.GREEN })
 
     return {
       allExcelProvinces,

app/__tests__/provinces/updateInstances.js

@@ -49,9 +49,9 @@ const updateInstances = ({
     // Provinces names do not match in 10-Day Excel file and the (PAGASA seasonal) config file
     if (fromExcel.length > 0 || fromConfig.length > 0) {
       let msg = `[INFO]: Original provinces count are: ${allProvinces.length} (PAGASA seasonal config) vs. ${allExcelProvinces.length} (10-Day Excel file)\n`
-      msg += '[INFO]: Removed incosistent provinces in the config and Excel file during check (see yellow WARNINGs)\n'
+      msg += '[INFO]: Removed incosistent provinces in the config and Excel file only during checking/testing (see yellow WARNINGs)\n'
       msg += `[INFO]: Modified provinces count are: ${uniqueProvinces.size} (PAGASA seasonal config) vs. ${uniqueExcelProvinces.size} (10-Day Excel file)\n\n`
-      msg += '[NOTE]: If these you believe these INFOs are incorrect, feel free to reach out or extend and override\n'
+      msg += '[NOTE]: If these you believe these INFOs are incorrect, feel free to reach out or you may extend and override\n'
       msg += 'the ExcelFile or ExcelFactory classes in your scripts to customize this behaviour and other settings.'
 
       logger.log(msg, {

app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ph-municipalities",
-  "version": "1.3.0",
+  "version": "1.3.1",
   "description": "List and write the `municipalities` of Philippines provinces or regions into JSON files",
   "main": "index.js",
   "engines": {

app/src/classes/excel/index.js

@@ -9,6 +9,8 @@ const Schema = require('../schema')
 const regionSchema = require('../../lib/schemas/regionSchema')
 const defaultRegionsConfig = require('../../../config/regions.json')
 
+const { capitalizeText } = require('../../lib/utils')
+
 /**
  * Load, process and parse an Excel File containing a list of PH municipalities.
  * The Excel File should contain a column with string pattern:
@@ -21,9 +23,30 @@ class ExcelFile {
   /** Full file path to excel file on local storage */
   #pathToFile = null
 
-  /** Region information from the /config/regions.json or other config file */
+  /** Region information from the /app/config/regions.json or other config file */
   #settings = null
 
+  /** 10-day Excel file information */
+  #metadata = {
+    // Weather forecast date
+    forecastDate: null
+  }
+
+  /** Other app settings and configurations */
+  #options = {
+    /**
+     * SheetJS array index number translated from the Excel headers row count
+     * before elements containing "municipalityName (provinceName)" data
+     */
+    dataRowStart: 0,
+
+    /** Internal excel file column name read by sheetjs.
+     *  This column contains strings following the pattern
+     *      "municipalityName (provinceName)"
+     */
+    SHEETJS_COL: process.env.SHEETJS_COLUMN || '__EMPTY'
+  }
+
   /** Excel workbook object parsed by sheetjs */
   #workbook = null
 
@@ -40,12 +63,6 @@ class ExcelFile {
    */
   #datalist = []
 
-  /** Internal excel file column name read by sheetjs.
-   *  This column contains strings following the pattern
-   *      "municipalityName (provinceName)"
-   */
-  #SHEETJS_COL = process.env.SHEETJS_COLUMN || '__EMPTY'
-
   /** Event emitter for listening to custom events */
   events = new EventEmitter()
 
@@ -66,7 +83,7 @@ class ExcelFile {
    * @param {Bool} [params.fastload] - (Optional) Start loading and parsing the local excel file on class initialization if the "url" param is not provided.
    *    - If `false` or not provided, call the `.init()` method later on a more convenient time.
    */
-  constructor ({ url, pathToFile, fastload = true, settings = null } = {}) {
+  constructor ({ url, pathToFile, fastload = true, settings = null, options = null } = {}) {
     if (url === '' || pathToFile === '') {
       throw new Error('Missing remote file url or local file path.')
     }
@@ -79,6 +96,8 @@ class ExcelFile {
       throw new Error('pathToFile should contain an excel file name ending in .xlsx')
     }
 
+    this.setOptions(options)
+
     // Set the local Excel file path
     this.#pathToFile = pathToFile
 
@@ -141,17 +160,33 @@ class ExcelFile {
       this.#data = XLSX.utils.sheet_to_json(this.#workbook.Sheets[this.#sheets[0]])
 
       // Extract the municipality and province names
-      this.#datalist = this.#data.reduce((acc, row) => {
-        if (row[this.#SHEETJS_COL] !== undefined && this.followsStringPattern(row[this.#SHEETJS_COL])) {
-          const municipality = this.getMunicipalityName(row[this.#SHEETJS_COL])
-          const province = this.getProvinceName(row[this.#SHEETJS_COL])
+      this.#datalist = this.#data.reduce((acc, row, index) => {
+        if (row[this.#options.SHEETJS_COL] !== undefined && this.followsStringPattern(row[this.#options.SHEETJS_COL])) {
+          const municipality = this.getMunicipalityName(row[this.#options.SHEETJS_COL])
+          const province = this.getProvinceName(row[this.#options.SHEETJS_COL])
 
           if (province !== null) {
             acc.push({
               municipality: municipality.trim(),
               province
             })
           }
+        } else {
+          // Find the SheetJS array index of rows containing data
+          // Note: this relies on the structure of the default Excel file in /app/data/day1.xlsx or similar
+          if (row[this.#options.SHEETJS_COL] === 'Project Areas') {
+            const OFFSET_FROM_FLAG = 2
+            this.#options.dataRowStart = index + OFFSET_FROM_FLAG
+          }
+
+          if (this.#metadata.forecastDate === null) {
+            const contentAsKeys = Object.keys(row ?? '')
+            const content = contentAsKeys.filter(item => item.includes('FORECAST DATE'))
+
+            this.#metadata.forecastDate = content.length > 0
+              ? capitalizeText(content[0])
+              : 'Forecast Date: n/a'
+          }
         }
 
         return acc
@@ -202,7 +237,22 @@ class ExcelFile {
    * @returns {Bool} true | false
    */
   followsStringPattern (str) {
-    return /[a-zA-Z,.] *\([^)]*\) */.test(str)
+    return /[a-zA-Z,.] *\([^)]*\) *$/.test(str)
+  }
+
+  /**
+   * Sets the local this.#options settings
+   * @param {Object} options - Miscellaneous app settings defined in this.#options
+   * @returns
+   */
+  setOptions (options) {
+    if (!options) return false
+
+    for (const key in this.#options) {
+      if (options[key] !== undefined) {
+        this.#options[key] = options[key]
+      }
+    }
   }
 
   /**
@@ -275,6 +325,8 @@ class ExcelFile {
    * @returns {null} Returns null if "provinceName" is not found
    */
   getProvinceName (str) {
+    if (!str) return null
+
     const match = str.match(/\(([^)]+)\)/)
     return (match !== null)
       ? match[1]
@@ -301,6 +353,16 @@ class ExcelFile {
     return this.#settings
   }
 
+  // Returns the local options object
+  get options () {
+    return this.#options
+  }
+
+  // Returns the loaded Excel file's metadata
+  get metadata () {
+    return this.#metadata
+  }
+
   // Returns the full path to the 10-day weather forecast Excel file
   get pathToFile () {
     return this.#pathToFile
@@ -426,7 +488,7 @@ class ExcelFile {
       const keys = [...Object.keys(this.#settings.data[0])]
 
       if (
-        !keys.includes(key) || !typeof key === 'string'
+        !keys.includes(key) || typeof key !== 'string'
       ) {
         return []
       }

app/src/lib/utils.js

@@ -9,9 +9,29 @@ const isObject = (item) => {
     item.constructor === Object
 }
 
-const arrayToString = (array) => array.toString().split(',').join(', ')
+/**
+ * Converts an Array of strings (text) into a single comma-separated string
+ * @param {String[]} arrayOfText - Array containing String items
+ * @returns {String} Comma-separated text
+ */
+const arrayToString = (arrayOfText) => arrayOfText.toString().split(',').join(', ')
+
+/**
+ * Capitalizes the first letter of words in a text
+ * @param {String} text - String text
+ * @returns {String} Capitalized text
+ */
+const capitalizeText = (text) => {
+  if (typeof text !== 'string') return null
+
+  return text
+    .split(' ')
+    .map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase())
+    .join(' ')
+}
 
 module.exports = {
   isObject,
-  arrayToString
+  arrayToString,
+  capitalizeText
 }