Merge pull request #1844 from NREL/avoid_csv_library

Avoid CSV library for speed
NREL · Sep 26, 2024 · 69b9cb7 · 69b9cb7
2 parents c9f71f3 + 7f06505
commit 69b9cb7
Show file tree

Hide file tree

Showing 9 changed files with 31 additions and 25 deletions.
diff --git a/BuildResidentialScheduleFile/measure.xml b/BuildResidentialScheduleFile/measure.xml
@@ -3,8 +3,8 @@
   <schema_version>3.1</schema_version>
   <name>build_residential_schedule_file</name>
   <uid>f770b2db-1a9f-4e99-99a7-7f3161a594b1</uid>
-  <version_id>1872e0df-889e-46ae-8cb9-849eadb97fda</version_id>
-  <version_modified>2024-09-13T21:54:22Z</version_modified>
+  <version_id>1fc75fd9-df4c-4ba6-8eff-2366ba375060</version_id>
+  <version_modified>2024-09-26T04:58:59Z</version_modified>
   <xml_checksum>03F02484</xml_checksum>
   <class_name>BuildResidentialScheduleFile</class_name>
   <display_name>Schedule File Builder</display_name>
@@ -229,7 +229,7 @@
       <filename>schedules.rb</filename>
       <filetype>rb</filetype>
       <usage_type>resource</usage_type>
-      <checksum>1E6EB1B1</checksum>
+      <checksum>85B9A7B9</checksum>
     </file>
     <file>
       <filename>shower_cluster_size_probability.csv</filename>

diff --git a/BuildResidentialScheduleFile/resources/schedules.rb b/BuildResidentialScheduleFile/resources/schedules.rb
@@ -853,12 +853,15 @@ def export(schedules_path:)
       schedule_keys = table[0] + schedule_keys
       schedule_rows = schedule_rows.map.with_index { |row, i| table[i + 1] + row }
     end
-    CSV.open(schedules_path, 'w') do |csv|
-      csv << schedule_keys
+
+    # Note: We don't use the CSV library here because it's slow for large files
+    File.open(schedules_path, 'w') do |csv|
+      csv << "#{schedule_keys.join(',')}\n"
       schedule_rows.each do |row|
-        csv << row
+        csv << "#{row.join(',')}\n"
       end
     end
+
     return true
   end
 

diff --git a/HPXMLtoOpenStudio/measure.xml b/HPXMLtoOpenStudio/measure.xml
@@ -3,8 +3,8 @@
   <schema_version>3.1</schema_version>
   <name>hpxm_lto_openstudio</name>
   <uid>b1543b30-9465-45ff-ba04-1d1f85e763bc</uid>
-  <version_id>453f8c5f-d922-42a5-8815-cd92b9b23fa0</version_id>
-  <version_modified>2024-09-25T21:40:48Z</version_modified>
+  <version_id>871ee4d0-efe2-4846-80e1-cdc147901ebd</version_id>
+  <version_modified>2024-09-26T05:18:34Z</version_modified>
   <xml_checksum>D8922A73</xml_checksum>
   <class_name>HPXMLtoOpenStudio</class_name>
   <display_name>HPXML to OpenStudio Translator</display_name>
@@ -357,7 +357,7 @@
       <filename>hpxml_defaults.rb</filename>
       <filetype>rb</filetype>
       <usage_type>resource</usage_type>
-      <checksum>ABD33AA9</checksum>
+      <checksum>AE852A11</checksum>
     </file>
     <file>
       <filename>hpxml_schema/HPXML.xsd</filename>
@@ -591,7 +591,7 @@
       <filename>schedules.rb</filename>
       <filetype>rb</filetype>
       <usage_type>resource</usage_type>
-      <checksum>14BD47A8</checksum>
+      <checksum>7BE6BB34</checksum>
     </file>
     <file>
       <filename>simcontrols.rb</filename>

diff --git a/HPXMLtoOpenStudio/resources/hpxml_defaults.rb b/HPXMLtoOpenStudio/resources/hpxml_defaults.rb
@@ -4292,7 +4292,7 @@ def self.get_weather_station_csv_data
     zipcode_csv_filepath = File.join(File.dirname(__FILE__), 'data', 'zipcode_weather_stations.csv')
 
     if $zip_csv_data.nil?
-      # Don't use the CSV library because it's much slower
+      # Note: We don't use the CSV library here because it's slow for large files
       $zip_csv_data = File.readlines(zipcode_csv_filepath).map(&:strip)
     end
 

diff --git a/HPXMLtoOpenStudio/resources/schedules.rb b/HPXMLtoOpenStudio/resources/schedules.rb
@@ -1287,7 +1287,8 @@ def import(schedules_paths)
     num_hrs_in_year = Calendar.num_hours_in_year(@year)
     @schedules = {}
     schedules_paths.each do |schedules_path|
-      columns = CSV.read(schedules_path).transpose
+      # Note: We don't use the CSV library here because it's slow for large files
+      columns = File.readlines(schedules_path).map(&:strip).map { |r| r.split(',') }.transpose
       columns.each do |col|
         col_name = col[0]
         column = Columns.values.find { |c| c.name == col_name }
@@ -1348,11 +1349,11 @@ def import(schedules_paths)
   def export()
     return false if @output_schedules_path.nil?
 
-    CSV.open(@output_schedules_path, 'wb') do |csv|
-      csv << @tmp_schedules.keys
-      rows = @tmp_schedules.values.transpose
-      rows.each do |row|
-        csv << row
+    # Note: We don't use the CSV library here because it's slow for large files
+    File.open(@output_schedules_path, 'w') do |csv|
+      csv << "#{@tmp_schedules.keys.join(',')}\n"
+      @tmp_schedules.values.transpose.each do |row|
+        csv << "#{row.join(',')}\n"
       end
     end
 

diff --git a/ReportSimulationOutput/measure.rb b/ReportSimulationOutput/measure.rb
@@ -1869,7 +1869,8 @@ def report_timeseries_output_results(runner, outputs, timeseries_output_path, ar
       end
 
       # Write file
-      CSV.open(timeseries_output_path, 'wb') { |csv| data.to_a.each { |elem| csv << elem } }
+      # Note: We don't use the CSV library here because it's slow for large files
+      File.open(timeseries_output_path, 'wb') { |csv| data.to_a.each { |elem| csv << "#{elem.join(',')}\n" } }
     elsif ['json', 'msgpack'].include? args[:output_format]
       # Assemble data
       h = {}

diff --git a/ReportSimulationOutput/measure.xml b/ReportSimulationOutput/measure.xml
@@ -3,8 +3,8 @@
   <schema_version>3.1</schema_version>
   <name>report_simulation_output</name>
   <uid>df9d170c-c21a-4130-866d-0d46b06073fd</uid>
-  <version_id>22b3e453-eba9-432d-8e44-e0afdb46bb9e</version_id>
-  <version_modified>2024-08-21T16:00:15Z</version_modified>
+  <version_id>29a26cd3-0ea4-4c33-9d29-e80f2f58b8f5</version_id>
+  <version_modified>2024-09-26T04:59:04Z</version_modified>
   <xml_checksum>9BF1E6AC</xml_checksum>
   <class_name>ReportSimulationOutput</class_name>
   <display_name>HPXML Simulation Output Report</display_name>
@@ -1929,7 +1929,7 @@
       <filename>measure.rb</filename>
       <filetype>rb</filetype>
       <usage_type>script</usage_type>
-      <checksum>C4D35B1C</checksum>
+      <checksum>877A9DB3</checksum>
     </file>
     <file>
       <filename>test_report_sim_output.rb</filename>

diff --git a/ReportUtilityBills/measure.rb b/ReportUtilityBills/measure.rb
@@ -488,7 +488,8 @@ def report_monthly_output_results(runner, args, timestamps, monthly_data, monthl
         data = data.zip(*monthly_data)
 
         # Write file
-        CSV.open(monthly_output_path, 'wb') { |csv| data.to_a.each { |elem| csv << elem } }
+        # Note: We don't use the CSV library here because it's slow for large files
+        File.open(monthly_output_path, 'wb') { |csv| data.to_a.each { |elem| csv << "#{elem.join(',')}\n" } }
       elsif ['json', 'msgpack'].include? args[:output_format]
         h = {}
         h['Time'] = data[2..-1]

diff --git a/ReportUtilityBills/measure.xml b/ReportUtilityBills/measure.xml
@@ -3,8 +3,8 @@
   <schema_version>3.1</schema_version>
   <name>report_utility_bills</name>
   <uid>ca88a425-e59a-4bc4-af51-c7e7d1e960fe</uid>
-  <version_id>057900e7-7e6b-4ea7-976a-3818a92bdde4</version_id>
-  <version_modified>2024-09-20T18:14:01Z</version_modified>
+  <version_id>dd47ff13-28e8-414a-abb6-940ee40e7c55</version_id>
+  <version_modified>2024-09-26T05:04:27Z</version_modified>
   <xml_checksum>15BF4E57</xml_checksum>
   <class_name>ReportUtilityBills</class_name>
   <display_name>Utility Bills Report</display_name>
@@ -180,7 +180,7 @@
       <filename>measure.rb</filename>
       <filetype>rb</filetype>
       <usage_type>script</usage_type>
-      <checksum>93B04330</checksum>
+      <checksum>CC656203</checksum>
     </file>
     <file>
       <filename>detailed_rates/Adams Electric Cooperative Inc - Rate Schedule T1 TOD (Effective 2013-02-01).json</filename>