From fba96d4069c7b46178cfd2bf79f678568323994d Mon Sep 17 00:00:00 2001 From: Scott Horowitz Date: Wed, 25 Sep 2024 23:00:07 -0600 Subject: [PATCH 1/3] Speeds up reading/writing/processing detailed schedule files and writing CSV timeseries output by avoiding the CSV library. --- BuildResidentialScheduleFile/measure.xml | 6 +++--- BuildResidentialScheduleFile/resources/schedules.rb | 9 ++++++--- HPXMLtoOpenStudio/measure.xml | 6 +++--- HPXMLtoOpenStudio/resources/schedules.rb | 13 +++++++------ ReportSimulationOutput/measure.rb | 3 ++- ReportSimulationOutput/measure.xml | 6 +++--- 6 files changed, 24 insertions(+), 19 deletions(-) diff --git a/BuildResidentialScheduleFile/measure.xml b/BuildResidentialScheduleFile/measure.xml index 45494d5ec8..a50e1fe292 100644 --- a/BuildResidentialScheduleFile/measure.xml +++ b/BuildResidentialScheduleFile/measure.xml @@ -3,8 +3,8 @@ 3.1 build_residential_schedule_file f770b2db-1a9f-4e99-99a7-7f3161a594b1 - 1872e0df-889e-46ae-8cb9-849eadb97fda - 2024-09-13T21:54:22Z + 1fc75fd9-df4c-4ba6-8eff-2366ba375060 + 2024-09-26T04:58:59Z 03F02484 BuildResidentialScheduleFile Schedule File Builder @@ -229,7 +229,7 @@ schedules.rb rb resource - 1E6EB1B1 + 85B9A7B9 shower_cluster_size_probability.csv diff --git a/BuildResidentialScheduleFile/resources/schedules.rb b/BuildResidentialScheduleFile/resources/schedules.rb index 747dbc0de2..8e1a8682e9 100644 --- a/BuildResidentialScheduleFile/resources/schedules.rb +++ b/BuildResidentialScheduleFile/resources/schedules.rb @@ -853,12 +853,15 @@ def export(schedules_path:) schedule_keys = table[0] + schedule_keys schedule_rows = schedule_rows.map.with_index { |row, i| table[i + 1] + row } end - CSV.open(schedules_path, 'w') do |csv| - csv << schedule_keys + + # Note: We don't use the CSV library here because it's slow for large files + File.open(schedules_path, 'w') do |csv| + csv << "#{schedule_keys.join(',')}\n" schedule_rows.each do |row| - csv << row + csv << "#{row.join(',')}\n" end end + return true end diff --git a/HPXMLtoOpenStudio/measure.xml b/HPXMLtoOpenStudio/measure.xml index 3f48f02227..a7375aee8c 100644 --- a/HPXMLtoOpenStudio/measure.xml +++ b/HPXMLtoOpenStudio/measure.xml @@ -3,8 +3,8 @@ 3.1 hpxm_lto_openstudio b1543b30-9465-45ff-ba04-1d1f85e763bc - 453f8c5f-d922-42a5-8815-cd92b9b23fa0 - 2024-09-25T21:40:48Z + 330fd56f-94ba-41d1-8d42-d2eaecbbacf0 + 2024-09-26T04:59:02Z D8922A73 HPXMLtoOpenStudio HPXML to OpenStudio Translator @@ -591,7 +591,7 @@ schedules.rb rb resource - 14BD47A8 + 7BE6BB34 simcontrols.rb diff --git a/HPXMLtoOpenStudio/resources/schedules.rb b/HPXMLtoOpenStudio/resources/schedules.rb index 1bc35dff94..348e4b0215 100644 --- a/HPXMLtoOpenStudio/resources/schedules.rb +++ b/HPXMLtoOpenStudio/resources/schedules.rb @@ -1287,7 +1287,8 @@ def import(schedules_paths) num_hrs_in_year = Calendar.num_hours_in_year(@year) @schedules = {} schedules_paths.each do |schedules_path| - columns = CSV.read(schedules_path).transpose + # Note: We don't use the CSV library here because it's slow for large files + columns = File.readlines(schedules_path).map(&:strip).map { |r| r.split(',') }.transpose columns.each do |col| col_name = col[0] column = Columns.values.find { |c| c.name == col_name } @@ -1348,11 +1349,11 @@ def import(schedules_paths) def export() return false if @output_schedules_path.nil? - CSV.open(@output_schedules_path, 'wb') do |csv| - csv << @tmp_schedules.keys - rows = @tmp_schedules.values.transpose - rows.each do |row| - csv << row + # Note: We don't use the CSV library here because it's slow for large files + File.open(@output_schedules_path, 'w') do |csv| + csv << "#{@tmp_schedules.keys.join(',')}\n" + @tmp_schedules.values.transpose.each do |row| + csv << "#{row.join(',')}\n" end end diff --git a/ReportSimulationOutput/measure.rb b/ReportSimulationOutput/measure.rb index 91018688b9..cedf83642e 100644 --- a/ReportSimulationOutput/measure.rb +++ b/ReportSimulationOutput/measure.rb @@ -1869,7 +1869,8 @@ def report_timeseries_output_results(runner, outputs, timeseries_output_path, ar end # Write file - CSV.open(timeseries_output_path, 'wb') { |csv| data.to_a.each { |elem| csv << elem } } + # Note: We don't use the CSV library here because it's slow for large files + File.open(timeseries_output_path, 'wb') { |csv| data.to_a.each { |elem| csv << "#{elem.join(',')}\n" } } elsif ['json', 'msgpack'].include? args[:output_format] # Assemble data h = {} diff --git a/ReportSimulationOutput/measure.xml b/ReportSimulationOutput/measure.xml index 5b595ae85c..1a9c7d902d 100644 --- a/ReportSimulationOutput/measure.xml +++ b/ReportSimulationOutput/measure.xml @@ -3,8 +3,8 @@ 3.1 report_simulation_output df9d170c-c21a-4130-866d-0d46b06073fd - 22b3e453-eba9-432d-8e44-e0afdb46bb9e - 2024-08-21T16:00:15Z + 29a26cd3-0ea4-4c33-9d29-e80f2f58b8f5 + 2024-09-26T04:59:04Z 9BF1E6AC ReportSimulationOutput HPXML Simulation Output Report @@ -1929,7 +1929,7 @@ measure.rb rb script - C4D35B1C + 877A9DB3 test_report_sim_output.rb From e6ab5eabfc6292aec3b35e3b184175a2a44bb752 Mon Sep 17 00:00:00 2001 From: Scott Horowitz Date: Wed, 25 Sep 2024 23:05:23 -0600 Subject: [PATCH 2/3] Might as well do it here too for consistency. --- ReportUtilityBills/measure.rb | 3 ++- ReportUtilityBills/measure.xml | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ReportUtilityBills/measure.rb b/ReportUtilityBills/measure.rb index c564a418a6..b1baf8317e 100644 --- a/ReportUtilityBills/measure.rb +++ b/ReportUtilityBills/measure.rb @@ -488,7 +488,8 @@ def report_monthly_output_results(runner, args, timestamps, monthly_data, monthl data = data.zip(*monthly_data) # Write file - CSV.open(monthly_output_path, 'wb') { |csv| data.to_a.each { |elem| csv << elem } } + # Note: We don't use the CSV library here because it's slow for large files + File.open(monthly_output_path, 'wb') { |csv| data.to_a.each { |elem| csv << "#{elem.join(',')}\n" } } elsif ['json', 'msgpack'].include? args[:output_format] h = {} h['Time'] = data[2..-1] diff --git a/ReportUtilityBills/measure.xml b/ReportUtilityBills/measure.xml index 8fe6f47907..1ac13ef931 100644 --- a/ReportUtilityBills/measure.xml +++ b/ReportUtilityBills/measure.xml @@ -3,8 +3,8 @@ 3.1 report_utility_bills ca88a425-e59a-4bc4-af51-c7e7d1e960fe - 057900e7-7e6b-4ea7-976a-3818a92bdde4 - 2024-09-20T18:14:01Z + dd47ff13-28e8-414a-abb6-940ee40e7c55 + 2024-09-26T05:04:27Z 15BF4E57 ReportUtilityBills Utility Bills Report @@ -180,7 +180,7 @@ measure.rb rb script - 93B04330 + CC656203 detailed_rates/Adams Electric Cooperative Inc - Rate Schedule T1 TOD (Effective 2013-02-01).json From 7f06505e8c1213228d600638846978b0162f947c Mon Sep 17 00:00:00 2001 From: Scott Horowitz Date: Wed, 25 Sep 2024 23:18:43 -0600 Subject: [PATCH 3/3] Update comment for consistency [ci skip] --- HPXMLtoOpenStudio/measure.xml | 6 +++--- HPXMLtoOpenStudio/resources/hpxml_defaults.rb | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/HPXMLtoOpenStudio/measure.xml b/HPXMLtoOpenStudio/measure.xml index a7375aee8c..770afc4840 100644 --- a/HPXMLtoOpenStudio/measure.xml +++ b/HPXMLtoOpenStudio/measure.xml @@ -3,8 +3,8 @@ 3.1 hpxm_lto_openstudio b1543b30-9465-45ff-ba04-1d1f85e763bc - 330fd56f-94ba-41d1-8d42-d2eaecbbacf0 - 2024-09-26T04:59:02Z + 871ee4d0-efe2-4846-80e1-cdc147901ebd + 2024-09-26T05:18:34Z D8922A73 HPXMLtoOpenStudio HPXML to OpenStudio Translator @@ -357,7 +357,7 @@ hpxml_defaults.rb rb resource - ABD33AA9 + AE852A11 hpxml_schema/HPXML.xsd diff --git a/HPXMLtoOpenStudio/resources/hpxml_defaults.rb b/HPXMLtoOpenStudio/resources/hpxml_defaults.rb index 23d8218fd3..ab10004106 100644 --- a/HPXMLtoOpenStudio/resources/hpxml_defaults.rb +++ b/HPXMLtoOpenStudio/resources/hpxml_defaults.rb @@ -4292,7 +4292,7 @@ def self.get_weather_station_csv_data zipcode_csv_filepath = File.join(File.dirname(__FILE__), 'data', 'zipcode_weather_stations.csv') if $zip_csv_data.nil? - # Don't use the CSV library because it's much slower + # Note: We don't use the CSV library here because it's slow for large files $zip_csv_data = File.readlines(zipcode_csv_filepath).map(&:strip) end