Changes to benchmark analysis for ASPLOS 2019 submission.

Author: arthurp

OrcTests/src/orc/test/analysis/porce/implTypeVsBenchmarkAnalysisAndPlotting.R

@@ -22,7 +22,7 @@ source(file.path(scriptDir, "porce", "utils.R"))
 
 #dataDir <- file.path(experimentDataDir, "PorcE", "impltype-comparison", "20180415-a001")
 #dataDir <- file.path(localExperimentDataDir, "20180730-a010")
-dataDir <- file.path(localExperimentDataDir, "20180802-a001")
+dataDir <- file.path(localExperimentDataDir, "20180805-a002")
 scalaDataDir <- file.path(localExperimentDataDir, "20180718-a002")
 
 loadData <- function(dataDir) {
@@ -165,61 +165,77 @@ print(kable(t, "latex"))
 filledInData <- processedData %>%
   full_join(processedData %>%
               filter(optLevel == 3, implType == "Orc") %>%
-              transmute(benchmarkProblemName, benchmarkName, optLevel = 0, elapsedTime_mean = 8 * 60),
-            by = c("benchmarkProblemName", "benchmarkName", "optLevel")) %>%
+              transmute(benchmarkProblemName, benchmarkName, nCPUs, optLevel = 0, elapsedTime_mean = 10 * 60),
+            by = c("benchmarkProblemName", "benchmarkName", "optLevel", "nCPUs")) %>%
   mutate(elapsedTime_mean = if_else(is.na(elapsedTime_mean.x), elapsedTime_mean.y, elapsedTime_mean.x), elapsedTime_mean.x = NULL, elapsedTime_mean.y = NULL)
 
 times2 <-
-  full_join(filledInData,
-            processedData %>%
-              filter(implType == "Orc", optLevel == 3, optimized == T) %>%
-              transmute(benchmarkProblemName, orc3Time = elapsedTime_mean)) %>%
+        processedData %>%
+          filter(implType == "Orc", optLevel == 3) %>%
+          transmute(benchmarkProblemName, nCPUs, microbenchmark, orc3Time = elapsedTime_mean) %>%
+  full_join(processedData %>%
+              filter(implType == "Orc", optLevel == 0) %>%
+              transmute(benchmarkProblemName, nCPUs, orc0Time = elapsedTime_mean),
+            by = c("benchmarkProblemName", "nCPUs")) %>%
   full_join(processedData %>%
               filter(implType == "Scala") %>%
-              transmute(benchmarkProblemName, scalaTime = elapsedTime_mean)) %>%
+              transmute(benchmarkProblemName, nCPUs, scalaTime = elapsedTime_mean),
+            by = c("benchmarkProblemName", "nCPUs")) %>%
   full_join(processedData %>%
               filter(implType == "Orc+Scala", optimized == T) %>%
-              transmute(benchmarkProblemName, orcScalaTime = elapsedTime_mean))
+              transmute(benchmarkProblemName, nCPUs, orcScalaTime = elapsedTime_mean),
+            by = c("benchmarkProblemName", "nCPUs")) %>%
+  dropBounds()
   #filter(implType == "Orc") %>%
   #mutate(normalizedTime = elapsedTime_mean / orc3Time)
 
 # print(geomean((times2 %>% mutate(normalizedTime = elapsedTime_mean / orc3Time) %>% filter(optLevel == 3))$normalizedTime))
 
-normalizedToScalaTimeAtMost <- times2 %>% mutate(normalizedTime = elapsedTime_mean / if_else(is.na(scalaTime), 8 * 60, scalaTime))
+times2 <- times2 %>% mutate(orcScalaTime = if_else(is.na(orcScalaTime), orc3Time, orcScalaTime))
+
+normalizedToScalaTimeAtMost <- times2 %>% mutate_at(vars(ends_with("Time")), funs(. / if_else(is.na(scalaTime), 10 * 60, scalaTime)))
 
-print(
-  geomean((normalizedToScalaTimeAtMost %>% filter(implType == "Orc", optLevel == 3, microbenchmark == F))$normalizedTime)
+print_expr <- function(expr) {
+  s <- deparse(substitute(expr))
+  cat(paste(s, "=", expr, "\n"))
+}
+
+print_expr(
+  geomean((normalizedToScalaTimeAtMost %>% filter(microbenchmark == F, nCPUs == 24))$orc3Time)
 )
-print(
-  geomean((normalizedToScalaTimeAtMost %>% filter(implType == "Orc", optLevel == 3, microbenchmark == T))$normalizedTime)
+print_expr(
+  geomean((normalizedToScalaTimeAtMost %>% filter(microbenchmark == T, nCPUs == 24))$orc3Time)
 )
-print(
-  geomean((normalizedToScalaTimeAtMost %>% filter(implType == "Orc+Scala", optLevel == 3, microbenchmark == F))$normalizedTime)
+print_expr(
+  geomean((normalizedToScalaTimeAtMost %>% filter(microbenchmark == F, nCPUs == 24))$orcScalaTime)
 )
 
-normalizedToOrcScalaTimeAtMost <- times2 %>% mutate(normalizedTime = elapsedTime_mean / orcScalaTime)
+normalizedToOrcScalaTimeAtMost <- times2 %>% mutate_at(vars(ends_with("Time")), funs(. / if_else(is.na(orcScalaTime), 10 * 60, orcScalaTime)))
 
-print(
-  geomean((normalizedToOrcScalaTimeAtMost %>% filter(implType == "Orc", optLevel == 3, optimized == T, microbenchmark == F))$normalizedTime)
+print_expr(
+  geomean((normalizedToOrcScalaTimeAtMost %>% filter(microbenchmark == F, nCPUs == 24))$orc3Time)
 )
-print(
-  geomean((normalizedToOrcScalaTimeAtMost %>% filter(implType == "Orc", optLevel == 0, optimized == T, microbenchmark == F))$normalizedTime)
+print_expr(
+  geomean((normalizedToOrcScalaTimeAtMost %>% filter(microbenchmark == F, nCPUs == 24))$orc0Time)
 )
-print(
-  geomean((normalizedToOrcScalaTimeAtMost %>% filter(implType == "Scala", microbenchmark == F))$normalizedTime)
+print_expr(
+  geomean((normalizedToOrcScalaTimeAtMost %>% filter(microbenchmark == F, nCPUs == 24))$scalaTime)
 )
 
-normalizedToOrc3Time <- times2 %>% mutate(normalizedTime = elapsedTime_mean / orc3Time)
+normalizedToOrc3TimeAtMost <- times2 %>% mutate_at(vars(ends_with("Time")), funs(. / if_else(is.na(orc3Time), 10 * 60, orc3Time)))
 
-print(
-  geomean((normalizedToOrc3Time %>% filter(implType == "Orc", optLevel == 3, optimized == T, microbenchmark == F))$normalizedTime)
+print_expr(
+  geomean((normalizedToOrc3TimeAtMost %>% filter(microbenchmark == F, nCPUs == 24))$orc0Time)
+)
+print_expr(
+  geomean((normalizedToOrc3TimeAtMost %>% filter(microbenchmark == F, nCPUs == 24))$scalaTime)
 )
 
 
-# print(
+# print_expr(
 #   geomean((times2 %>% mutate(normalizedTime = elapsedTime_mean / scalaTime) %>% filter(implType == "Orc", optLevel == 3, normalizedTime > 1))$normalizedTime)
 # )
-# print(
+# print_expr(
 #   geomean((times2 %>% mutate(normalizedTime = elapsedTime_mean / scalaTime) %>% filter(implType == "Orc+Scala", optLevel == 3, normalizedTime > 1))$normalizedTime)
 # )
 

OrcTests/src/orc/test/analysis/porce/scalingAnalysisAndPlotting.R

@@ -10,6 +10,7 @@
 library(knitr)
 library(tidyr)
 library(stringr)
+library(svglite)
 
 scriptDir <- normalizePath(".") # dirname(dirname(sys.frame(1)$ofile))
 experimentDataDir <- file.path(dirname(dirname(dirname(dirname(dirname(scriptDir))))), "experiment-data")
@@ -23,7 +24,7 @@ source(file.path(scriptDir, "porce", "utils.R"))
 
 # dataDir <- file.path(experimentDataDir, "PorcE", "strong-scaling", "20180203-a009")
 #dataDir <- file.path(localExperimentDataDir, "20180730-a010")
-dataDir <- file.path(localExperimentDataDir, "20180802-a001")
+dataDir <- file.path(localExperimentDataDir, "20180805-a002")
 scalaDataDir <- file.path(localExperimentDataDir, "20180718-a002")
 
 if(!exists("processedData")) {
@@ -60,7 +61,7 @@ processedData <- processedData %>%
   addBenchmarkMetadata() %>%
   group_by(benchmarkProblemName) %>%
   addBaseline(elapsedTime_mean, c(language="Scala", nCPUs=1, isBaseline = T), baseline = elapsedTime_mean_problembaseline) %>%
-  mutate(elapsedTime_mean_problembaseline = replace_na(elapsedTime_mean_problembaseline, 60*6)) %>%
+  mutate(elapsedTime_mean_problembaseline = replace_na(elapsedTime_mean_problembaseline, 60*10)) %>%
   group_by(benchmarkName) %>%
   addBaseline(elapsedTime_mean, c(nCPUs=24), baseline = elapsedTime_mean_selfbaseline) %>%
   ungroup()
@@ -312,6 +313,7 @@ overallScalingPlot <- processedData %>%
     y = elapsedTime_mean_problembaseline / elapsedTime_mean,
     #ymin = elapsedTime_mean_problembaseline / elapsedTime_mean_lowerBound,
     #ymax = elapsedTime_mean_problembaseline / elapsedTime_mean_upperBound,
+    shape = implType,
     color = implType,
     group = benchmarkName
     #linetype = factor(if_else((language == "Orc") & scalaCompute, "Orc+Scala", as.character(language)), levels = c("Orc+Scala", "Orc", "Scala"))
@@ -320,16 +322,21 @@ overallScalingPlot <- processedData %>%
   theme_minimal() +
   #scale_fill_brewer(palette="Set3") +
   #scale_color_brewer(palette="PuBuGn", direction = -1) +
-  scale_color_manual(values = c("#555555", "#E69F00", "#56B4E9")) + # "#67a9cf", "#1c9099", "#016c59"
-  # geom_point(data = processedData %>% filter(benchmarkProblemName != "Swaptions"), alpha = 0.5, shape = 4) +
+  scale_color_manual(name = "Language",
+                     labels = c("Orc", "Orc+Scala", "Scala"),
+                     values = c("#555555", "#E69F00", "#56B4E9")) + # "#67a9cf", "#1c9099", "#016c59"
+  scale_shape_manual(name = "Language",
+                     labels = c("Orc", "Orc+Scala", "Scala"),
+                     values = c(0, 1, 2)) +
+# geom_point(data = processedData %>% filter(benchmarkProblemName != "Swaptions"), alpha = 0.5, shape = 4) +
   # geom_point(aes(shape = granularity), processedData, alpha = 0.7) +
+  #geom_line(aes(y = gcTime_mean), linetype = "dotted") +
   geom_line() +
-  #geom_point() +
+  geom_point() +
   geom_hline(yintercept = 1, alpha = 0.4, color = "blue") +
   scale_y_continuous(limits = c(0, NA)) +
   scale_x_continuous(breaks = c(1, 12, 24), minor_breaks = c(6, 18)) +
   # scale_x_continuous_breaks_from(breaks_from = processedData$nCPUs) +
-  # scale_shape_discrete(solid = F) +
   facet_wrap(~benchmarkProblemName, scales = "free_y", nrow = 1) +
   theme(
     #legend.justification = c("right", "top"),
@@ -341,14 +348,16 @@ overallScalingPlot <- processedData %>%
     legend.spacing = grid::unit(45, "points"),
     text = element_text(size=9),
     legend.text = element_text(size=8),
-    strip.text = element_text(size=9)
+    strip.text = element_text(size=9, angle = 10)
   )
 
 print(overallScalingPlot + theme(legend.position = "bottom"))
 
 #print(overallScalingPlot + scale_y_log10() + theme(legend.position = "bottom"))
 
-ggsave(file.path(outputDir, "allScalingPlot.pdf"), overallScalingPlot + theme(legend.position = "bottom"), width = 9, height = 2, units = "in")
+ggsave(file.path(outputDir, "allScalingPlot.svg"), device = "svg", overallScalingPlot + theme(legend.position = "bottom"), width = 10, height = 1.9, units = "in")
+
+#ggsave(file.path(outputDir, "allScalingPlot.pdf"), overallScalingPlot + theme(legend.position = "bottom"), width = 12, height = 2.1, units = "in")
 
 # svg( file.path(outputDir, "allScalingPlot-legend.svg"), width = 7.5, height = 2 )
 # print(overallScalingPlot + theme(legend.position = "bottom"))