Explicitly initialize contents of scratch buffer to avoid performance anomalies.

elliottslaughter · elliottslaughter · commit 86c018dab8fa · 2019-07-26T14:44:17.000-07:00
diff --git a/build_all.sh b/build_all.sh
@@ -76,7 +76,9 @@ if [[ $USE_LEGION -eq 1 ]]; then
     make -C legion clean
 fi
 if [[ $USE_REGENT -eq 1 ]]; then
-    make -C regent clean
+    SHARD_SIZE=30 make -C regent clean
+    SHARD_SIZE=15 make -C regent clean
+    SHARD_SIZE=14 make -C regent clean
 fi
 if [[ $USE_REALM -eq 1 ]]; then
     make -C realm clean
diff --git a/chapel/task_benchmark.chpl b/chapel/task_benchmark.chpl
@@ -131,6 +131,7 @@ proc execute_task_graph2(graph, task_result, task_ready, task_used) {
 
     var scratch_bytes = graph.scratch_bytes_per_task;
     var scratch_ptr = c_malloc(int(8), scratch_bytes);
+    task_graph_prepare_scratch(scratch_ptr, scratch_bytes);
 
     // Initialize input_ptr and input_bytes... these don't need to
     // change because we can just set n_inputs dynamically.
diff --git a/charm++/subchare.C b/charm++/subchare.C
@@ -96,6 +96,8 @@ void Subchare::initGraph(MulticastMsg* msg) {
   output.resize(graph.output_bytes_per_task);
   scratch.resize(graph.scratch_bytes_per_task);
 
+  TaskGraph::prepare_scratch(scratch.data(), scratch.size());
+
   CProxySection_Subchare::contribute(sid, CkCallback(CkReductionTarget(Main, workerReady), mainProxy));
 }
 
diff --git a/core/core.cc b/core/core.cc
@@ -543,6 +543,8 @@ size_t TaskGraph::num_dependencies(long dset, long point) const
   return SIZE_MAX;
 }
 
+#define MAGIC_VALUE UINT64_C(0x5C4A7C8B) // can you read it? it says "SCRATCHB" (kinda)
+
 void TaskGraph::execute_point(long timestep, long point,
                               char *output_ptr, size_t output_bytes,
                               const char **input_ptr, const size_t *input_bytes,
@@ -607,12 +609,25 @@ void TaskGraph::execute_point(long timestep, long point,
 
   // Validate scratch
   assert(scratch_bytes == scratch_bytes_per_task);
+  if (scratch_bytes > 0) {
+    uint64_t *scratch = reinterpret_cast<uint64_t *>(scratch_ptr);
+    assert(*scratch == MAGIC_VALUE);
+  }
 
   // Execute kernel
   Kernel k(kernel);
   k.execute(graph_index, timestep, point, scratch_ptr, scratch_bytes);
 }
 
+void TaskGraph::prepare_scratch(char *scratch_ptr, size_t scratch_bytes)
+{
+  assert(scratch_bytes % sizeof(uint64_t) == 0);
+  uint64_t *base_ptr = reinterpret_cast<uint64_t *>(scratch_ptr);
+  for (long i = 0; i < scratch_bytes/sizeof(uint64_t); ++i) {
+    base_ptr[i] = MAGIC_VALUE;
+  }
+}
+
 static TaskGraph default_graph(long graph_index)
 {
   TaskGraph graph;
diff --git a/core/core.h b/core/core.h
@@ -69,6 +69,7 @@ struct TaskGraph : public task_graph_t {
                      const char **input_ptr, const size_t *input_bytes,
                      size_t n_inputs,
                      char *scratch_ptr, size_t scratch_bytes) const;
+  static void prepare_scratch(char *scratch_ptr, size_t scratch_bytes);
 };
 
 struct App {
diff --git a/core/core_c.cc b/core/core_c.cc
@@ -148,6 +148,11 @@ void task_graph_execute_point_scratch_nonconst(task_graph_t graph, long timestep
                   scratch_ptr, scratch_bytes);
 }
 
+void task_graph_prepare_scratch(char *scratch_ptr, size_t scratch_bytes)
+{
+  TaskGraph::prepare_scratch(scratch_ptr, scratch_bytes);
+}
+
 void interval_list_destroy(interval_list_t intervals)
 {
   std::vector<std::pair<long, long> > *i = unwrap(intervals);
diff --git a/core/core_c.h b/core/core_c.h
@@ -116,6 +116,7 @@ void task_graph_execute_point_scratch_nonconst(task_graph_t graph, long timestep
                                                int64_t **input_ptr, const size_t *input_bytes,
                                                size_t n_inputs,
                                                char *scratch_ptr, size_t scratch_bytes);
+void task_graph_prepare_scratch(char *scratch_ptr, size_t scratch_bytes);
 
 typedef struct task_graph_list_t {
   void *impl;
diff --git a/dask/task_bench_core.py b/dask/task_bench_core.py
@@ -151,7 +151,10 @@ def execute_point_no_scratch(graph_array, timestep, point, *inputs):
 
 
 def init_scratch_direct(scratch_bytes):
-    return np.empty(scratch_bytes, dtype=np.ubyte)
+    scratch = np.empty(scratch_bytes, dtype=np.ubyte)
+    scratch_ptr = ffi.cast("char *", scratch.ctypes.data)
+    c.task_graph_prepare_scratch(scratch_ptr, scratch_bytes)
+    return scratch
 
 
 @dask.delayed
diff --git a/kernel_bench/main.cc b/kernel_bench/main.cc
@@ -144,6 +144,7 @@ KernelBenchApp::KernelBenchApp(int argc, char **argv)
   scratch_buff.reserve(nb_workers);
   for (i = 0; i < nb_workers; i++) {
     scratch_buff.emplace_back(graph.scratch_bytes_per_task, 0);
+    TaskGraph::prepare_scratch(scratch_buff[i].data(), graph.scratch_bytes_per_task);
   }
 
   // init timer array
diff --git a/legion/main.cc b/legion/main.cc
@@ -28,6 +28,7 @@ using namespace Legion::Mapping;
 
 enum TaskIDs {
   TID_TOP,
+  TID_INIT,
   TID_LEAF,
   TID_DUMMY,
 };
@@ -97,6 +98,20 @@ void get_base_and_size(Runtime *runtime,
   bytes = rect.volume();
 }
 
+void init(const Task *task,
+          const std::vector<PhysicalRegion> &regions,
+          Context ctx, Runtime *runtime)
+{
+  log_taskbench.info("Init at point %lld", task->index_point[0]);
+
+  char *scratch_ptr = NULL;
+  size_t scratch_bytes = 0;
+  Rect<1> scratch_rect = runtime->get_index_space_domain(
+    regions[0].get_logical_region().get_index_space());
+  get_base_and_size(runtime, regions[0], task->regions[0], scratch_rect, scratch_ptr, scratch_bytes);
+  TaskGraph::prepare_scratch(scratch_ptr, scratch_bytes);
+}
+
 void leaf(const Task *task,
           const std::vector<PhysicalRegion> &regions,
           Context ctx, Runtime *runtime)
@@ -160,6 +175,8 @@ struct LegionApp : public App {
 
   void execute_timestep(size_t i, long t);
 
+  void init(size_t i);
+
   void issue_execution_fence_and_block();
 
 private:
@@ -269,6 +286,10 @@ void LegionApp::run()
     display();
   }
 
+  for (size_t idx = 0; idx < graphs.size(); ++idx) {
+    init(idx);
+  }
+
   execute_main_loop(); // warm-up
 
   issue_execution_fence_and_block();
@@ -329,6 +350,29 @@ void LegionApp::execute_main_loop()
   }
 }
 
+void LegionApp::init(size_t idx)
+{
+  const TaskGraph &g = graphs[idx];
+
+  Rect<1> bounds(0, g.max_width-1);
+
+  if (g.scratch_bytes_per_task != 0) {
+    for (long i = 0; i < num_fields; ++i) {
+      FieldID fout(FID_FIRST + i);
+      IndexLauncher launcher(TID_INIT, bounds, TaskArgument(), ArgumentMap());
+      MappingTagID tag = exact_instance ? Legion::Mapping::DefaultMapper::EXACT_REGION : 0;
+      const LogicalRegionT<1> &sratch_region = scratch_regions[idx];
+      const LogicalPartitionT<1> &scratch = scratch_partitions[idx];
+      launcher.add_region_requirement(
+        RegionRequirement(scratch, 0 /* default projection */,
+                          READ_WRITE, EXCLUSIVE, sratch_region, tag)
+        .add_field(fout));
+
+      runtime->execute_index_space(ctx, launcher);
+    }
+  }
+}
+
 void LegionApp::execute_timestep(size_t idx, long t)
 {
   const TaskGraph &g = graphs[idx];
@@ -372,7 +416,7 @@ void LegionApp::execute_timestep(size_t idx, long t)
     const LogicalPartitionT<1> &scratch = scratch_partitions[idx];
     launcher.add_region_requirement(
       RegionRequirement(scratch, 0 /* default projection */,
-                        WRITE_DISCARD, EXCLUSIVE, sratch_region, tag)
+                        READ_WRITE, EXCLUSIVE, sratch_region, tag)
       .add_field(fout));
   }
 
@@ -419,6 +463,13 @@ int main(int argc, char **argv)
     Runtime::preregister_task_variant<top>(registrar, "top");
   }
 
+  {
+    TaskVariantRegistrar registrar(TID_INIT, "init");
+    registrar.add_constraint(ProcessorConstraint(Processor::LOC_PROC));
+    registrar.set_leaf();
+    Runtime::preregister_task_variant<init>(registrar, "init");
+  }
+
   {
     TaskVariantRegistrar registrar(TID_LEAF, "leaf");
     registrar.add_constraint(ProcessorConstraint(Processor::LOC_PROC));
diff --git a/mpi/alltoall.cc b/mpi/alltoall.cc
@@ -161,6 +161,7 @@ int main(int argc, char *argv[])
       size_t scratch_bytes = graph.scratch_bytes_per_task;
       char *scratch_ptr = (char *)malloc(scratch_bytes);
       assert(scratch_ptr);
+      TaskGraph::prepare_scratch(scratch_ptr, scratch_bytes);
 
       int *final_send_counts;
       int *final_recv_counts;
diff --git a/mpi/basic.cc b/mpi/basic.cc
@@ -121,6 +121,7 @@ int main(int argc, char *argv[])
       size_t scratch_bytes = graph.scratch_bytes_per_task;
       char *scratch_ptr = (char *)malloc(scratch_bytes);
       assert(scratch_ptr);
+      TaskGraph::prepare_scratch(scratch_ptr, scratch_bytes);
 
       for (long timestep = 0L; timestep < graph.timesteps; timestep += 1) {
         long old_dset = graph.dependence_set_at_timestep(timestep);
diff --git a/mpi/bcast.cc b/mpi/bcast.cc
@@ -296,6 +296,7 @@ int main(int argc, char *argv[])
       size_t scratch_bytes = graph.scratch_bytes_per_task;
       char *scratch_ptr = (char *)malloc(scratch_bytes);
       assert(scratch_ptr);
+      TaskGraph::prepare_scratch(scratch_ptr, scratch_bytes);
 
       char *saved_ptr = NULL;
       for (long timestep = 0L; timestep < graph.timesteps; timestep++) {
diff --git a/mpi/buffered_send.cc b/mpi/buffered_send.cc
@@ -156,6 +156,7 @@ int main(int argc, char *argv[])
       size_t scratch_bytes = graph.scratch_bytes_per_task;
       char *scratch_ptr = (char *)malloc(scratch_bytes);
       assert(scratch_ptr);
+      TaskGraph::prepare_scratch(scratch_ptr, scratch_bytes);
 
       int current_buf = 0;
       // bool looped_buffers = false;
diff --git a/mpi/bulk_synchronous.cc b/mpi/bulk_synchronous.cc
@@ -31,6 +31,17 @@ int main(int argc, char *argv[])
   App app(argc, argv);
   if (rank == 0) app.display();
 
+  std::vector<std::vector<char> > scratch;
+  for (auto graph : app.graphs) {
+    long first_point = rank * graph.max_width / n_ranks;
+    long last_point = (rank + 1) * graph.max_width / n_ranks - 1;
+    long n_points = last_point - first_point + 1;
+
+    size_t scratch_bytes = graph.scratch_bytes_per_task;
+    scratch.emplace_back(scratch_bytes * n_points);
+    TaskGraph::prepare_scratch(scratch.back().data(), scratch.back().size());
+  }
+
   double elapsed_time = 0.0;
   for (int iter = 0; iter < 2; ++iter) {
     MPI_Barrier(MPI_COMM_WORLD);
@@ -45,8 +56,7 @@ int main(int argc, char *argv[])
       long n_points = last_point - first_point + 1;
 
       size_t scratch_bytes = graph.scratch_bytes_per_task;
-      char *scratch_ptr = (char *)malloc(scratch_bytes * n_points);
-      assert(scratch_ptr);
+      char *scratch_ptr = scratch[graph.graph_index].data();
 
       std::vector<int> rank_by_point(graph.max_width);
       std::vector<int> tag_bits_by_point(graph.max_width);
@@ -204,7 +214,6 @@ int main(int argc, char *argv[])
                               scratch_ptr + scratch_bytes * point_index, scratch_bytes);
         }
       }
-      free(scratch_ptr);
     }
 
     MPI_Barrier(MPI_COMM_WORLD);
diff --git a/mpi/nonblock.cc b/mpi/nonblock.cc
@@ -31,6 +31,17 @@ int main(int argc, char *argv[])
   App app(argc, argv);
   if (rank == 0) app.display();
 
+  std::vector<std::vector<char> > scratch;
+  for (auto graph : app.graphs) {
+    long first_point = rank * graph.max_width / n_ranks;
+    long last_point = (rank + 1) * graph.max_width / n_ranks - 1;
+    long n_points = last_point - first_point + 1;
+
+    size_t scratch_bytes = graph.scratch_bytes_per_task;
+    scratch.emplace_back(scratch_bytes * n_points);
+    TaskGraph::prepare_scratch(scratch.back().data(), scratch.back().size());
+  }
+
   double elapsed_time = 0.0;
   for (int iter = 0; iter < 2; ++iter) {
     MPI_Barrier(MPI_COMM_WORLD);
@@ -45,8 +56,7 @@ int main(int argc, char *argv[])
       long n_points = last_point - first_point + 1;
 
       size_t scratch_bytes = graph.scratch_bytes_per_task;
-      char *scratch_ptr = (char *)malloc(scratch_bytes * n_points);
-      assert(scratch_ptr);
+      char *scratch_ptr = scratch[graph.graph_index].data();
 
       std::vector<int> rank_by_point(graph.max_width);
       std::vector<int> tag_bits_by_point(graph.max_width);
@@ -202,7 +212,6 @@ int main(int argc, char *argv[])
                               scratch_ptr + scratch_bytes * point_index, scratch_bytes);
         }
       }
-      free(scratch_ptr);
     }
 
     MPI_Barrier(MPI_COMM_WORLD);
diff --git a/mpi_openmp/forall.cc b/mpi_openmp/forall.cc
@@ -31,6 +31,17 @@ int main(int argc, char *argv[])
   App app(argc, argv);
   if (rank == 0) app.display();
 
+  std::vector<std::vector<char> > scratch;
+  for (auto graph : app.graphs) {
+    long first_point = rank * graph.max_width / n_ranks;
+    long last_point = (rank + 1) * graph.max_width / n_ranks - 1;
+    long n_points = last_point - first_point + 1;
+
+    size_t scratch_bytes = graph.scratch_bytes_per_task;
+    scratch.emplace_back(scratch_bytes * n_points);
+    TaskGraph::prepare_scratch(scratch.back().data(), scratch.back().size());
+  }
+
   double elapsed_time = 0.0;
   for (int iter = 0; iter < 2; ++iter) {
     MPI_Barrier(MPI_COMM_WORLD);
@@ -45,8 +56,7 @@ int main(int argc, char *argv[])
       long n_points = last_point - first_point + 1;
 
       size_t scratch_bytes = graph.scratch_bytes_per_task;
-      char *scratch_ptr = (char *)malloc(scratch_bytes * n_points);
-      assert(scratch_ptr);
+      char *scratch_ptr = scratch[graph.graph_index].data();
 
       std::vector<int> rank_by_point(graph.max_width);
       std::vector<int> tag_bits_by_point(graph.max_width);
@@ -203,7 +213,6 @@ int main(int argc, char *argv[])
                               scratch_ptr + scratch_bytes * point_index, scratch_bytes);
         }
       }
-      free(scratch_ptr);
     }
 
     MPI_Barrier(MPI_COMM_WORLD);
diff --git a/ompss/main.cc b/ompss/main.cc
@@ -364,6 +364,7 @@ OmpSsApp::OmpSsApp(int argc, char **argv)
   for (int k = 0; k < (nb_workers+1); k++) {
     if (max_scratch_bytes_per_task > 0) {
       extra_local_memory[k] = (char*)malloc(sizeof(char)*max_scratch_bytes_per_task);
+      TaskGraph::prepare_scratch(extra_local_memory[k], sizeof(char)*max_scratch_bytes_per_task);
     } else {
       extra_local_memory[k] = NULL;
     }
diff --git a/openmp/main.cc b/openmp/main.cc
@@ -381,6 +381,7 @@ OpenMPApp::OpenMPApp(int argc, char **argv)
   for (int k = 0; k < nb_workers; k++) {
     if (max_scratch_bytes_per_task > 0) {
       extra_local_memory[k] = (char*)malloc(sizeof(char)*max_scratch_bytes_per_task);
+      TaskGraph::prepare_scratch(extra_local_memory[k], sizeof(char)*max_scratch_bytes_per_task);
     } else {
       extra_local_memory[k] = NULL;
     }
diff --git a/parsec/main.cc b/parsec/main.cc
@@ -528,6 +528,7 @@ ParsecApp::ParsecApp(int argc, char **argv)
   for (i = 0; i < cores; i++) {
     if (max_scratch_bytes_per_task > 0) {
       extra_local_memory[i] = (char*)malloc(sizeof(char)*max_scratch_bytes_per_task);
+      TaskGraph::prepare_scratch(extra_local_memory[i], sizeof(char)*max_scratch_bytes_per_task);
     } else {
       extra_local_memory[i] = NULL;
     }
diff --git a/realm/main.cc b/realm/main.cc
@@ -627,6 +627,7 @@ void shard_task(const void *args, size_t arglen, const void *userdata,
   // will be effectively serialized on this processor.
   char *scratch_ptr = reinterpret_cast<char *>(malloc(max_scratch_bytes));
   assert(scratch_ptr);
+  TaskGraph::prepare_scratch(scratch_ptr, max_scratch_bytes);
 
   // Statically allocate buffer to use for task input
   size_t leaf_bufsize = 0;
diff --git a/realm_old/main.cc b/realm_old/main.cc
@@ -457,6 +457,7 @@ void shard_task(const void *args, size_t arglen, const void *userdata,
   for (size_t i = 0; i < graphs.size(); i++) {
     size_t scratch_bytes = graphs[i].scratch_bytes_per_task;
     scratch_ptrs.emplace_back(scratch_bytes);
+    TaskGraph::prepare_scratch(scratch_ptrs[i].data(), scratch_bytes);
   }
 
   Barrier share_barrier = Barrier::NO_BARRIER;
diff --git a/realm_subgraph/main.cc b/realm_subgraph/main.cc
diff --git a/regent/main.rg b/regent/main.rg
diff --git a/starpu/main.cc b/starpu/main.cc
diff --git a/tensorflow/ops/tf_core_wrapper.h b/tensorflow/ops/tf_core_wrapper.h
diff --git a/x10/TaskBench.x10 b/x10/TaskBench.x10

Original file line number	Diff line number	Diff line change
`@@ -96,6 +96,8 @@ void Subchare::initGraph(MulticastMsg* msg) {`
`96`	`96`	`output.resize(graph.output_bytes_per_task);`
`97`	`97`	`scratch.resize(graph.scratch_bytes_per_task);`
`98`	`98`
	`99`	`+ TaskGraph::prepare_scratch(scratch.data(), scratch.size());`
	`100`	`+`
`99`	`101`	`CProxySection_Subchare::contribute(sid, CkCallback(CkReductionTarget(Main, workerReady), mainProxy));`
`100`	`102`	`}`
`101`	`103`
Original file line number	Diff line number	Diff line change
`@@ -148,6 +148,11 @@ void task_graph_execute_point_scratch_nonconst(task_graph_t graph, long timestep`
`148`	`148`	`scratch_ptr, scratch_bytes);`
`149`	`149`	`}`
`150`	`150`
	`151`	`+void task_graph_prepare_scratch(char *scratch_ptr, size_t scratch_bytes)`
	`152`	`+{`
	`153`	`+ TaskGraph::prepare_scratch(scratch_ptr, scratch_bytes);`
	`154`	`+}`
	`155`	`+`
`151`	`156`	`void interval_list_destroy(interval_list_t intervals)`
`152`	`157`	`{`
`153`	`158`	`std::vector<std::pair<long, long> > *i = unwrap(intervals);`
Original file line number	Diff line number	Diff line change
`@@ -144,6 +144,7 @@ KernelBenchApp::KernelBenchApp(int argc, char **argv)`
`144`	`144`	`scratch_buff.reserve(nb_workers);`
`145`	`145`	`for (i = 0; i < nb_workers; i++) {`
`146`	`146`	`scratch_buff.emplace_back(graph.scratch_bytes_per_task, 0);`
	`147`	`+ TaskGraph::prepare_scratch(scratch_buff[i].data(), graph.scratch_bytes_per_task);`
`147`	`148`	`}`
`148`	`149`
`149`	`150`	`// init timer array`