fix for finding the eager loading data without iterating through files 2*N

Author: danmayer

changes.md

@@ -1,3 +1,7 @@
+### Coverband 6.1.1
+
+* Performance fix making paged report loading 10X faster
+
 ### Coverband 6.1.0
 
 This release has a number of smaller fixes and improvements. It includes a sizable refactoring around the UI which should simplify improvements going forward. This release is mostly targetting large projects with 6K+ ruby files, use the new `config.paged_reporting = true` option with the HashRedisStore to enable paged reporting for large projects. The HashRedisStore now also includes the last time a line in a file was executed.

coverband.gemspec

@@ -36,6 +36,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "memory_profiler"
   # breaking change in minitest and mocha...
   # note: we are also adding 'spy' as mocha doesn't want us to spy on redis calls...
+  spec.add_development_dependency "spy"
   # ^^^ probably need a large test cleanup refactor
   spec.add_development_dependency "minitest", "= 5.18.1"
   spec.add_development_dependency "minitest-fork_executor"

lib/coverband/adapters/hash_redis_store.rb

@@ -231,21 +231,22 @@ def coverage_for_types(_types, opts = {})
           end
         end
 
+        # NOTE: This is kind of hacky, we find all the matching eager loading data
+        # for current page of runtime data.
         eager_key_pre = key_prefix(Coverband::EAGER_TYPE)
         runtime_key_pre = key_prefix(Coverband::RUNTIME_TYPE)
-        matched_file_set = files_set(Coverband::EAGER_TYPE)
-          .select do |eager_key, _val|
-          runtime_file_set.any? do |runtime_key|
-            (eager_key.sub(eager_key_pre, "") == runtime_key.sub(runtime_key_pre, ""))
-          end
-        end || []
+        matched_file_set = runtime_file_set.map do |runtime_key|
+          runtime_key.sub(runtime_key_pre, eager_key_pre)
+        end
+
         hash_data[Coverband::EAGER_TYPE] = matched_file_set.each_slice(page_size).flat_map do |key_batch|
           @redis.pipelined do |pipeline|
             key_batch.each do |key|
               pipeline.hgetall(key)
             end
           end
         end
+
         hash_data[Coverband::RUNTIME_TYPE] = hash_data[Coverband::RUNTIME_TYPE].each_with_object({}) do |data_from_redis, hash|
           add_coverage_for_file(data_from_redis, hash)
         end

lib/coverband/version.rb

@@ -5,5 +5,5 @@
 # use format "4.2.1.rc.1" ~> 4.2.1.rc to prerelease versions like v4.2.1.rc.2 and v4.2.1.rc.3
 ###
 module Coverband
-  VERSION = "6.1.0"
+  VERSION = "6.1.1"
 end

test/coverband/adapters/hash_redis_store_test.rb

@@ -242,4 +242,52 @@ def test_get_coverage_cache
       @store.coverage["./dog.rb"]
     )
   end
+
+  def test_split_coverage
+    @store = Coverband::Adapters::HashRedisStore.new(
+      @redis,
+      redis_namespace: "coverband_test",
+      relative_file_converter: MockRelativeFileConverter
+    )
+
+    mock_file_hash
+    yesterday = DateTime.now.prev_day.to_time
+    mock_time(yesterday)
+
+    @store.type = :eager_loading
+    data = {
+      "app_path/dog.rb" => [0, nil, 1]
+    }
+    @store.save_report(data)
+
+    @store.type = :runtime
+    @store.save_report(
+      "app_path/dog.rb" => [0, 1, 2]
+    )
+    redis_pipelined = Spy.on(@redis, :pipelined).and_call_through
+    assert_equal(
+      {
+        runtime: {
+          "./dog.rb" => {
+            "first_updated_at" => yesterday.to_i,
+            "last_updated_at" => yesterday.to_i,
+            "file_hash" => "abcd",
+            "data" => [0, 1, 2],
+            "timedata" => [nil, Time.at(yesterday.to_i), Time.at(yesterday.to_i)]
+          }
+        },
+        eager_loading: {
+          "./dog.rb" => {
+            "first_updated_at" => yesterday.to_i,
+            "last_updated_at" => nil,
+            "file_hash" => "abcd",
+            "data" => [0, nil, 1],
+            "timedata" => [nil, nil, Time.at(yesterday.to_i)]
+          }
+        }
+      },
+      @store.split_coverage([Coverband::RUNTIME_TYPE, Coverband::EAGER_TYPE], {}, {page: 1})
+    )
+    assert_equal 2, redis_pipelined.calls.count
+  end
 end

test/test_helper.rb

@@ -35,6 +35,7 @@
 require "coverband/reporters/html_report"
 require "coverband/reporters/json_report"
 require "webmock/minitest"
+require "spy/integration"
 
 require_relative "unique_files"
 $VERBOSE = original_verbosity