Skip to content

Commit 4670d07

Browse files
danmayerdanmayer
authored
Merge pull request #516 from danmayer/paging
Add experimental reporting for large projects via paging
2 parents e815dcd + 7a183da commit 4670d07

23 files changed

+432
-168
lines changed

‎coverband.gemspec

+3-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,9 @@ Gem::Specification.new do |spec|
3434
spec.add_development_dependency "capybara"
3535
spec.add_development_dependency "m"
3636
spec.add_development_dependency "memory_profiler"
37-
# breaking change in minitest and mocha
37+
# breaking change in minitest and mocha...
38+
# note: we are also adding 'spy' as mocha doesn't want us to spy on redis calls...
39+
# ^^^ probably need a large test cleanup refactor
3840
spec.add_development_dependency "minitest", "= 5.18.1"
3941
spec.add_development_dependency "minitest-fork_executor"
4042
spec.add_development_dependency "minitest-stub-const"

‎lib/coverband/adapters/base.rb

+5-5
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ def save_coverage
3535
raise ABSTRACT_KEY
3636
end
3737

38-
def coverage(_local_type = nil)
38+
def coverage(_local_type = nil, opts = {})
3939
raise ABSTRACT_KEY
4040
end
4141

@@ -51,9 +51,9 @@ def save_report(_report)
5151
raise "abstract"
5252
end
5353

54-
def get_coverage_report
54+
def get_coverage_report(options = {})
5555
coverage_cache = {}
56-
data = Coverband.configuration.store.split_coverage(Coverband::TYPES, coverage_cache)
56+
data = Coverband.configuration.store.split_coverage(Coverband::TYPES, coverage_cache, options)
5757
data.merge(Coverband::MERGED_TYPE => Coverband.configuration.store.merged_coverage(Coverband::TYPES, coverage_cache))
5858
end
5959

@@ -67,12 +67,12 @@ def raw_store
6767

6868
protected
6969

70-
def split_coverage(types, coverage_cache)
70+
def split_coverage(types, coverage_cache, options = {})
7171
types.reduce({}) do |data, type|
7272
if type == Coverband::RUNTIME_TYPE && Coverband.configuration.simulate_oneshot_lines_coverage
7373
data.update(type => coverage_cache[type] ||= simulated_runtime_coverage)
7474
else
75-
data.update(type => coverage_cache[type] ||= coverage(type))
75+
data.update(type => coverage_cache[type] ||= coverage(type, options))
7676
end
7777
end
7878
end

‎lib/coverband/adapters/file_store.rb

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ def migrate!
5252
raise NotImplementedError, "FileStore doesn't support migrations"
5353
end
5454

55-
def coverage(_local_type = nil)
55+
def coverage(_local_type = nil, opts = {})
5656
if merge_mode
5757
data = {}
5858
Dir[path.sub(/\.\d+/, ".*")].each do |path|

‎lib/coverband/adapters/hash_redis_store.rb

+97-16
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ def clear!(local_types = Coverband::TYPES)
5656
# sleep in between to avoid holding other redis commands..
5757
# with a small random offset so runtime and eager types can be processed "at the same time"
5858
def deferred_time
59-
rand(3.0..4.0)
59+
rand(2.0..3.0)
6060
end
6161

6262
def del(local_type)
@@ -89,7 +89,7 @@ def unlock!(local_type)
8989
# used to store data to redis. It is changed only when breaking changes to our
9090
# redis format are required.
9191
###
92-
REDIS_STORAGE_FORMAT_VERSION = "coverband_hash_3_3"
92+
REDIS_STORAGE_FORMAT_VERSION = "coverband_hash_4_0"
9393

9494
JSON_PAYLOAD_EXPIRATION = 5 * 60
9595

@@ -116,8 +116,8 @@ def initialize(redis, opts = {})
116116

117117
def supported?
118118
Gem::Version.new(@redis.info["redis_version"]) >= Gem::Version.new("2.6.0")
119-
rescue Redis::CannotConnectError => error
120-
Coverband.configuration.logger.info "Redis is not available (#{error}), Coverband not configured"
119+
rescue Redis::CannotConnectError => e
120+
Coverband.configuration.logger.info "Redis is not available (#{e}), Coverband not configured"
121121
Coverband.configuration.logger.info "If this is a setup task like assets:precompile feel free to ignore"
122122
end
123123

@@ -128,6 +128,7 @@ def clear!
128128
file_keys = files_set
129129
@redis.del(*file_keys) if file_keys.any?
130130
@redis.del(files_key)
131+
@redis.del(files_key(type))
131132
@get_coverage_cache.clear!(type)
132133
end
133134
self.type = old_type
@@ -148,7 +149,7 @@ def save_report(report)
148149
updated_time = (type == Coverband::EAGER_TYPE) ? nil : report_time
149150
keys = []
150151
report.each_slice(@save_report_batch_size) do |slice|
151-
files_data = slice.map { |(file, data)|
152+
files_data = slice.map do |(file, data)|
152153
relative_file = @relative_file_converter.convert(file)
153154
file_hash = file_hash(relative_file)
154155
key = key(relative_file, file_hash: file_hash)
@@ -161,7 +162,7 @@ def save_report(report)
161162
report_time: report_time,
162163
updated_time: updated_time
163164
)
164-
}
165+
end
165166
next unless files_data.any?
166167

167168
arguments_key = [@redis_namespace, SecureRandom.uuid].compact.join(".")
@@ -171,12 +172,24 @@ def save_report(report)
171172
@redis.sadd(files_key, keys) if keys.any?
172173
end
173174

174-
def coverage(local_type = nil)
175+
# NOTE: This method should be used for full coverage or filename coverage look ups
176+
# When paging code should use coverage_for_types and pull eager and runtime together as matched pairs
177+
def coverage(local_type = nil, opts = {})
178+
page_size = opts[:page_size] || 250
175179
cached_results = @get_coverage_cache.fetch(local_type || type) do |sleep_time|
176-
files_set = files_set(local_type)
177-
178-
# use batches with a sleep in between to avoid overloading redis
179-
files_set.each_slice(250).flat_map do |key_batch|
180+
files_set = if opts[:page]
181+
raise "call coverage_for_types with paging"
182+
elsif opts[:filename]
183+
type_key_prefix = key_prefix(local_type)
184+
# NOTE: a better way to extract filename from key would be better
185+
files_set(local_type).select do |cache_key|
186+
cache_key.sub(type_key_prefix, "").match(short_name(opts[:filename]))
187+
end || {}
188+
else
189+
files_set(local_type)
190+
end
191+
# below uses batches with a sleep in between to avoid overloading redis
192+
files_set.each_slice(page_size).flat_map do |key_batch|
180193
sleep sleep_time
181194
@redis.pipelined do |pipeline|
182195
key_batch.each do |key|
@@ -191,6 +204,70 @@ def coverage(local_type = nil)
191204
end
192205
end
193206

207+
def split_coverage(types, coverage_cache, options = {})
208+
if types.is_a?(Array) && !options[:filename] && options[:page]
209+
data = coverage_for_types(types, options)
210+
coverage_cache[Coverband::RUNTIME_TYPE] = data[Coverband::RUNTIME_TYPE]
211+
coverage_cache[Coverband::EAGER_TYPE] = data[Coverband::EAGER_TYPE]
212+
data
213+
else
214+
super
215+
end
216+
end
217+
218+
def coverage_for_types(_types, opts = {})
219+
page_size = opts[:page_size] || 250
220+
hash_data = {}
221+
222+
runtime_file_set = files_set(Coverband::RUNTIME_TYPE)
223+
@cached_file_count = runtime_file_set.length
224+
runtime_file_set = runtime_file_set.each_slice(page_size).to_a[opts[:page] - 1] || []
225+
226+
hash_data[Coverband::RUNTIME_TYPE] = runtime_file_set.each_slice(page_size).flat_map do |key_batch|
227+
@redis.pipelined do |pipeline|
228+
key_batch.each do |key|
229+
pipeline.hgetall(key)
230+
end
231+
end
232+
end
233+
234+
eager_key_pre = key_prefix(Coverband::EAGER_TYPE)
235+
runtime_key_pre = key_prefix(Coverband::RUNTIME_TYPE)
236+
matched_file_set = files_set(Coverband::EAGER_TYPE)
237+
.select do |eager_key, _val|
238+
runtime_file_set.any? do |runtime_key|
239+
(eager_key.sub(eager_key_pre, "") == runtime_key.sub(runtime_key_pre, ""))
240+
end
241+
end || []
242+
hash_data[Coverband::EAGER_TYPE] = matched_file_set.each_slice(page_size).flat_map do |key_batch|
243+
@redis.pipelined do |pipeline|
244+
key_batch.each do |key|
245+
pipeline.hgetall(key)
246+
end
247+
end
248+
end
249+
hash_data[Coverband::RUNTIME_TYPE] = hash_data[Coverband::RUNTIME_TYPE].each_with_object({}) do |data_from_redis, hash|
250+
add_coverage_for_file(data_from_redis, hash)
251+
end
252+
hash_data[Coverband::EAGER_TYPE] = hash_data[Coverband::EAGER_TYPE].each_with_object({}) do |data_from_redis, hash|
253+
add_coverage_for_file(data_from_redis, hash)
254+
end
255+
hash_data
256+
end
257+
258+
def short_name(filename)
259+
filename.sub(/^#{Coverband.configuration.root}/, ".")
260+
.gsub(%r{^\./}, "")
261+
end
262+
263+
def file_count(local_type = nil)
264+
files_set(local_type).count { |filename| !Coverband.configuration.ignore.any? { |i| filename.match(i) } }
265+
end
266+
267+
def cached_file_count
268+
@cached_file_count ||= file_count(Coverband::RUNTIME_TYPE)
269+
end
270+
194271
def raw_store
195272
@redis
196273
end
@@ -212,9 +289,13 @@ def add_coverage_for_file(data_from_redis, hash)
212289
return unless file_hash(file) == data_from_redis[FILE_HASH]
213290

214291
data = coverage_data_from_redis(data_from_redis)
215-
hash[file] = data_from_redis.select { |meta_data_key, _value| META_DATA_KEYS.include?(meta_data_key) }.merge!("data" => data)
216-
hash[file][LAST_UPDATED_KEY] = (hash[file][LAST_UPDATED_KEY].nil? || hash[file][LAST_UPDATED_KEY] == "") ? nil : hash[file][LAST_UPDATED_KEY].to_i
217-
hash[file].merge!(LAST_UPDATED_KEY => hash[file][LAST_UPDATED_KEY], FIRST_UPDATED_KEY => hash[file][FIRST_UPDATED_KEY].to_i)
292+
hash[file] = data_from_redis.select do |meta_data_key, _value|
293+
META_DATA_KEYS.include?(meta_data_key)
294+
end.merge!("data" => data)
295+
hash[file][LAST_UPDATED_KEY] =
296+
(hash[file][LAST_UPDATED_KEY].nil? || hash[file][LAST_UPDATED_KEY] == "") ? nil : hash[file][LAST_UPDATED_KEY].to_i
297+
hash[file].merge!(LAST_UPDATED_KEY => hash[file][LAST_UPDATED_KEY],
298+
FIRST_UPDATED_KEY => hash[file][FIRST_UPDATED_KEY].to_i)
218299
end
219300

220301
def coverage_data_from_redis(data_from_redis)
@@ -226,9 +307,9 @@ def coverage_data_from_redis(data_from_redis)
226307
end
227308

228309
def script_input(key:, file:, file_hash:, data:, report_time:, updated_time:)
229-
coverage_data = data.each_with_index.each_with_object({}) { |(coverage, index), hash|
310+
coverage_data = data.each_with_index.each_with_object({}) do |(coverage, index), hash|
230311
hash[index] = coverage if coverage
231-
}
312+
end
232313
meta = {
233314
first_updated_at: report_time,
234315
file: file,

‎lib/coverband/adapters/null_store.rb

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ def migrate!
2626
raise NotImplementedError, "NullStore doesn't support migrations"
2727
end
2828

29-
def coverage(_local_type = nil)
29+
def coverage(_local_type = nil, opts = {})
3030
{}
3131
end
3232

‎lib/coverband/adapters/stdout_store.rb

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ def migrate!
2525
raise NotImplementedError, "StdoutStore doesn't support migrations"
2626
end
2727

28-
def coverage(_local_type = nil)
28+
def coverage(_local_type = nil, opts = {})
2929
{}
3030
end
3131

‎lib/coverband/configuration.rb

+27-8
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
# frozen_string_literal: true
22

33
module Coverband
4+
###
5+
# Configuration parsing and options for the coverband gem.
6+
###
47
class Configuration
58
attr_accessor :root_paths, :root,
69
:verbose,
@@ -17,7 +20,7 @@ class Configuration
1720
:s3_secret_access_key, :password, :api_key, :service_url, :coverband_timeout, :service_dev_mode,
1821
:service_test_mode, :process_type, :track_views, :redis_url,
1922
:background_reporting_sleep_seconds, :reporting_wiggle,
20-
:send_deferred_eager_loading_data
23+
:send_deferred_eager_loading_data, :paged_reporting
2124

2225
attr_reader :track_gems, :ignore, :use_oneshot_lines_coverage
2326

@@ -132,8 +135,8 @@ def railtie!
132135
trackers << Coverband.configuration.view_tracker
133136
end
134137
trackers.each { |tracker| tracker.railtie! }
135-
rescue Redis::CannotConnectError => error
136-
Coverband.configuration.logger.info "Redis is not available (#{error}), Coverband not configured"
138+
rescue Redis::CannotConnectError => e
139+
Coverband.configuration.logger.info "Redis is not available (#{e}), Coverband not configured"
137140
Coverband.configuration.logger.info "If this is a setup task like assets:precompile feel free to ignore"
138141
end
139142

@@ -170,7 +173,10 @@ def reporting_wiggle
170173

171174
def store
172175
@store ||= if service?
173-
raise "invalid configuration: unclear default store coverband expects either api_key or redis_url" if ENV["COVERBAND_REDIS_URL"]
176+
if ENV["COVERBAND_REDIS_URL"]
177+
raise "invalid configuration: unclear default store coverband expects either api_key or redis_url"
178+
end
179+
174180
require "coverband/adapters/web_service_store"
175181
Coverband::Adapters::WebServiceStore.new(service_url)
176182
else
@@ -180,8 +186,14 @@ def store
180186

181187
def store=(store)
182188
raise "Pass in an instance of Coverband::Adapters" unless store.is_a?(Coverband::Adapters::Base)
183-
raise "invalid configuration: only coverband service expects an API Key" if api_key && store.class.to_s != "Coverband::Adapters::WebServiceStore"
184-
raise "invalid configuration: coverband service shouldn't have redis url set" if ENV["COVERBAND_REDIS_URL"] && store.instance_of?(::Coverband::Adapters::WebServiceStore)
189+
if api_key && store.class.to_s != "Coverband::Adapters::WebServiceStore"
190+
raise "invalid configuration: only coverband service expects an API Key"
191+
end
192+
if ENV["COVERBAND_REDIS_URL"] &&
193+
defined?(::Coverband::Adapters::WebServiceStore) &&
194+
store.instance_of?(::Coverband::Adapters::WebServiceStore)
195+
raise "invalid configuration: coverband service shouldn't have redis url set"
196+
end
185197

186198
@store = store
187199
end
@@ -241,7 +253,10 @@ def to_h
241253
end
242254

243255
def use_oneshot_lines_coverage=(value)
244-
raise(StandardError, "One shot line coverage is only available in ruby >= 2.6") unless one_shot_coverage_implemented_in_ruby_version? || !value
256+
unless one_shot_coverage_implemented_in_ruby_version? || !value
257+
raise(StandardError,
258+
"One shot line coverage is only available in ruby >= 2.6")
259+
end
245260

246261
@use_oneshot_lines_coverage = value
247262
end
@@ -294,6 +309,10 @@ def send_deferred_eager_loading_data?
294309
@send_deferred_eager_loading_data
295310
end
296311

312+
def paged_reporting
313+
!!@paged_reporting
314+
end
315+
297316
def service_disabled_dev_test_env?
298317
return false unless service?
299318

@@ -318,7 +337,7 @@ def s3_secret_access_key
318337
end
319338

320339
def track_gems=(_value)
321-
puts "gem tracking is deprecated, setting this will be ignored"
340+
puts "gem tracking is deprecated, setting this will be ignored & eventually removed"
322341
end
323342

324343
private

‎lib/coverband/reporters/base.rb

+5-4
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ class Base
1010
class << self
1111
DATA_KEY = "data"
1212

13-
def report(store, _options = {})
13+
def report(store, options = {})
1414
all_roots = Coverband.configuration.all_root_paths
15-
get_current_scov_data_imp(store, all_roots)
15+
get_current_scov_data_imp(store, all_roots, options)
1616

1717
# These are extremelhy verbose but useful during coverband development, not generally for users
1818
# Only available by uncommenting this mode is never released
@@ -85,12 +85,13 @@ def merge_arrays(first, second)
8585
# why do we need to merge covered files data?
8686
# basically because paths on machines or deployed hosts could be different, so
8787
# two different keys could point to the same filename or `line_key`
88+
# this happens when deployment has a dynmaic path or the path change during deployment (hot code reload)
8889
# TODO: think we are filtering based on ignore while sending to the store
8990
# and as we also pull it out here
9091
###
91-
def get_current_scov_data_imp(store, roots)
92+
def get_current_scov_data_imp(store, roots, options = {})
9293
scov_style_report = {}
93-
store.get_coverage_report.each_pair do |name, data|
94+
store.get_coverage_report(options).each_pair do |name, data|
9495
data.each_pair do |key, line_data|
9596
next if Coverband.configuration.ignore.any? { |i| key.match(i) }
9697
next unless line_data

0 commit comments

Comments
 (0)