Skip to content

Commit cdd817d

Browse files
author
Dane Springmeyer
committed
benchmarks: refactor to allow early exit if tests do not validate
1 parent 85d954a commit cdd817d

18 files changed

+141
-86
lines changed

‎benchmark/bench_framework.hpp‎

Lines changed: 31 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ class test_case
3737
return iterations_;
3838
}
3939
virtual bool validate() const = 0;
40-
virtual void operator()() const = 0;
40+
virtual bool operator()() const = 0;
4141
virtual ~test_case() {}
4242
};
4343

@@ -85,35 +85,40 @@ int run(T const& test_runner, std::string const& name)
8585
std::clog << "test did not validate: " << name << "\n";
8686
return -1;
8787
}
88-
std::chrono::high_resolution_clock::time_point start;
89-
std::chrono::high_resolution_clock::duration elapsed;
90-
std::stringstream s;
91-
s << name << ":"
92-
<< std::setw(45 - (int)s.tellp()) << std::right
93-
<< " t:" << test_runner.threads()
94-
<< " i:" << test_runner.iterations();
95-
if (test_runner.threads() > 0)
88+
// run test once before timing
89+
// if it returns false then we'll abort timing
90+
if (test_runner())
9691
{
97-
using thread_group = std::vector<std::unique_ptr<std::thread> >;
98-
using value_type = thread_group::value_type;
99-
thread_group tg;
100-
for (std::size_t i=0;i<test_runner.threads();++i)
92+
std::chrono::high_resolution_clock::time_point start;
93+
std::chrono::high_resolution_clock::duration elapsed;
94+
std::stringstream s;
95+
s << name << ":"
96+
<< std::setw(45 - (int)s.tellp()) << std::right
97+
<< " t:" << test_runner.threads()
98+
<< " i:" << test_runner.iterations();
99+
if (test_runner.threads() > 0)
101100
{
102-
tg.emplace_back(new std::thread(test_runner));
101+
using thread_group = std::vector<std::unique_ptr<std::thread> >;
102+
using value_type = thread_group::value_type;
103+
thread_group tg;
104+
for (std::size_t i=0;i<test_runner.threads();++i)
105+
{
106+
tg.emplace_back(new std::thread(test_runner));
107+
}
108+
start = std::chrono::high_resolution_clock::now();
109+
std::for_each(tg.begin(), tg.end(), [](value_type & t) {if (t->joinable()) t->join();});
110+
elapsed = std::chrono::high_resolution_clock::now() - start;
103111
}
104-
start = std::chrono::high_resolution_clock::now();
105-
std::for_each(tg.begin(), tg.end(), [](value_type & t) {if (t->joinable()) t->join();});
106-
elapsed = std::chrono::high_resolution_clock::now() - start;
107-
}
108-
else
109-
{
110-
start = std::chrono::high_resolution_clock::now();
111-
test_runner();
112-
elapsed = std::chrono::high_resolution_clock::now() - start;
112+
else
113+
{
114+
start = std::chrono::high_resolution_clock::now();
115+
test_runner();
116+
elapsed = std::chrono::high_resolution_clock::now() - start;
117+
}
118+
s << std::setw(65 - (int)s.tellp()) << std::right
119+
<< std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count() << " milliseconds\n";
120+
std::clog << s.str();
113121
}
114-
s << std::setw(65 - (int)s.tellp()) << std::right
115-
<< std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count() << " milliseconds\n";
116-
std::clog << s.str();
117122
return 0;
118123
}
119124
catch (std::exception const& ex)

‎benchmark/test_array_allocation.cpp‎

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ class test1 : public benchmark::test_case
4040
{
4141
return true;
4242
}
43-
void operator()() const
43+
bool operator()() const
4444
{
4545
for (std::size_t i=0;i<iterations_;++i) {
4646
// NOTE: sizeof(uint8_t) == 1
@@ -49,6 +49,7 @@ class test1 : public benchmark::test_case
4949
ensure_zero(data,size_);
5050
free(data);
5151
}
52+
return true;
5253
}
5354
};
5455

@@ -65,7 +66,7 @@ class test1b : public benchmark::test_case
6566
{
6667
return true;
6768
}
68-
void operator()() const
69+
bool operator()() const
6970
{
7071
for (std::size_t i=0;i<iterations_;++i) {
7172
// NOTE: sizeof(uint8_t) == 1
@@ -74,6 +75,7 @@ class test1b : public benchmark::test_case
7475
ensure_zero(data,size_);
7576
free(data);
7677
}
78+
return true;
7779
}
7880
};
7981

@@ -90,14 +92,15 @@ class test1c : public benchmark::test_case
9092
{
9193
return true;
9294
}
93-
void operator()() const
95+
bool operator()() const
9496
{
9597
for (std::size_t i=0;i<iterations_;++i) {
9698
uint8_t *data = static_cast<uint8_t *>(::operator new(sizeof(uint8_t) * size_));
9799
std::fill(data,data + size_,0);
98100
ensure_zero(data,size_);
99101
::operator delete(data);
100102
}
103+
return true;
101104
}
102105
};
103106

@@ -115,14 +118,15 @@ class test2 : public benchmark::test_case
115118
{
116119
return true;
117120
}
118-
void operator()() const
121+
bool operator()() const
119122
{
120123
for (std::size_t i=0;i<iterations_;++i) {
121124
uint8_t * data = static_cast<uint8_t*>(::operator new(sizeof(uint8_t)*size_));
122125
memcpy(data, &array_[0], size_);
123126
ensure_zero(data,size_);
124127
::operator delete(data),data=0;
125128
}
129+
return true;
126130
}
127131
};
128132

@@ -139,12 +143,13 @@ class test3 : public benchmark::test_case
139143
{
140144
return true;
141145
}
142-
void operator()() const
146+
bool operator()() const
143147
{
144148
for (std::size_t i=0;i<iterations_;++i) {
145149
std::vector<uint8_t> data(size_);
146150
ensure_zero(&data[0],data.size());
147151
}
152+
return true;
148153
}
149154
};
150155

@@ -162,13 +167,14 @@ class test3b : public benchmark::test_case
162167
{
163168
return true;
164169
}
165-
void operator()() const
170+
bool operator()() const
166171
{
167172
for (std::size_t i=0;i<iterations_;++i) {
168173
std::vector<uint8_t> data(0);
169174
data.resize(size_,0);
170175
ensure_zero(&data[0],data.size());
171176
}
177+
return true;
172178
}
173179
};
174180

@@ -186,13 +192,14 @@ class test3c : public benchmark::test_case
186192
{
187193
return true;
188194
}
189-
void operator()() const
195+
bool operator()() const
190196
{
191197
for (std::size_t i=0;i<iterations_;++i) {
192198
std::vector<uint8_t> data(0);
193199
data.assign(size_,0);
194200
ensure_zero(&data[0],data.size());
195201
}
202+
return true;
196203
}
197204
};
198205

@@ -209,13 +216,14 @@ class test4 : public benchmark::test_case
209216
{
210217
return true;
211218
}
212-
void operator()() const
219+
bool operator()() const
213220
{
214221
for (std::size_t i=0;i<iterations_;++i) {
215222
uint8_t *data = (uint8_t *)calloc(size_,sizeof(uint8_t));
216223
ensure_zero(data,size_);
217224
free(data);
218225
}
226+
return true;
219227
}
220228
};
221229

@@ -232,12 +240,13 @@ class test5 : public benchmark::test_case
232240
{
233241
return true;
234242
}
235-
void operator()() const
243+
bool operator()() const
236244
{
237245
for (std::size_t i=0;i<iterations_;++i) {
238246
std::string data(array_.begin(),array_.end());
239247
ensure_zero((uint8_t *)&data[0],size_);
240248
}
249+
return true;
241250
}
242251
};
243252

@@ -254,12 +263,13 @@ class test5b : public benchmark::test_case
254263
{
255264
return true;
256265
}
257-
void operator()() const
266+
bool operator()() const
258267
{
259268
for (std::size_t i=0;i<iterations_;++i) {
260269
std::string data(&array_[0],array_.size());
261270
ensure_zero((uint8_t *)&data[0],size_);
262271
}
272+
return true;
263273
}
264274
};
265275

@@ -281,12 +291,13 @@ class test6 : public benchmark::test_case
281291
{
282292
return true;
283293
}
284-
void operator()() const
294+
bool operator()() const
285295
{
286296
for (std::size_t i=0;i<iterations_;++i) {
287297
std::valarray<uint8_t> data(static_cast<uint8_t>(0),static_cast<size_t>(size_));
288298
ensure_zero(&data[0],size_);
289299
}
300+
return true;
290301
}
291302
};
292303

@@ -307,12 +318,13 @@ class test7 : public benchmark::test_case
307318
{
308319
return true;
309320
}
310-
void operator()() const
321+
bool operator()() const
311322
{
312323
for (std::size_t i=0;i<iterations_;++i) {
313324
boost::container::static_vector<uint8_t,256*256> data(size_,0);
314325
ensure_zero(&data[0],size_);
315326
}
327+
return true;
316328
}
317329
};
318330
#endif

‎benchmark/test_expression_parse.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,12 @@ class test : public benchmark::test_case
2222
}
2323
return ret;
2424
}
25-
void operator()() const
25+
bool operator()() const
2626
{
2727
for (std::size_t i=0;i<iterations_;++i) {
2828
mapnik::expression_ptr expr = mapnik::parse_expression(expr_);
2929
}
30+
return true;
3031
}
3132
};
3233

‎benchmark/test_face_ptr_creation.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ class test : public benchmark::test_case
2626
}
2727
return count == expected_count;
2828
}
29-
void operator()() const
29+
bool operator()() const
3030
{
3131
std::size_t expected_count = mapnik::freetype_engine::face_names().size();
3232
for (unsigned i=0;i<iterations_;++i)
@@ -49,6 +49,7 @@ class test : public benchmark::test_case
4949
std::clog << "warning: face creation not working as expected\n";
5050
}
5151
}
52+
return true;
5253
}
5354
};
5455

‎benchmark/test_font_registration.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,15 @@ class test : public benchmark::test_case
1212
{
1313
return mapnik::freetype_engine::register_fonts("./fonts", true);
1414
}
15-
void operator()() const
15+
bool operator()() const
1616
{
1717
unsigned long count = 0;
1818
for (unsigned i=0;i<iterations_;++i)
1919
{
2020
mapnik::freetype_engine::register_fonts("./fonts", true);
2121
count++;
2222
}
23+
return true;
2324
}
2425
};
2526

‎benchmark/test_png_encoding1.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,15 @@ class test : public benchmark::test_case
1313
{
1414
return true;
1515
}
16-
void operator()() const
16+
bool operator()() const
1717
{
1818
std::string out;
1919
for (std::size_t i=0;i<iterations_;++i) {
2020
out.clear();
2121
out = mapnik::save_to_string(im_,"png8:m=h:z=1");
2222
}
2323
}
24+
return true;
2425
};
2526

2627
BENCHMARK(test,"encoding blank png")

‎benchmark/test_png_encoding2.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,15 @@ class test : public benchmark::test_case
2323
mapnik::save_to_file(im_->data(),actual, "png8:m=h:z=1");
2424
return benchmark::compare_images(actual,expected);
2525
}
26-
void operator()() const
26+
bool operator()() const
2727
{
2828
std::string out;
2929
for (std::size_t i=0;i<iterations_;++i) {
3030
out.clear();
3131
out = mapnik::save_to_string(im_->data(),"png8:m=h:z=1");
3232
}
3333
}
34+
return true;
3435
};
3536

3637
BENCHMARK(test,"encoding multicolor png")

‎benchmark/test_polygon_clipping.cpp‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ class test1 : public benchmark::test_case
108108
render(geom2,geom.envelope(),actual);
109109
return benchmark::compare_images(actual,expect);
110110
}
111-
void operator()() const
111+
bool operator()() const
112112
{
113113
boost::ptr_vector<mapnik::geometry_type> paths;
114114
if (!mapnik::from_wkt(wkt_in_, paths))
@@ -130,6 +130,7 @@ class test1 : public benchmark::test_case
130130
while ((cmd = clipped.vertex(&x, &y)) != mapnik::SEG_END) {}
131131
}
132132
}
133+
return true;
133134
}
134135
};
135136

@@ -189,7 +190,7 @@ class test2 : public benchmark::test_case
189190
render(geom2,geom.envelope(),actual);
190191
return benchmark::compare_images(actual,expect);
191192
}
192-
void operator()() const
193+
bool operator()() const
193194
{
194195
boost::ptr_vector<mapnik::geometry_type> paths;
195196
if (!mapnik::from_wkt(wkt_in_, paths))
@@ -217,6 +218,7 @@ class test2 : public benchmark::test_case
217218
while ((cmd = clipped.vertex(&x, &y)) != mapnik::SEG_END) {}
218219
}
219220
}
221+
return true;
220222
}
221223
};
222224

@@ -265,7 +267,7 @@ class test3 : public benchmark::test_case
265267
render(geom2,geom.envelope(),actual);
266268
return benchmark::compare_images(actual,expect);
267269
}
268-
void operator()() const
270+
bool operator()() const
269271
{
270272
boost::ptr_vector<mapnik::geometry_type> paths;
271273
if (!mapnik::from_wkt(wkt_in_, paths))
@@ -282,6 +284,7 @@ class test3 : public benchmark::test_case
282284
while ((cmd = clipped.vertex(&x, &y)) != mapnik::SEG_END) {}
283285
}
284286
}
287+
return true;
285288
}
286289
};
287290

0 commit comments

Comments
 (0)