Skip to content

Commit fef5be9

Browse files
cessenpascalkuthe
cessen
and
pascalkuthe
committed
Fix Hash impl on Rope and RopeSlice.
The previous impl misunderstood the Hasher API, and assumed that as long as the same data was passed in the same order, the hash output would be the same. But in fact, the data must be split among the `write()` calls exactly the same as well. This new impl fixes that by always passing the data in exact fixed-sized blocks, regardless of the underlying chunks boundaries of the rope. Co-authored-by: Pascal Kuthe <[email protected]>
1 parent 69cfb1e commit fef5be9

File tree

6 files changed

+316
-9
lines changed

6 files changed

+316
-9
lines changed

‎Cargo.toml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ rand = "0.8"
2626
proptest = "1.0"
2727
criterion = { version = "0.3", features = ["html_reports"] }
2828
unicode-segmentation = "1.3"
29+
fnv = "1"
30+
fxhash = "0.2"
2931

3032
#-----------------------------------------
3133

@@ -37,6 +39,10 @@ harness = false
3739
name = "insert"
3840
harness = false
3941

42+
[[bench]]
43+
name = "hash"
44+
harness = false
45+
4046
[[bench]]
4147
name = "remove"
4248
harness = false

‎benches/hash.rs‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
extern crate criterion;
2+
extern crate fnv;
3+
extern crate fxhash;
4+
extern crate ropey;
5+
6+
use std::collections::hash_map::DefaultHasher;
7+
use std::hash::{Hash, Hasher};
8+
9+
use criterion::{black_box, criterion_group, criterion_main, Criterion};
10+
use fnv::FnvHasher;
11+
use fxhash::FxHasher;
12+
use ropey::Rope;
13+
14+
const TEXT: &str = include_str!("large.txt");
15+
const TEXT_SMALL: &str = include_str!("small.txt");
16+
const TEXT_TINY: &str = "hello";
17+
18+
//----
19+
20+
fn hash_large(c: &mut Criterion) {
21+
let mut group = c.benchmark_group("hash_large");
22+
23+
group.bench_function("default", |bench| {
24+
let r = Rope::from_str(TEXT);
25+
bench.iter(|| {
26+
let mut hasher = DefaultHasher::default();
27+
r.hash(black_box(&mut hasher));
28+
black_box(hasher.finish());
29+
})
30+
});
31+
32+
group.bench_function("fnv", |bench| {
33+
let r = Rope::from_str(TEXT);
34+
bench.iter(|| {
35+
let mut hasher = FnvHasher::default();
36+
r.hash(black_box(&mut hasher));
37+
black_box(hasher.finish());
38+
})
39+
});
40+
41+
group.bench_function("fxhash", |bench| {
42+
let r = Rope::from_str(TEXT);
43+
bench.iter(|| {
44+
let mut hasher = FxHasher::default();
45+
r.hash(black_box(&mut hasher));
46+
black_box(hasher.finish());
47+
})
48+
});
49+
}
50+
51+
fn hash_small(c: &mut Criterion) {
52+
let mut group = c.benchmark_group("hash_small");
53+
54+
group.bench_function("default", |bench| {
55+
let r = Rope::from_str(TEXT_SMALL);
56+
bench.iter(|| {
57+
let mut hasher = DefaultHasher::default();
58+
r.hash(black_box(&mut hasher));
59+
black_box(hasher.finish());
60+
})
61+
});
62+
63+
group.bench_function("fnv", |bench| {
64+
let r = Rope::from_str(TEXT_SMALL);
65+
bench.iter(|| {
66+
let mut hasher = FnvHasher::default();
67+
r.hash(black_box(&mut hasher));
68+
black_box(hasher.finish());
69+
})
70+
});
71+
72+
group.bench_function("fxhash", |bench| {
73+
let r = Rope::from_str(TEXT_SMALL);
74+
bench.iter(|| {
75+
let mut hasher = FxHasher::default();
76+
r.hash(black_box(&mut hasher));
77+
black_box(hasher.finish());
78+
})
79+
});
80+
}
81+
82+
fn hash_tiny(c: &mut Criterion) {
83+
let mut group = c.benchmark_group("hash_tiny");
84+
85+
group.bench_function("default", |bench| {
86+
let r = Rope::from_str(TEXT_TINY);
87+
bench.iter(|| {
88+
let mut hasher = DefaultHasher::default();
89+
r.hash(black_box(&mut hasher));
90+
black_box(hasher.finish());
91+
})
92+
});
93+
94+
group.bench_function("fnv", |bench| {
95+
let r = Rope::from_str(TEXT_TINY);
96+
bench.iter(|| {
97+
let mut hasher = FnvHasher::default();
98+
r.hash(black_box(&mut hasher));
99+
black_box(hasher.finish());
100+
})
101+
});
102+
103+
group.bench_function("fxhash", |bench| {
104+
let r = Rope::from_str(TEXT_TINY);
105+
bench.iter(|| {
106+
let mut hasher = FxHasher::default();
107+
r.hash(black_box(&mut hasher));
108+
black_box(hasher.finish());
109+
})
110+
});
111+
}
112+
113+
//----
114+
115+
criterion_group!(benches, hash_large, hash_small, hash_tiny,);
116+
criterion_main!(benches);

‎src/rope.rs‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2052,14 +2052,7 @@ impl std::cmp::PartialOrd<Rope> for Rope {
20522052

20532053
impl std::hash::Hash for Rope {
20542054
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
2055-
for chunk in self.chunks() {
2056-
state.write(chunk.as_bytes());
2057-
}
2058-
2059-
// Same strategy as `&str` in stdlib, so that e.g. two adjacent
2060-
// fields in a `#[derive(Hash)]` struct with "Hi " and "there"
2061-
// vs "Hi t" and "here" give the struct a different hash.
2062-
state.write_u8(0xff)
2055+
self.slice(..).hash(state)
20632056
}
20642057
}
20652058

‎src/slice.rs‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1974,8 +1974,51 @@ impl<'a, 'b> std::cmp::PartialOrd<RopeSlice<'b>> for RopeSlice<'a> {
19741974

19751975
impl<'a> std::hash::Hash for RopeSlice<'a> {
19761976
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
1977+
// `std::hash::Hasher` only guarantees the same hash output for
1978+
// exactly the same calls to `Hasher::write()`. Just submitting
1979+
// the same data in the same order isn't enough--it also has to
1980+
// be split the same between calls. So we go to some effort here
1981+
// to ensure that we always submit the text data in the same
1982+
// fixed-size blocks, even if those blocks don't align with chunk
1983+
// boundaries at all.
1984+
//
1985+
// The naive approach is to always copy to a fixed-size buffer
1986+
// and submit the buffer whenever it fills up. We conceptually
1987+
// follow that approach here, but we do a little better by
1988+
// skipping the buffer and directly passing the data without
1989+
// copying when possible.
1990+
const BLOCK_SIZE: usize = 256;
1991+
1992+
let mut buffer = [0u8; BLOCK_SIZE];
1993+
let mut buffer_len = 0;
1994+
19771995
for chunk in self.chunks() {
1978-
state.write(chunk.as_bytes());
1996+
let mut data = chunk.as_bytes();
1997+
1998+
while !data.is_empty() {
1999+
if buffer_len == 0 && data.len() >= BLOCK_SIZE {
2000+
// Process data directly, skipping the buffer.
2001+
let (head, tail) = data.split_at(BLOCK_SIZE);
2002+
state.write(head);
2003+
data = tail;
2004+
} else if buffer_len == BLOCK_SIZE {
2005+
// Process the filled buffer.
2006+
state.write(&buffer[..]);
2007+
buffer_len = 0;
2008+
} else {
2009+
// Append to the buffer.
2010+
let n = (BLOCK_SIZE - buffer_len).min(data.len());
2011+
let (head, tail) = data.split_at(n);
2012+
(&mut buffer[buffer_len..(buffer_len + n)]).copy_from_slice(head);
2013+
buffer_len += n;
2014+
data = tail;
2015+
}
2016+
}
2017+
}
2018+
2019+
// Write any remaining unprocessed data in the buffer.
2020+
if buffer_len > 0 {
2021+
state.write(&buffer[..buffer_len]);
19792022
}
19802023

19812024
// Same strategy as `&str` in stdlib, so that e.g. two adjacent

‎tests/hash.rs‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
extern crate ropey;
2+
3+
use std::hash::{Hash, Hasher};
4+
5+
use ropey::{Rope, RopeBuilder};
6+
7+
const SMALL_TEXT: &str = include_str!("small_ascii.txt");
8+
9+
/// This is an example `Hasher` to demonstrate a property guaranteed by
10+
/// the documentation that is not exploited by the default `Hasher` (SipHash)
11+
/// Relevant excerpt from the `Hasher` documentation:
12+
/// > Nor can you assume that adjacent
13+
/// > `write` calls are merged, so it's possible, for example, that
14+
/// > ```
15+
/// > # fn foo(hasher: &mut impl std::hash::Hasher) {
16+
/// > hasher.write(&[1, 2]);
17+
/// > hasher.write(&[3, 4, 5, 6]);
18+
/// > # }
19+
/// > ```
20+
/// > and
21+
/// > ```
22+
/// > # fn foo(hasher: &mut impl std::hash::Hasher) {
23+
/// > hasher.write(&[1, 2, 3, 4]);
24+
/// > hasher.write(&[5, 6]);
25+
/// > # }
26+
/// > ```
27+
/// > end up producing different hashes.
28+
///
29+
/// This dummy hasher simply collects all bytes and inserts a separator byte (0xFF) at the end of `write`.
30+
/// While this hasher might seem a little silly, it is perfectly inline with the std documentation.
31+
/// Many other commonly used high performance `Hasher`s (fxhash, ahash, fnvhash) exploit the same property
32+
/// to improve the performance of `write`, so violating this property will cause issues in practice.
33+
#[derive(Default)]
34+
struct TestHasher(std::collections::hash_map::DefaultHasher);
35+
impl Hasher for TestHasher {
36+
fn finish(&self) -> u64 {
37+
self.0.finish()
38+
}
39+
40+
fn write(&mut self, bytes: &[u8]) {
41+
self.0.write(bytes);
42+
self.0.write_u8(0xFF);
43+
}
44+
}
45+
46+
#[test]
47+
#[cfg_attr(miri, ignore)]
48+
fn hash_1() {
49+
// Build two ropes with the same contents but different chunk boundaries.
50+
let r1 = {
51+
let mut b = RopeBuilder::new();
52+
b._append_chunk("Hello w");
53+
b._append_chunk("orld");
54+
b._finish_no_fix()
55+
};
56+
let r2 = {
57+
let mut b = RopeBuilder::new();
58+
b._append_chunk("Hell");
59+
b._append_chunk("o world");
60+
b._finish_no_fix()
61+
};
62+
63+
let mut hasher1 = TestHasher::default();
64+
let mut hasher2 = TestHasher::default();
65+
r1.hash(&mut hasher1);
66+
r2.hash(&mut hasher2);
67+
68+
assert_eq!(hasher1.finish(), hasher2.finish());
69+
}
70+
71+
#[test]
72+
#[cfg_attr(miri, ignore)]
73+
fn hash_2() {
74+
// Build two ropes with the same contents but different chunk boundaries.
75+
let r1 = {
76+
let mut b = RopeBuilder::new();
77+
for chunk in SMALL_TEXT.as_bytes().chunks(5) {
78+
b._append_chunk(std::str::from_utf8(chunk).unwrap());
79+
}
80+
b._finish_no_fix()
81+
};
82+
let r2 = {
83+
let mut b = RopeBuilder::new();
84+
for chunk in SMALL_TEXT.as_bytes().chunks(7) {
85+
b._append_chunk(std::str::from_utf8(chunk).unwrap());
86+
}
87+
b._finish_no_fix()
88+
};
89+
90+
for (l1, l2) in r1.lines().zip(r2.lines()) {
91+
let mut hasher1 = TestHasher::default();
92+
let mut hasher2 = TestHasher::default();
93+
l1.hash(&mut hasher1);
94+
l2.hash(&mut hasher2);
95+
96+
assert_eq!(hasher1.finish(), hasher2.finish());
97+
}
98+
}
99+
100+
#[test]
101+
#[cfg_attr(miri, ignore)]
102+
fn hash_3() {
103+
// Build two ropes with the same contents but different chunk boundaries.
104+
let r1 = {
105+
let mut b = RopeBuilder::new();
106+
for chunk in SMALL_TEXT.as_bytes().chunks(521) {
107+
b._append_chunk(std::str::from_utf8(chunk).unwrap());
108+
}
109+
b._finish_no_fix()
110+
};
111+
let r2 = {
112+
let mut b = RopeBuilder::new();
113+
for chunk in SMALL_TEXT.as_bytes().chunks(547) {
114+
b._append_chunk(std::str::from_utf8(chunk).unwrap());
115+
}
116+
b._finish_no_fix()
117+
};
118+
119+
let mut hasher1 = TestHasher::default();
120+
let mut hasher2 = TestHasher::default();
121+
r1.hash(&mut hasher1);
122+
r2.hash(&mut hasher2);
123+
124+
assert_eq!(hasher1.finish(), hasher2.finish());
125+
}

‎tests/small_ascii.txt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas sit amet tellus
2+
nec turpis feugiat semper. Nam at nulla laoreet, finibus eros sit amet, fringilla
3+
mauris. Fusce vestibulum nec ligula efficitur laoreet. Nunc orci leo, varius eget
4+
ligula vulputate, consequat eleifend nisi. Cras justo purus, imperdiet a augue
5+
malesuada, convallis cursus libero. Fusce pretium arcu in elementum laoreet. Duis
6+
mauris nulla, suscipit at est nec, malesuada pellentesque eros. Quisque semper porta
7+
malesuada. Nunc hendrerit est ac faucibus mollis. Nam fermentum id libero sed
8+
egestas. Duis a accumsan sapien. Nam neque diam, congue non erat et, porta sagittis
9+
turpis. Vivamus vitae mauris sit amet massa mollis molestie. Morbi scelerisque,
10+
augue id congue imperdiet, felis lacus euismod dui, vitae facilisis massa dui quis
11+
sapien. Vivamus hendrerit a urna a lobortis.
12+
13+
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas sit amet tellus
14+
nec turpis feugiat semper. Nam at nulla laoreet, finibus eros sit amet, fringilla
15+
mauris. Fusce vestibulum nec ligula efficitur laoreet. Nunc orci leo, varius eget
16+
ligula vulputate, consequat eleifend nisi. Cras justo purus, imperdiet a augue
17+
malesuada, convallis cursus libero. Fusce pretium arcu in elementum laoreet. Duis
18+
mauris nulla, suscipit at est nec, malesuada pellentesque eros. Quisque semper porta
19+
malesuada. Nunc hendrerit est ac faucibus mollis. Nam fermentum id libero sed
20+
egestas. Duis a accumsan sapien. Nam neque diam, congue non erat et, porta sagittis
21+
turpis. Vivamus vitae mauris sit amet massa mollis molestie. Morbi scelerisque,
22+
augue id congue imperdiet, felis lacus euismod dui, vitae facilisis massa dui quis
23+
sapien. Vivamus hendrerit a urna a lobortis.
24+

0 commit comments

Comments
 (0)