Skip to content

Commit 47dff1b

Browse files
stroomdev66stroomdev66
authored
Merge pull request #5122 from gchq/gh-5121_proxy_aggregation
#5121 Proxy aggregation now keeps only common headers in aggregated data
2 parents 3065d65 + 9e48754 commit 47dff1b

File tree

2 files changed

+96
-9
lines changed

2 files changed

+96
-9
lines changed

‎stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/handler/Aggregator.java‎

Lines changed: 36 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package stroom.proxy.app.handler;
22

3+
import stroom.meta.api.AttributeMap;
4+
import stroom.meta.api.AttributeMapUtil;
35
import stroom.proxy.app.DataDirProvider;
46
import stroom.util.io.FileName;
57
import stroom.util.io.FileUtil;
@@ -17,6 +19,8 @@
1719
import java.nio.file.Files;
1820
import java.nio.file.Path;
1921
import java.util.Iterator;
22+
import java.util.Map;
23+
import java.util.Objects;
2024
import java.util.concurrent.atomic.AtomicBoolean;
2125
import java.util.concurrent.atomic.AtomicLong;
2226
import java.util.function.Consumer;
@@ -76,23 +80,38 @@ public void addDir(final Path dir) {
7680
final Path tempDir = tempAggregatesDirProvider.get();
7781
final FileGroup outputFileGroup = new FileGroup(tempDir);
7882
final AtomicLong count = new AtomicLong();
79-
final AtomicBoolean doneMeta = new AtomicBoolean();
83+
final AttributeMap commonHeaders = new AttributeMap();
84+
final AtomicBoolean doneFirstMeta = new AtomicBoolean();
8085
// Get a buffer to help us transfer data.
8186
final byte[] buffer = LocalByteBuffer.get();
8287

8388
try (final ProxyZipWriter zipWriter = new ProxyZipWriter(outputFileGroup.getZip(), buffer)) {
8489
FileUtil.forEachChild(dir, fileGroupDir -> {
8590
final FileGroup fileGroup = new FileGroup(fileGroupDir);
8691

87-
// Output meta if this is the first.
88-
if (!doneMeta.get()) {
89-
try {
90-
Files.copy(fileGroup.getMeta(), outputFileGroup.getMeta());
91-
doneMeta.set(true);
92-
} catch (final IOException e) {
93-
LOGGER.error(e::getMessage, e);
94-
throw new UncheckedIOException(e);
92+
// Combine common header keys and values.
93+
try {
94+
if (doneFirstMeta.compareAndSet(false, true)) {
95+
// Load initial common headers from the first meta.
96+
AttributeMapUtil.read(fileGroup.getMeta(), commonHeaders);
97+
} else {
98+
// Remove headers that don't exist or are different in subsequent meta files.
99+
final AttributeMap headers = new AttributeMap();
100+
AttributeMapUtil.read(fileGroup.getMeta(), headers);
101+
final Iterator<Map.Entry<String, String>> iterator =
102+
commonHeaders.entrySet().iterator();
103+
while (iterator.hasNext()) {
104+
final Map.Entry<String, String> entry = iterator.next();
105+
final String otherValue = headers.get(entry.getKey());
106+
// If this header is different then remove the common header.
107+
if (!Objects.equals(entry.getValue(), otherValue)) {
108+
iterator.remove();
109+
}
110+
}
95111
}
112+
} catch (final IOException e) {
113+
LOGGER.error(e::getMessage, e);
114+
throw new UncheckedIOException(e);
96115
}
97116

98117
try (final ZipFile zipFile = ZipUtil.createZipFile(fileGroup.getZip())) {
@@ -122,6 +141,14 @@ public void addDir(final Path dir) {
122141
throw new UncheckedIOException(e);
123142
}
124143
});
144+
145+
// Now write the common header arguments to the aggregate meta file.
146+
try {
147+
AttributeMapUtil.write(commonHeaders, outputFileGroup.getMeta());
148+
} catch (final IOException e) {
149+
LOGGER.error(e::getMessage, e);
150+
throw new UncheckedIOException(e);
151+
}
125152
}
126153

127154
// We have finished the merge so transfer the new item to be forwarded.

‎unreleased_changes/20250917_111258_412__5121.md‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
* Issue **#5121** : Proxy aggregation now keeps only common headers in aggregated data.
2+
3+
4+
```sh
5+
# ********************************************************************************
6+
# Issue title: Proxy aggregation is using headers from first set of headers for merged data but should use only common
7+
# Issue link: https://github.com/gchq/stroom/issues/5121
8+
# ********************************************************************************
9+
10+
# ONLY the top line will be included as a change entry in the CHANGELOG.
11+
# The entry should be in GitHub flavour markdown and should be written on a SINGLE
12+
# line with no hard breaks. You can have multiple change files for a single GitHub issue.
13+
# The entry should be written in the imperative mood, i.e. 'Fix nasty bug' rather than
14+
# 'Fixed nasty bug'.
15+
#
16+
# Examples of acceptable entries are:
17+
#
18+
#
19+
# * Issue **123** : Fix bug with an associated GitHub issue in this repository
20+
#
21+
# * Issue **namespace/other-repo#456** : Fix bug with an associated GitHub issue in another repository
22+
#
23+
# * Fix bug with no associated GitHub issue.
24+
25+
26+
# --------------------------------------------------------------------------------
27+
# The following is random text to make this file unique for git's change detection
28+
# 0EtMqVeb4gy7lF38eYANQGpQhzsF1zd1d2FokjyzVdhBsBZb7atlzgxj3VTde2kT41sgIVzkI2nQf6Dt
29+
# cXpAqR163TLz17MtSyTc5TBzU7JMdg6KCMON2eNMDNQYzCQbDGJUSWn8ejpRg6NxOqzXu9Dn2A0pwvQO
30+
# xOQPHnoXO9CUl8FHLsDVBMo1jnStXn5NI4fqjnfm5pbp5UTgT9dTU9b0SJm0p84moxGMlTsc7bF75Fa5
31+
# I925MDkJsXX8EUP3ZAcJxJ9NhzAPk7P9Zb6rYvTlW1L8gGd14IFpUkCg1NUxZT9t9smVQaqKlZOuCwV8
32+
# TkU4UhTb489Z59uCdA3MvGhyMCa1CYW3NgpfSxYqBkzVVNdgiIhT0Pip75bRbgzxhuwdVYt9dnDk47Jo
33+
# Z5hf7oOX2Hyjc9kJ1NaAezfw7ugcMx3n3WyCRKBICTBghRKuzFqexRnarmOTW3Pi6eY0xayPEdBGmMQm
34+
# eUgbWWgdYwQCx5iNyx08PJSVWi7uCC2qJCRKmbzNrmXDsK03yus59Lwvj2DNicI6VLyxaJk8WN7kDGAs
35+
# 1lHkyUno7BTYHFVGdIH0SRPRKRGa7ki0nDR116KAr6R8oD8Fwgb4Qttcw1y0UIt007svzcdQIMMTq4Cr
36+
# fig5WSg5RIR25wW7uTflEpz3DPV0c6DiUv4uu55vMeFen26l7FZqogQOKQmTEQAYv15xiury09M9371w
37+
# TkPo4Qu26EoYHyhVVuxe7rEMfp1ydCUpiJrFyYcpnWFkfcqsEomUzZcwkDFO2tmJU0b6DMAzpWeDtnYN
38+
# UKxwINkqeguGK4PwLQyGXB9yheYgEI69EWSPQAChYp9rPo8SfgKIpN5CNRDtwwrRgjxAwRtMJDpIiDj7
39+
# MILae0E1eKPZDmMYN9rPYbPyubG0S6nlSU4CcvEH74wyJdLGT8oBPH2gRAtbKtCfoP96q63lTbHg5JOK
40+
# mbtZhru56WGYrkGQM6kQ0hJ0U4JZJyjL6zAR4kQ0Doi3jji1ZZT2XxJe7WfDtpeN4W0LvN71EESA2Tqx
41+
# I7fFD8iC3KAQKEcy01L0BHWPXmAgH6JbNDr4nozo87FzTBUoW83uH7gsNTAq0xclRGT0kjWivSHOCjUm
42+
# zra60MWvQ2ILDUOQEJZLWQOLDcwT0SDKndRlHxEF0DK1qT01HieQwsVf53rFZIBMXrFFD5QACIraWe3C
43+
# rgniwUk8bdwhadwTHu2UIfzuDF57hvEbZbNqAOCkbsQebYNeIYV9IIbH98wQa36XXNFg54tyEXDRWhLA
44+
# UDGht9Dw0iqBr6ii8p7NVAC6dyylvJByLeNpo3kuSWcipd9hwOz2jTryEqTJbSBIaAlN3EvOslH3xb5g
45+
# kYVEbSmelVnLRFpxasUQN7OI033fKOz7KfRfzMwBYrlBM9z8qnjWNmbVxw7z1s7FuLNU6003T09424ep
46+
# bSIwr7raM1051vCB6kKjd2dXREb7JmYuCAhpNlgeqAmITMyfQqurIwccCuaMBHCDurG4pstePVPjA1V1
47+
# YDhUao2g9kGzo4vCrVe0Vh91tagE4EwFSgUPLq72OnUhfENeh5QCX7xI3Irm6Xxob5LM6ImJ6kynzxKY
48+
# 3ferUEqNxSb02hL5KEWvH3oRAqZ2o4YDpgQOCijPUwhQ3Ahtlb6tFx8I9M9qjobbEQIYASmBghBiLv9Z
49+
# 8AC1nAOvGgwfcRPaVmUx55nJNeYj3c9NblkWeIZG8DpZlGvG5PphVYaePaRA7sA3ftnTqvh56IPilTit
50+
# pzqqZLUEJiHiSt9xsgVUcJVKwXnGKbdWFqvd6Tpbf3zeRTDKb7hlxqGHMu87JRYgsnZZzsmaiDVa8Tow
51+
# sg00eb7YCFNaoiHAaa3e6ylkfsXFSffgV86Hkb70RAGCwUscDM7bRRQyUuJBsVW7GjBtmcGWtaKgB8ST
52+
# 6vYTMgMGcSSb8nyqz28RBV6YFjaOjRWeoWKSTUjF5lpcPQPydfOJLAFSYj3XwZLxnWev4kVS7voH5vTx
53+
# LnDN4QbSfWJ6PRrVQbd9ryQl1CCqoMga2UozzKcxy8oCmgwsK0Mq0GlAPhFiEphSihsEU5c9Y3y9tmRx
54+
# xdq9nQsYR3PEee7mYsfenkl0nh0OSawCGsXYiNXMdPJ7cJSVEHpwwxhWUbZotaI9OyLJvrSpivAtPNzd
55+
# GOH9qf9Tgy5gkFZSZgwd65Ufn1cdHAzaprwFZbcLjm4ISnAHISrzFDd3y81egrYr7La0w36QfpO0bdL5
56+
# bNl7o4JbcHbIhDQ6aVmt43J1ZXDsXMCOcdUZAZnANOGCW5oUSL6KLVyiIYmUkEUuszOuKx8JcocpuRL3
57+
# LMPRQr591ED8vrtpTbF0UMDNJtp8GoMiappafNAcVDcLBdP8GYR1z8Rv2xpl6Mgr7ME6BSWVmnqEdSdU
58+
# --------------------------------------------------------------------------------
59+
60+
```

0 commit comments

Comments
 (0)