Skip to content

Commit 34dadec

Browse files
at055612at055612
authored
Merge pull request #5094 from gchq/gh-5091-empty-ref-load
PR for #5091 - Ref loads logging error `We should have at least one UID, else how did we get here`
2 parents 12fa2f9 + 88e2f7b commit 34dadec

File tree

5 files changed

+184
-29
lines changed

5 files changed

+184
-29
lines changed

‎stroom-pipeline/src/main/java/stroom/pipeline/refdata/store/offheapstore/OffHeapRefDataLoader.java‎

Lines changed: 31 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -455,25 +455,32 @@ public void setRangePutOutcomeHandler(final RangePutOutcomeHandler rangePutOutco
455455
private void transferStagedEntries() {
456456
checkCurrentState(LoaderState.STAGED);
457457

458-
transferStagedEntriesTimer = DurationTimer.start();
459-
try (final BatchingWriteTxn destBatchingWriteTxn = refStoreLmdbEnv.openBatchingWriteTxn(maxPutsBeforeCommit)) {
460-
// We now hold the single write lock for the main ref store
461-
462-
updateTaskContextInfoSupplier("Loading staged entries");
463-
transferStagedKeyValueEntries(destBatchingWriteTxn);
464-
transferStagedRangeValueEntries(destBatchingWriteTxn);
465-
466-
// Final commit
467-
destBatchingWriteTxn.commit();
458+
final int putsToStagingStoreCount = putsToStagingStoreCounter.get();
459+
LOGGER.debug("transferStagedEntries() - putsToStagingStoreCount: {}", putsToStagingStoreCount);
460+
461+
if (putsToStagingStoreCount > 0) {
462+
transferStagedEntriesTimer = DurationTimer.start();
463+
try (final BatchingWriteTxn destBatchingWriteTxn = refStoreLmdbEnv.openBatchingWriteTxn(
464+
maxPutsBeforeCommit)) {
465+
// We now hold the single write lock for the main ref store
466+
updateTaskContextInfoSupplier("Loading staged entries");
467+
transferStagedKeyValueEntries(destBatchingWriteTxn);
468+
transferStagedRangeValueEntries(destBatchingWriteTxn);
469+
470+
// Final commit
471+
destBatchingWriteTxn.commit();
472+
}
473+
transferStagedEntriesTimer.stop();
474+
475+
LOGGER.debug(() -> LogUtil.getDurationMessage(
476+
LogUtil.message(
477+
"Transfer of {} entries from staging store to ref data store for pipe",
478+
ModelStringUtil.formatCsv(putsToStagingStoreCounter), getPipelineNameStr()),
479+
transferStagedEntriesTimer.get(),
480+
putsToStagingStoreCounter.get()));
481+
} else {
482+
updateTaskContextInfoSupplier("No staged entries");
468483
}
469-
transferStagedEntriesTimer.stop();
470-
471-
LOGGER.debug(() -> LogUtil.getDurationMessage(
472-
LogUtil.message(
473-
"Transfer of {} entries from staging store to ref data store for pipe",
474-
ModelStringUtil.formatCsv(putsToStagingStoreCounter), getPipelineNameStr()),
475-
transferStagedEntriesTimer.get(),
476-
putsToStagingStoreCounter.get()));
477484
}
478485

479486
private <K> boolean isAppendableData(final BatchingWriteTxn batchingWriteTxn,
@@ -484,14 +491,14 @@ private <K> boolean isAppendableData(final BatchingWriteTxn batchingWriteTxn,
484491
final Optional<UID> optMaxUidInDb = entryStoreDb.getMaxUid(batchingWriteTxn.getTxn(), pooledUidBuffer);
485492
final Set<UID> stagedUids = offHeapStagingStore.getStagedUids();
486493

487-
if (stagedUids.isEmpty()) {
488-
throw new RuntimeException(LogUtil.message(
489-
"We should have at least one staged UID, else how did we get here"));
490-
}
491-
492494
final boolean isAppendable;
493-
if (optMaxUidInDb.isEmpty()) {
495+
if (stagedUids.isEmpty()) {
496+
LOGGER.debug("isAppendableData() - No staged UIDs");
497+
// Return value doesn't really matter as there is nothing to append/put
498+
isAppendable = true;
499+
} else if (optMaxUidInDb.isEmpty()) {
494500
// Totally empty DB, so we are appending
501+
LOGGER.debug("isAppendableData() - Empty optMaxUidInDb");
495502
isAppendable = true;
496503
} else {
497504
final UID maxUidInDb = optMaxUidInDb.get();

‎stroom-pipeline/src/test/java/stroom/pipeline/refdata/store/offheapstore/TestRefDataOffHeapStore.java‎

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ void testConsumeEntryStream() {
107107
refDataStore.consumeEntries(
108108
refStoreEntry ->
109109
refStoreEntry.getKey().equals("key5")
110-
|| refStoreEntry.getKey().equals("key2"),
110+
|| refStoreEntry.getKey().equals("key2"),
111111
null,
112112
entries::add);
113113

@@ -489,6 +489,38 @@ void loader_reloadAfterCompleted() {
489489
.isEmpty();
490490
}
491491

492+
@Test
493+
void testNoEntries() {
494+
final boolean overwriteExisting = true;
495+
final RefStreamDefinition refStreamDefinition = buildUniqueRefStreamDefinition();
496+
final long effectiveTimeMs = System.currentTimeMillis();
497+
498+
assertThat(refDataStore.getLoadState(refStreamDefinition))
499+
.isEmpty();
500+
501+
assertThat(refDataStore.getKeyValueEntryCount())
502+
.isEqualTo(0);
503+
504+
final AtomicBoolean wasWorkDone = new AtomicBoolean(false);
505+
refDataStore.doWithLoaderUnlessComplete(refStreamDefinition, effectiveTimeMs, loader -> {
506+
wasWorkDone.set(true);
507+
loader.initialise(overwriteExisting);
508+
509+
// No puts at all
510+
511+
loader.markPutsComplete();
512+
loader.completeProcessing(ProcessingState.COMPLETE);
513+
});
514+
515+
assertThat(wasWorkDone)
516+
.isTrue();
517+
518+
refDataStore.logAllContents(LOGGER::debug);
519+
520+
assertThat(refDataStore.getLoadState(refStreamDefinition))
521+
.hasValue(ProcessingState.COMPLETE);
522+
}
523+
492524
@TestFactory
493525
Stream<DynamicTest> loader_reloadAfterOtherStates() {
494526

@@ -961,7 +993,7 @@ void testPurgeOldData_partial() {
961993
expectedRefStreamDefCount * keyValueMapCount * entryCount,
962994
expectedRefStreamDefCount * rangeValueMapCount * entryCount,
963995
(expectedRefStreamDefCount * rangeValueMapCount * entryCount) +
964-
(expectedRefStreamDefCount * rangeValueMapCount * entryCount));
996+
(expectedRefStreamDefCount * rangeValueMapCount * entryCount));
965997
}
966998

967999
@Test
@@ -1029,7 +1061,7 @@ void testPurgeOldData_partial_2() {
10291061
expectedRefStreamDefCount * keyValueMapCount * entryCount,
10301062
expectedRefStreamDefCount * rangeValueMapCount * entryCount,
10311063
(expectedRefStreamDefCount * rangeValueMapCount * entryCount) +
1032-
(expectedRefStreamDefCount * rangeValueMapCount * entryCount));
1064+
(expectedRefStreamDefCount * rangeValueMapCount * entryCount));
10331065
}
10341066

10351067
@Test

‎stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/StroomStreamStatus.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,12 @@ public String toString() {
6060
StandardHeaderArguments.X_FORWARDED_FOR,
6161
StandardHeaderArguments.REMOTE_HOST,
6262
StandardHeaderArguments.REMOTE_ADDRESS,
63-
StandardHeaderArguments.RECEIVED_PATH);
63+
StandardHeaderArguments.RECEIVED_PATH,
64+
StandardHeaderArguments.RECEIPT_ID);
6465

6566
final String clientDetailsStr = clientDetailsStringBuilder.isEmpty()
6667
? ""
67-
: " - " + clientDetailsStringBuilder;
68+
: " -" + clientDetailsStringBuilder;
6869

6970
return stroomStatusCode.getHttpCode() + " - " + buildStatusMessage() + clientDetailsStr;
7071
}

‎unreleased_changes/20250904_100327_231__5091.md‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
* Issue **#5091** : Stop reference data loads failing if there are no entries in the stream.
2+
3+
4+
```sh
5+
# ********************************************************************************
6+
# Issue title: Ref loads logging error `We should have at least one UID, else how did we get here`
7+
# Issue link: https://github.com/gchq/stroom/issues/5091
8+
# ********************************************************************************
9+
10+
# ONLY the top line will be included as a change entry in the CHANGELOG.
11+
# The entry should be in GitHub flavour markdown and should be written on a SINGLE
12+
# line with no hard breaks. You can have multiple change files for a single GitHub issue.
13+
# The entry should be written in the imperative mood, i.e. 'Fix nasty bug' rather than
14+
# 'Fixed nasty bug'.
15+
#
16+
# Examples of acceptable entries are:
17+
#
18+
#
19+
# * Issue **123** : Fix bug with an associated GitHub issue in this repository
20+
#
21+
# * Issue **namespace/other-repo#456** : Fix bug with an associated GitHub issue in another repository
22+
#
23+
# * Fix bug with no associated GitHub issue.
24+
25+
26+
# --------------------------------------------------------------------------------
27+
# The following is random text to make this file unique for git's change detection
28+
# ZDjhYDU7niB6dfQeO2iFur5sQOJxqYxS7zZNqOBYxJKYPMYDDjNgKWhtoCV8A6ODDQHyfyXXT0LdS31y
29+
# i0uxWsStqqoeSR9fXZ2XkKi82zpGOr9KChsbY9tCDhujMenaGYqNy12eMOeHTDEdLPsthkN805camQnB
30+
# t6PjxUsWrjqMOsflNb4j7wYvBEWnL0CYTTwIJ1NfL0Ss2AQyFqZCF3L1KNWyAPFEtQFViyJ9AYIa1ALV
31+
# wuODVmieQ1cucqVPME85znx8OFRcydhixzA13SnpqqUfOmssJ0FRfO1QCoWbtaS2LXHxZ6NtZlJDK5gv
32+
# 8HTYVhfFqZDMHjfRZjfelqzQJb6IsWNSZL0lRh8cRsuWo6pDOWghYKm5o9Nd8WNLVMrADmz6xwKpR0Ql
33+
# qA57SFh6P2SM5bKkr8kJEILVHb7p1BrBwXhJMkIC7g6ByJNf56ClHGKmehDYd4VqXMdXk6x6U004XPcY
34+
# nGQRWVIW7Uz07Ef3IdUsqtKgoPTETytF5zOCNigqEc6cV9Fb1i0bMU0nGDGJ7zGLldIU6K1oVEFooK8X
35+
# tQ7DaH8UusfePoIpt5AY5NmN0ZPYFCALtS7E3wLTDMRM8zB2bcaRO9bSjkpeHOBR7AwWInJ5xBcNg4xS
36+
# irVKMIoEAGawS5aTux78GdngxFZZGQPRwH3wjlcnU8VvauPwYUIkKZ8Pajy064E27V9TwyyZlCHJW21s
37+
# BtXJVTCZEs7FEAujPx5MwNXUk9OTpqAkcMkx4mYLxsAN9HMBSRmXKU4fH5AOxJI3lZdWFgBvgnhoRKr5
38+
# 604I5Lp4c1OlwFW0xu3d3Dlbtx87kc9tejSAhskGyiIoqL5NVF1HxXjtEkwqQrvpjwr0LUtf73oJe6cM
39+
# F8PxPqsksEKky6Fq6znA0FL09lN6SyzPFs8FGSTMCwvqp5hRqGtMsnVzvxA1McJHJxR3jYisZLwOjODa
40+
# vVarAegEG567Ha9K0QQ39mP3CPh0D0lzXL3QQkbIpVbEZVMSSPS0bCg99fIjPXwFYpzrH0b3JFiJzean
41+
# wpz2aFq2dUcFT9pySWq0cqehB2VyrRY0sLyebAJgD8pRM3SACrh6bBNsdwDArPZzBX5aUFyC4NufHwDy
42+
# 3IdupLH3L6v7T30xmEnbiu1HkJLA339pRlVQwg45JRrmIrDLz7f8eQM7RsFqRE8qHeUKmZWyAE5A4OMi
43+
# H9wZk0A7J524czWT0YA6V18HzxpRLUDnGdbG6aAuW9HcBodOMe10mb9DXWNnOKkr9ZHw6pd8LAt2WLms
44+
# t2qMNlWz3aYsvxmM7hVXllHq9EFjhf467nVp39EcWbrPSx8rXrndKOx77MbVJcdmnXPRD9fZm51vygDn
45+
# IUEXKm8JBnOSooQoCF53yyKCBOciQAOeQQ2fT9rTnH7UO3Wm7bWBVJ1gBWncs6RlDMxjiT18s9qkgarv
46+
# EFiwrVtJrvFLCRfZqGJaQ8KXgj7xMWsHhnA5pKruMhmxFOYgxirxtQTqPucxAI9tHKB4C9xl3bEE2FLp
47+
# 5HFZyTVZdrgNgV9eOO45dP5dbHUWsbYINduPWXzsyRRJiNLe7FwmZAJezJsGHXgtlNdp00QkfVv2x8zY
48+
# 7jfFVlOCSinhlhW1LtNXXDYoLaEE5uDJgmqgPfg6UVhLVz8ECIGeaGAr4gvGrIMWbO7y2rciwbH6x2b6
49+
# FlJUVNBOghlTWkOxpIjFxMziXVyfy31CLA1bxNyFBa4YlHEvk5lsI8K50kzab7QIksEhxz1iMPagr9Qb
50+
# um3fGiDWKoJ92rIgRLtLEwLU3Ag6QfWicHSX4qIDj7RpaFrwg7XPZJMaHiOCjfqZq2KvxERkqHdBYeWy
51+
# l2Fzf2k9RITbx9hybAo0rX1QfveONybXyvuy1vWiZxR8aYReL3QkhQlNCwoOswo2LD0dtlVxZ6OyN5R4
52+
# cMmmw29zX1boHx2YEzO0nlJev9mabsRBzsE5ZN5crmsVbIdSZtwk7f29hPaZ6ZlqjzPxhjYkj7PcbvrS
53+
# 79g7ePOVAzPyox7Sy8Nlmz8F3MavFt4lRTIf9sbxp2r85DgYj5ruEbQBT6ekhpy7BlpmsJ3tmDGM4hzm
54+
# ccCSlzBh32uI4Cz8ku4TbmLoZwpwhPtSLBI5c0S3RjKw7sKFgdjKKReUMq5oLI9WKYCmn8nBP0Y06M5t
55+
# 1xNECzCuUu6RNpdlyS15lCqYR0fqcmhRThNLbLDNQV5DRKs8L75oWZTiXoph8tnAPUYLlD5sBDdfX8vA
56+
# 3AC3OGfATkxf4YmO8oCzgTuQHUWcqYeY8xXeYgIrigotcfKzcn0EaPm933L2MOSRLRk4jDLbWU2ylTVl
57+
# 0t8Me7c0OX5nRCkb0tcLJbpDhcNG7N27EzVUjAGbYf6G0YA6oI0xocXliXX59uqvxEpLd8c4H9xt53Az
58+
# --------------------------------------------------------------------------------
59+
60+
```

‎unreleased_changes/20250904_104631_765__0.md‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
* Add `ReceiptId` to the INFO message on data receipt.
2+
3+
4+
```sh
5+
# ONLY the top line will be included as a change entry in the CHANGELOG.
6+
# The entry should be in GitHub flavour markdown and should be written on a SINGLE
7+
# line with no hard breaks. You can have multiple change files for a single GitHub issue.
8+
# The entry should be written in the imperative mood, i.e. 'Fix nasty bug' rather than
9+
# 'Fixed nasty bug'.
10+
#
11+
# Examples of acceptable entries are:
12+
#
13+
#
14+
# * Issue **123** : Fix bug with an associated GitHub issue in this repository
15+
#
16+
# * Issue **namespace/other-repo#456** : Fix bug with an associated GitHub issue in another repository
17+
#
18+
# * Fix bug with no associated GitHub issue.
19+
20+
21+
# --------------------------------------------------------------------------------
22+
# The following is random text to make this file unique for git's change detection
23+
# bqQURaUsOfNNjLGViU5HGh7OT8iM6z3EUK4D0DvIfpIONBgFW8Tv0xGNddy2pEZOQ0Ok7TSl5N4ZSjRH
24+
# pI01YF21DmaKafCWvNLF0ozHUKBvQ6OcTFaxLsg1LAeeyXyGQLssxckCpdqS2LZ34OzMGwPojvoKj4J6
25+
# sGhcS80hAgUO81ju1MOu2w3VjN7rpOTxPlWpdKz2rrpLdLm3tuFPuuBky3WcbaTr13bS7lIgO2U3UYSc
26+
# I1sdKo34HwpzBm7KqmjsI8nrf6QKwAEwEg49R6x1WQqeXJ1zCLwemgpsRQO808I2Gfp4E50IjGNSSLcX
27+
# wM2jdztQg5IIFRa1poLSlEY1qOEhmhI5L4fCstgLS2PX0bdKcod29yUalCBzwx9mlvaKBdzJ9JIZpJ5V
28+
# k7sCgcAjElsZlTzm56zwAYFXlLd5jYlsoJpe1zWTKaDZqQU1CTZSJRfxoIOVRb24rqVJBLtaXXEKP5j8
29+
# 4aPyxsBDhCkWfnfx8ECOEDPZ9JrAyIglntiH5UzSwgjdJwe82ITfceDq7Sw6p86OnPZbSmPFvUp82TLH
30+
# XNfQJHQf5Y6ffatr7kBTmUxu7RxLxF99yMxQEvF6G1XhLBGZpTYMVh1OVSWV0tHq9DxiP53a1NrqBPMh
31+
# ExwIwFbkivBe8CUYiAI3rXXHloLewy0H7wbScUWq2fJPf4LWNYxos5uXtVXITOl4Sjl24WQZGxmbK6Pr
32+
# UrowTzP2HwzUc9xbqUEYYHkpbpskRykywoNCWSmpd96FA3ObWLeVxiU1gXss9pCsHiFHvcriC5czK8V2
33+
# zmGoo2h6ugLoyn19XyKSQuagmAIPF435O2a4AXMRZFy7XUX6BUyPUng5a6louyML9LZ49KxttlNIKn0Q
34+
# 8RQNlncvehqba7BChA3361H42RkpNDzRl7Hnrnjsz43LgaqmXGzJWW2PYYc37uMrlP090KoTeKGcPFBZ
35+
# GbDfPipK8bAfarYlrQ2A1PvTfg3hyyVdjsidNJYDSMgdXwo56beUUb0Wq9OYJcQREML92owyoLBvPDAe
36+
# nFb96xfjOHeWHSD3oS1bOoRWPebgGF16TZ7VuOOBt1JBCtxTVgsmN0uQsRuDMu1VAbnEZ7CNtdEDzIXF
37+
# yjYcBCYSAVV1DQgoLrdXGTPDPlC9f35XchjEVcTLWtKvCqiaa0LKYSWxpLgvUBMiS2g9Wb6Ytg2tAube
38+
# ck3if9NFaBSVpqLy5ekAyfofHPe1LPsYUEb6pqWTqTqFspsMnHh79PhPskK0ziLMm9Wrd6xRk37vsDoS
39+
# v9XYSnIn2pAbHnPpbtmnFRtuGKruWUI0JeMkXN6G9slusrhPfNNcMDrTvvjXG26iWhHbWq1xx4sVXD58
40+
# uPGxX9xiHyANbYuiSs9kl4TMVhiaKBz4OvpvYfEBg9ByNV8IpNO6NI871WRUeKax0RuuYPJEvIVjhqgi
41+
# GaQym5znivqnapzmULkBHblOg2lDtjRkQxSYFkU6v0e1oH90njXG3yYEMJfwBbh2utpcpsYPOaapa6Dp
42+
# NHi84Bub1k4xyWXrnI3m9eYI3FI2IroUnCtpYfmCQtOVzRMFSaYZ2GycxLOsm91M0kkl15m8OayD8iwT
43+
# X53v8asrGXHzWNho7nJxPH8Ii7vmCahwK4MuCJmxY8W9U7ppyceMpwOBy5hS4Ulmvv0QeDaGXwVvik0f
44+
# kK7rJGqUUeAWIY2UzE3m4VsbLIfAFbkfoHeS0whmV835NV0L8wbXVroEyiUxqIIivhK1Sj5usIJWsgft
45+
# ep4Dp9eZpizppH3EShccQdnjuYZANrY67LzAWqB4Cbf4zvzmu99WjujC1FCbUJ3bJu9MfLIiHnkQrAq2
46+
# q3IUJJgt3t7pmKQT8OwmO2XF5KudtXw0E7eNEfNpJN1ZaW9xZvbMvssLYCrdptcBYOl0MmU6XreaQSTk
47+
# 71Bd6itY91gUvBMfBSeURcJnOv4JzrNmrcO1ambXIxnQ2sLFUc5tXzeWrLbvvmiwtaA6DFwBazeglJTC
48+
# LcDWbIHBBy2KON4SuVq6RfMwaOHZhs9GHKcD7JLkJ3mORQdxJhxSFd56K7mJ6ab1FNvYjnNyT8Tdva4H
49+
# 8bVwSo3g2MksRoV95TuNj9O77I0L8FN5JcOkkRhDvkht1HDjMFJLIrKzVkYg333D65NOZ65mFCSDX3bb
50+
# tSx43JQCgN7fwNSTHDiTJmLIaUh0f2sJPBCrsfvFE03HD8iG87AG8HAAmKr4f9cD1EVMsGlc9ziXBDeE
51+
# GzZMgnEQ0haWmxibVCchlCWVFcoB6hZVM9QxYkJ1T7rCzePAXs8tbcZc83U4BG1o7gqOz5EAC3ogeAmC
52+
# qkEIMlSRbantsXEGO6qYLl86IFFmvSOXKepvY2WrvNu5MZIpjuoksakDtp1uKNfHr0ZGFoWSB8s1Og6u
53+
# --------------------------------------------------------------------------------
54+
55+
```

0 commit comments

Comments
 (0)