From 35dad44a81820d9bf876fd265e95d20e61baee9f Mon Sep 17 00:00:00 2001 From: cecemei Date: Fri, 30 Jan 2026 00:44:15 -0800 Subject: [PATCH 01/19] wip --- .../indexing/compact/CompactionJobQueue.java | 9 +- .../ClientCompactionTaskQuerySerdeTest.java | 2 +- .../ClientCompactionIntervalSpec.java | 35 ++++- .../compaction/BaseCandidateSearchPolicy.java | 2 +- .../compaction/CompactionCandidate.java | 97 +++++++++++- .../CompactionCandidateSearchPolicy.java | 24 ++- .../compaction/CompactionRunSimulator.java | 7 +- .../server/compaction/CompactionStatus.java | 55 ++++--- .../compaction/CompactionStatusTracker.java | 14 +- .../DataSourceCompactibleSegmentIterator.java | 15 +- .../compaction/FixedIntervalOrderPolicy.java | 2 +- .../MostFragmentedIntervalFirstPolicy.java | 57 +++++-- .../coordinator/duty/CompactSegments.java | 29 +++- .../ClientCompactionIntervalSpecTest.java | 42 ++++++ .../CompactionStatusTrackerTest.java | 6 +- ...MostFragmentedIntervalFirstPolicyTest.java | 139 +++++++++++++++--- .../coordinator/duty/CompactSegmentsTest.java | 1 + 17 files changed, 437 insertions(+), 99 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobQueue.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobQueue.java index 9446ac664f29..cdfb41c5f129 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobQueue.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobQueue.java @@ -217,7 +217,7 @@ public void runReadyJobs() final List pendingJobs = new ArrayList<>(); while (!queue.isEmpty()) { final CompactionJob job = queue.poll(); - if (startJobIfPendingAndReady(job, searchPolicy, pendingJobs, slotManager)) { + if (startJobIfPendingAndReady(job, pendingJobs, slotManager)) { runStats.add(Stats.Compaction.SUBMITTED_TASKS, RowKey.of(Dimension.DATASOURCE, job.getDataSource()), 1); } } @@ -267,7 +267,6 @@ public Map getSnapshots() */ private boolean startJobIfPendingAndReady( CompactionJob job, - CompactionCandidateSearchPolicy policy, List pendingJobs, CompactionSlotManager slotManager ) @@ -282,7 +281,7 @@ private boolean startJobIfPendingAndReady( } // Check if the job is already running, completed or skipped - final CompactionStatus compactionStatus = getCurrentStatusForJob(job, policy); + final CompactionStatus compactionStatus = getCurrentStatusForJob(job); switch (compactionStatus.getState()) { case RUNNING: return false; @@ -378,9 +377,9 @@ private void persistPendingIndexingState(CompactionJob job) } } - public CompactionStatus getCurrentStatusForJob(CompactionJob job, CompactionCandidateSearchPolicy policy) + public CompactionStatus getCurrentStatusForJob(CompactionJob job) { - final CompactionStatus compactionStatus = statusTracker.computeCompactionStatus(job.getCandidate(), policy); + final CompactionStatus compactionStatus = statusTracker.computeCompactionStatus(job.getCandidate()); final CompactionCandidate candidatesWithStatus = job.getCandidate().withCurrentStatus(null); statusTracker.onCompactionStatusComputed(candidatesWithStatus, null); return compactionStatus; diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java index 1c04c7b5bd2b..842227efb01e 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java @@ -301,7 +301,7 @@ private ClientCompactionTaskQuery createCompactionTaskQuery(String id, Compactio id, "datasource", new ClientCompactionIOConfig( - new ClientCompactionIntervalSpec(Intervals.of("2019/2020"), "testSha256OfSortedSegmentIds"), true + new ClientCompactionIntervalSpec(Intervals.of("2019/2020"), null, "testSha256OfSortedSegmentIds"), true ), new ClientCompactionTaskQueryTuningConfig( 100, diff --git a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpec.java b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpec.java index 7a7f65572319..46707e1ea55a 100644 --- a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpec.java +++ b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpec.java @@ -20,12 +20,16 @@ package org.apache.druid.client.indexing; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.java.util.common.IAE; +import org.apache.druid.query.SegmentDescriptor; import org.joda.time.Interval; import javax.annotation.Nullable; +import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; /** * InputSpec for {@link ClientCompactionIOConfig}. @@ -38,11 +42,14 @@ public class ClientCompactionIntervalSpec private final Interval interval; @Nullable + private final List uncompactedSegments; + @Nullable private final String sha256OfSortedSegmentIds; @JsonCreator public ClientCompactionIntervalSpec( @JsonProperty("interval") Interval interval, + @JsonProperty("uncompactedSegments") @Nullable List uncompactedSegments, @JsonProperty("sha256OfSortedSegmentIds") @Nullable String sha256OfSortedSegmentIds ) { @@ -50,6 +57,22 @@ public ClientCompactionIntervalSpec( throw new IAE("Interval[%s] is empty, must specify a nonempty interval", interval); } this.interval = interval; + if (uncompactedSegments == null) { + // perform a full compaction + } else if (uncompactedSegments.isEmpty()) { + throw new IAE("Can not supply empty segments as input, please use either null or non-empty segments."); + } else if (interval != null) { + List segmentsNotInInterval = + uncompactedSegments.stream().filter(s -> !interval.contains(s.getInterval())).collect(Collectors.toList()); + if (!segmentsNotInInterval.isEmpty()) { + throw new IAE( + "Can not supply segments outside interval[%s], got segments[%s].", + interval, + segmentsNotInInterval + ); + } + } + this.uncompactedSegments = uncompactedSegments; this.sha256OfSortedSegmentIds = sha256OfSortedSegmentIds; } @@ -65,6 +88,14 @@ public Interval getInterval() return interval; } + @Nullable + @JsonProperty + @JsonInclude(JsonInclude.Include.NON_NULL) + public List getUncompactedSegments() + { + return uncompactedSegments; + } + @Nullable @JsonProperty public String getSha256OfSortedSegmentIds() @@ -83,13 +114,14 @@ public boolean equals(Object o) } ClientCompactionIntervalSpec that = (ClientCompactionIntervalSpec) o; return Objects.equals(interval, that.interval) && + Objects.equals(uncompactedSegments, that.uncompactedSegments) && Objects.equals(sha256OfSortedSegmentIds, that.sha256OfSortedSegmentIds); } @Override public int hashCode() { - return Objects.hash(interval, sha256OfSortedSegmentIds); + return Objects.hash(interval, uncompactedSegments, sha256OfSortedSegmentIds); } @Override @@ -97,6 +129,7 @@ public String toString() { return "ClientCompactionIntervalSpec{" + "interval=" + interval + + ", uncompactedSegments=" + uncompactedSegments + ", sha256OfSortedSegmentIds='" + sha256OfSortedSegmentIds + '\'' + '}'; } diff --git a/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java index 2a9107132623..c707929b8f72 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java @@ -73,7 +73,7 @@ public Eligibility checkEligibilityForCompaction( CompactionTaskStatus latestTaskStatus ) { - return Eligibility.OK; + return Eligibility.FULL_COMPACTION_OK; } /** diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java index af8b32ebe6db..cf11eaf31703 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java @@ -19,15 +19,19 @@ package org.apache.druid.server.compaction; +import org.apache.druid.error.DruidException; import org.apache.druid.error.InvalidInput; import org.apache.druid.java.util.common.JodaUtils; +import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.segment.SegmentUtils; +import org.apache.druid.server.coordinator.DataSourceCompactionConfig; import org.apache.druid.timeline.DataSegment; import org.joda.time.Interval; import javax.annotation.Nullable; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -44,6 +48,7 @@ public class CompactionCandidate private final long totalBytes; private final int numIntervals; + private final CompactionCandidateSearchPolicy.Eligibility policyEligiblity; private final CompactionStatus currentStatus; public static CompactionCandidate from( @@ -68,6 +73,7 @@ public static CompactionCandidate from( umbrellaInterval, compactionInterval, segmentIntervals.size(), + null, null ); } @@ -77,6 +83,7 @@ private CompactionCandidate( Interval umbrellaInterval, Interval compactionInterval, int numDistinctSegmentIntervals, + CompactionCandidateSearchPolicy.Eligibility policyEligiblity, @Nullable CompactionStatus currentStatus ) { @@ -88,6 +95,7 @@ private CompactionCandidate( this.numIntervals = numDistinctSegmentIntervals; this.dataSource = segments.get(0).getDataSource(); + this.policyEligiblity = policyEligiblity; this.currentStatus = currentStatus; } @@ -160,12 +168,99 @@ public CompactionStatus getCurrentStatus() return currentStatus; } + @Nullable + public CompactionCandidateSearchPolicy.Eligibility getPolicyEligibility() + { + return policyEligiblity; + } + /** * Creates a copy of this CompactionCandidate object with the given status. */ public CompactionCandidate withCurrentStatus(CompactionStatus status) { - return new CompactionCandidate(segments, umbrellaInterval, compactionInterval, numIntervals, status); + return new CompactionCandidate( + segments, + umbrellaInterval, + compactionInterval, + numIntervals, + policyEligiblity, + status + ); + } + + public CompactionCandidate withPolicyEligibility(CompactionCandidateSearchPolicy.Eligibility eligibility) + { + return new CompactionCandidate( + segments, + umbrellaInterval, + compactionInterval, + numIntervals, + eligibility, + currentStatus + ); + } + + /** + * Evaluates this candidate for compaction eligibility based on the provided + * compaction configuration and search policy. + *

+ * This method first evaluates the candidate against the compaction configuration + * using a {@link CompactionStatus.Evaluator} to determine if any segments need + * compaction. If segments are pending compaction, the search policy is consulted + * to determine the type of compaction: + *

    + *
  • NOT_ELIGIBLE: Returns a candidate with status SKIPPED, indicating + * the policy decided compaction should not occur at this time
    • + *
  • FULL_COMPACTION: Returns this candidate with status PENDING, + * indicating all segments should be compacted
    • + *
  • INCREMENTAL_COMPACTION: Returns a new candidate containing only + * the uncompacted segments (as determined by the evaluator), with status + * PENDING for incremental compaction
    • + *
+ * + * @param config the compaction configuration for the datasource + * @param searchPolicy the policy used to determine compaction eligibility + * @return a CompactionCandidate with updated status and potentially filtered segments + */ + public CompactionCandidate evaluate(DataSourceCompactionConfig config, CompactionCandidateSearchPolicy searchPolicy) + { + CompactionStatus.Evaluator evaluator = new CompactionStatus.Evaluator(this, config, null, null); + Pair evaluated = evaluator.evaluate(); + switch (Objects.requireNonNull(evaluated.lhs).getPolicyEligibility()) { + case NOT_ELIGIBLE: // failed evaluator check + return this.withPolicyEligibility(evaluated.lhs) + .withCurrentStatus(CompactionStatus.skipped("Rejected[%s]", evaluated.lhs.getReason())); + case FULL_COMPACTION: + final CompactionCandidateSearchPolicy.Eligibility searchPolicyEligibility = + searchPolicy.checkEligibilityForCompaction(this, null); + switch (searchPolicyEligibility.getPolicyEligibility()) { + case + NOT_ELIGIBLE: // although evaluator thinks this interval qualifies for compaction, but policy decided not its turn yet. + return this.withPolicyEligibility(searchPolicyEligibility) + .withCurrentStatus(CompactionStatus.skipped( + "Rejected by search policy: %s", + searchPolicyEligibility.getReason() + )); + case FULL_COMPACTION: + return this.withPolicyEligibility(searchPolicyEligibility).withCurrentStatus(evaluated.rhs); + case + INCREMENTAL_COMPACTION: // policy decided to perform an incremental compaction, the uncompactedSegments is a subset of the original segments. + return new CompactionCandidate( + evaluator.uncompactedSegments, + umbrellaInterval, + compactionInterval, + numIntervals, + searchPolicyEligibility, + evaluated.rhs + ); + default: + throw DruidException.defensive("Unexpected eligibility[%s]", searchPolicyEligibility); + } + case INCREMENTAL_COMPACTION: + default: + throw DruidException.defensive("Unexpected eligibility[%s]", evaluated.rhs); + } } @Override diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java index bfb69787dd84..4d07d3bb182e 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java @@ -51,8 +51,6 @@ public interface CompactionCandidateSearchPolicy * Checks if the given {@link CompactionCandidate} is eligible for compaction * in the current iteration. A policy may implement this method to skip * compacting intervals or segments that do not fulfil some required criteria. - * - * @return {@link Eligibility#OK} only if eligible. */ Eligibility checkEligibilityForCompaction( CompactionCandidate candidate, @@ -64,18 +62,25 @@ Eligibility checkEligibilityForCompaction( */ class Eligibility { - public static final Eligibility OK = new Eligibility(true, null); + public enum PolicyEligibility + { + FULL_COMPACTION, + INCREMENTAL_COMPACTION, + NOT_ELIGIBLE + } + + public static final Eligibility FULL_COMPACTION_OK = new Eligibility(PolicyEligibility.FULL_COMPACTION, null); - private final boolean eligible; + private final PolicyEligibility eligible; private final String reason; - private Eligibility(boolean eligible, String reason) + private Eligibility(PolicyEligibility eligible, String reason) { this.eligible = eligible; this.reason = reason; } - public boolean isEligible() + public PolicyEligibility getPolicyEligibility() { return eligible; } @@ -85,9 +90,14 @@ public String getReason() return reason; } + public static Eligibility incrementalCompaction(String messageFormat, Object... args) + { + return new Eligibility(PolicyEligibility.INCREMENTAL_COMPACTION, StringUtils.format(messageFormat, args)); + } + public static Eligibility fail(String messageFormat, Object... args) { - return new Eligibility(false, StringUtils.format(messageFormat, args)); + return new Eligibility(PolicyEligibility.NOT_ELIGIBLE, StringUtils.format(messageFormat, args)); } @Override diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java index bab65f90bf94..919c4815bfab 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java @@ -88,12 +88,9 @@ public CompactionSimulateResult simulateRunWithConfig( final CompactionStatusTracker simulationStatusTracker = new CompactionStatusTracker() { @Override - public CompactionStatus computeCompactionStatus( - CompactionCandidate candidate, - CompactionCandidateSearchPolicy searchPolicy - ) + public CompactionStatus computeCompactionStatus(CompactionCandidate candidate) { - return statusTracker.computeCompactionStatus(candidate, searchPolicy); + return statusTracker.computeCompactionStatus(candidate); } @Override diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java index 99e1eef21465..0a10aa9b8554 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java @@ -27,6 +27,7 @@ import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; import org.apache.druid.indexer.partitions.HashedPartitionsSpec; import org.apache.druid.indexer.partitions.PartitionsSpec; +import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.granularity.GranularityType; @@ -273,7 +274,7 @@ static CompactionStatus compute( expectedState ); } - return new Evaluator(candidateSegments, config, expectedFingerprint, fingerprintMapper).evaluate(); + return new Evaluator(candidateSegments, config, expectedFingerprint, fingerprintMapper).evaluate().rhs; } @Nullable @@ -344,7 +345,7 @@ static DimensionRangePartitionsSpec getEffectiveRangePartitionsSpec(DimensionRan * Evaluates {@link #CHECKS} to determine the compaction status of a * {@link CompactionCandidate}. */ - private static class Evaluator + static class Evaluator { private final DataSourceCompactionConfig compactionConfig; private final CompactionCandidate candidateSegments; @@ -353,14 +354,14 @@ private static class Evaluator private final List fingerprintedSegments = new ArrayList<>(); private final List compactedSegments = new ArrayList<>(); - private final List uncompactedSegments = new ArrayList<>(); + final List uncompactedSegments = new ArrayList<>(); private final Map> unknownStateToSegments = new HashMap<>(); @Nullable private final String targetFingerprint; private final IndexingStateFingerprintMapper fingerprintMapper; - private Evaluator( + Evaluator( CompactionCandidate candidateSegments, DataSourceCompactionConfig compactionConfig, @Nullable String targetFingerprint, @@ -375,11 +376,11 @@ private Evaluator( this.fingerprintMapper = fingerprintMapper; } - private CompactionStatus evaluate() + Pair evaluate() { - final CompactionStatus inputBytesCheck = inputBytesAreWithinLimit(); - if (inputBytesCheck.isSkipped()) { - return inputBytesCheck; + final CompactionCandidateSearchPolicy.Eligibility inputBytesCheck = inputBytesAreWithinLimit(); + if (inputBytesCheck != null) { + return Pair.of(inputBytesCheck, CompactionStatus.skipped(inputBytesCheck.getReason())); } List reasonsForCompaction = new ArrayList<>(); @@ -411,7 +412,7 @@ private CompactionStatus evaluate() ); // Any segments left in unknownStateToSegments passed all checks and are considered compacted - this.compactedSegments.addAll( + compactedSegments.addAll( unknownStateToSegments .values() .stream() @@ -421,12 +422,18 @@ private CompactionStatus evaluate() } if (reasonsForCompaction.isEmpty()) { - return COMPLETE; + return Pair.of( + CompactionCandidateSearchPolicy.Eligibility.fail("All checks are passed, no reason to compact"), + CompactionStatus.COMPLETE + ); } else { - return CompactionStatus.pending( - createStats(this.compactedSegments), - createStats(uncompactedSegments), - reasonsForCompaction.get(0) + return Pair.of( + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + CompactionStatus.pending( + createStats(compactedSegments), + createStats(uncompactedSegments), + reasonsForCompaction.get(0) + ) ); } } @@ -464,9 +471,9 @@ private CompactionStatus allFingerprintedCandidatesHaveExpectedFingerprint() // Cannot evaluate further without a fingerprint mapper uncompactedSegments.addAll( mismatchedFingerprintToSegmentMap.values() - .stream() - .flatMap(List::stream) - .collect(Collectors.toList()) + .stream() + .flatMap(List::stream) + .collect(Collectors.toList()) ); return CompactionStatus.pending("Segments have a mismatched fingerprint and no fingerprint mapper is available"); } @@ -490,7 +497,8 @@ private CompactionStatus allFingerprintedCandidatesHaveExpectedFingerprint() } segments.addAll(e.getValue()); return segments; - }); + } + ); } } @@ -579,7 +587,8 @@ private CompactionStatus partitionsSpecIsUpToDate(CompactionState lastCompaction } else if (existingPartionsSpec instanceof DynamicPartitionsSpec) { existingPartionsSpec = new DynamicPartitionsSpec( existingPartionsSpec.getMaxRowsPerSegment(), - ((DynamicPartitionsSpec) existingPartionsSpec).getMaxTotalRowsOr(Long.MAX_VALUE)); + ((DynamicPartitionsSpec) existingPartionsSpec).getMaxTotalRowsOr(Long.MAX_VALUE) + ); } return CompactionStatus.completeIfNullOrEqual( "partitionsSpec", @@ -609,17 +618,17 @@ private CompactionStatus projectionsAreUpToDate(CompactionState lastCompactionSt ); } - private CompactionStatus inputBytesAreWithinLimit() + @Nullable + private CompactionCandidateSearchPolicy.Eligibility inputBytesAreWithinLimit() { final long inputSegmentSize = compactionConfig.getInputSegmentSizeBytes(); if (candidateSegments.getTotalBytes() > inputSegmentSize) { - return CompactionStatus.skipped( + return CompactionCandidateSearchPolicy.Eligibility.fail( "'inputSegmentSize' exceeded: Total segment size[%d] is larger than allowed inputSegmentSize[%d]", candidateSegments.getTotalBytes(), inputSegmentSize ); - } else { - return COMPLETE; } + return null; } private CompactionStatus segmentGranularityIsUpToDate(CompactionState lastCompactionState) diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusTracker.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusTracker.java index 1dc409e7361e..924d1300ea33 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusTracker.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusTracker.java @@ -80,10 +80,7 @@ public Set getSubmittedTaskIds() * This method assumes that the given candidate is eligible for compaction * based on the current compaction config/supervisor of the datasource. */ - public CompactionStatus computeCompactionStatus( - CompactionCandidate candidate, - CompactionCandidateSearchPolicy searchPolicy - ) + public CompactionStatus computeCompactionStatus(CompactionCandidate candidate) { // Skip intervals that already have a running task final CompactionTaskStatus lastTaskStatus = getLatestTaskStatus(candidate); @@ -101,14 +98,7 @@ public CompactionStatus computeCompactionStatus( ); } - // Skip intervals that have been filtered out by the policy - final CompactionCandidateSearchPolicy.Eligibility eligibility - = searchPolicy.checkEligibilityForCompaction(candidate, lastTaskStatus); - if (eligibility.isEligible()) { - return CompactionStatus.pending("Not compacted yet"); - } else { - return CompactionStatus.skipped("Rejected by search policy: %s", eligibility.getReason()); - } + return CompactionStatus.pending("Not compacted yet"); } /** diff --git a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java index 1994e87a6388..71125724b5a8 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java +++ b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java @@ -80,6 +80,7 @@ public class DataSourceCompactibleSegmentIterator implements CompactionSegmentIt // run of the compaction job and skip any interval that was already previously compacted. private final Set queuedIntervals = new HashSet<>(); + private final CompactionCandidateSearchPolicy searchPolicy; private final PriorityQueue queue; public DataSourceCompactibleSegmentIterator( @@ -92,6 +93,7 @@ public DataSourceCompactibleSegmentIterator( { this.config = config; this.dataSource = config.getDataSource(); + this.searchPolicy = searchPolicy; this.queue = new PriorityQueue<>(searchPolicy::compareCandidates); this.fingerprintMapper = indexingStateFingerprintMapper; @@ -329,17 +331,16 @@ private void findAndEnqueueSegmentsToCompact(CompactibleSegmentIterator compacti continue; } - final CompactionCandidate candidates = CompactionCandidate.from(segments, config.getSegmentGranularity()); - final CompactionStatus compactionStatus = CompactionStatus.compute(candidates, config, fingerprintMapper); - final CompactionCandidate candidatesWithStatus = candidates.withCurrentStatus(compactionStatus); + final CompactionCandidate candidatesWithStatus = + CompactionCandidate.from(segments, config.getSegmentGranularity()).evaluate(config, searchPolicy); - if (compactionStatus.isComplete()) { + if (candidatesWithStatus.getCurrentStatus().isComplete()) { compactedSegments.add(candidatesWithStatus); - } else if (compactionStatus.isSkipped()) { + } else if (candidatesWithStatus.getCurrentStatus().isSkipped()) { skippedSegments.add(candidatesWithStatus); - } else if (!queuedIntervals.contains(candidates.getUmbrellaInterval())) { + } else if (!queuedIntervals.contains(candidatesWithStatus.getUmbrellaInterval())) { queue.add(candidatesWithStatus); - queuedIntervals.add(candidates.getUmbrellaInterval()); + queuedIntervals.add(candidatesWithStatus.getUmbrellaInterval()); } } } diff --git a/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java index 24a2f001afe3..2974f57881cd 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java @@ -62,7 +62,7 @@ public Eligibility checkEligibilityForCompaction( ) { return findIndex(candidate) < Integer.MAX_VALUE - ? Eligibility.OK + ? Eligibility.FULL_COMPACTION_OK : Eligibility.fail("Datasource/Interval is not in the list of 'eligibleCandidates'"); } diff --git a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java index 345988ee7fc2..41c946c87eda 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java @@ -47,6 +47,7 @@ public class MostFragmentedIntervalFirstPolicy extends BaseCandidateSearchPolicy private final int minUncompactedCount; private final HumanReadableBytes minUncompactedBytes; private final HumanReadableBytes maxAverageUncompactedBytesPerSegment; + private final double incrementalCompactionUncompactedBytesRatioThreshold; @JsonCreator public MostFragmentedIntervalFirstPolicy( @@ -54,6 +55,8 @@ public MostFragmentedIntervalFirstPolicy( @JsonProperty("minUncompactedBytes") @Nullable HumanReadableBytes minUncompactedBytes, @JsonProperty("maxAverageUncompactedBytesPerSegment") @Nullable HumanReadableBytes maxAverageUncompactedBytesPerSegment, + @JsonProperty("incrementalCompactionUncompactedBytesRatioThreshold") @Nullable + Double incrementalCompactionUncompactedBytesRatioThreshold, @JsonProperty("priorityDatasource") @Nullable String priorityDatasource ) { @@ -69,11 +72,19 @@ public MostFragmentedIntervalFirstPolicy( "'minUncompactedCount'[%s] must be greater than 0", maxAverageUncompactedBytesPerSegment ); + InvalidInput.conditionalException( + incrementalCompactionUncompactedBytesRatioThreshold == null + || incrementalCompactionUncompactedBytesRatioThreshold > 0, + "'incrementalCompactionUncompactedBytesRatioThreshold'[%s] must be greater than 0", + incrementalCompactionUncompactedBytesRatioThreshold + ); this.minUncompactedCount = Configs.valueOrDefault(minUncompactedCount, 100); this.minUncompactedBytes = Configs.valueOrDefault(minUncompactedBytes, SIZE_10_MB); this.maxAverageUncompactedBytesPerSegment = Configs.valueOrDefault(maxAverageUncompactedBytesPerSegment, SIZE_2_GB); + this.incrementalCompactionUncompactedBytesRatioThreshold = + Configs.valueOrDefault(incrementalCompactionUncompactedBytesRatioThreshold, 0.0d); } /** @@ -106,6 +117,17 @@ public HumanReadableBytes getMaxAverageUncompactedBytesPerSegment() return maxAverageUncompactedBytesPerSegment; } + /** + * Threshold ratio of uncompacted bytes to compacted bytes below which + * incremental compaction is eligible instead of full compaction. + * Default value is 0.0. + */ + @JsonProperty + public Double getIncrementalCompactionUncompactedRatioThreshold() + { + return incrementalCompactionUncompactedBytesRatioThreshold; + } + @Override protected Comparator getSegmentComparator() { @@ -123,6 +145,10 @@ public boolean equals(Object o) } MostFragmentedIntervalFirstPolicy policy = (MostFragmentedIntervalFirstPolicy) o; return minUncompactedCount == policy.minUncompactedCount + && Double.compare( + incrementalCompactionUncompactedBytesRatioThreshold, + policy.incrementalCompactionUncompactedBytesRatioThreshold + ) == 0 && Objects.equals(minUncompactedBytes, policy.minUncompactedBytes) && Objects.equals(maxAverageUncompactedBytesPerSegment, policy.maxAverageUncompactedBytesPerSegment); } @@ -134,19 +160,22 @@ public int hashCode() super.hashCode(), minUncompactedCount, minUncompactedBytes, - maxAverageUncompactedBytesPerSegment + maxAverageUncompactedBytesPerSegment, + incrementalCompactionUncompactedBytesRatioThreshold ); } @Override public String toString() { - return "MostFragmentedIntervalFirstPolicy{" + - "minUncompactedCount=" + minUncompactedCount + - ", minUncompactedBytes=" + minUncompactedBytes + - ", maxAverageUncompactedBytesPerSegment=" + maxAverageUncompactedBytesPerSegment + - ", priorityDataSource='" + getPriorityDatasource() + '\'' + - '}'; + return + "MostFragmentedIntervalFirstPolicy{" + + "minUncompactedCount=" + minUncompactedCount + + ", minUncompactedBytes=" + minUncompactedBytes + + ", maxAverageUncompactedBytesPerSegment=" + maxAverageUncompactedBytesPerSegment + + ", incrementalCompactionUncompactedBytesRatioThreshold=" + incrementalCompactionUncompactedBytesRatioThreshold + + ", priorityDataSource='" + getPriorityDatasource() + '\'' + + '}'; } private int compare(CompactionCandidate candidateA, CompactionCandidate candidateB) @@ -164,7 +193,7 @@ public Eligibility checkEligibilityForCompaction( { final CompactionStatistics uncompacted = candidate.getUncompactedStats(); if (uncompacted == null) { - return Eligibility.OK; + return Eligibility.FULL_COMPACTION_OK; } else if (uncompacted.getNumSegments() < 1) { return Eligibility.fail("No uncompacted segments in interval"); } else if (uncompacted.getNumSegments() < minUncompactedCount) { @@ -185,8 +214,18 @@ public Eligibility checkEligibilityForCompaction( "Average size[%,d] of uncompacted segments in interval must be at most [%,d]", avgSegmentSize, maxAverageUncompactedBytesPerSegment.getBytes() ); + } + + final double uncompactedBytesRatio = (double) uncompacted.getTotalBytes() / + (uncompacted.getTotalBytes() + candidate.getCompactedStats().getTotalBytes()); + if (uncompactedBytesRatio < incrementalCompactionUncompactedBytesRatioThreshold) { + return Eligibility.incrementalCompaction( + "Uncompacted bytes ratio[%.2f] is below threshold[%.2f]", + uncompactedBytesRatio, + incrementalCompactionUncompactedBytesRatioThreshold + ); } else { - return Eligibility.OK; + return Eligibility.FULL_COMPACTION_OK; } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java index 060c16edfadb..594bcdb53b83 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java @@ -34,6 +34,7 @@ import org.apache.druid.common.guava.FutureUtils; import org.apache.druid.common.utils.IdUtils; import org.apache.druid.data.input.impl.AggregateProjectionSpec; +import org.apache.druid.error.DruidException; import org.apache.druid.indexer.CompactionEngine; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.granularity.GranularityType; @@ -244,8 +245,7 @@ private int submitCompactionTasks( final String dataSourceName = entry.getDataSource(); final DataSourceCompactionConfig config = compactionConfigs.get(dataSourceName); - final CompactionStatus compactionStatus = - statusTracker.computeCompactionStatus(entry, policy); + final CompactionStatus compactionStatus = statusTracker.computeCompactionStatus(entry); final CompactionCandidate candidatesWithStatus = entry.withCurrentStatus(compactionStatus); statusTracker.onCompactionStatusComputed(candidatesWithStatus, config); @@ -366,6 +366,10 @@ public static ClientCompactionTaskQuery createCompactionTask( } final CompactionEngine compactionEngine = config.getEngine() == null ? defaultEngine : config.getEngine(); + if (CompactionEngine.MSQ.equals(compactionEngine) && !Boolean.TRUE.equals(dropExisting)) { + LOG.info("Forcing dropExisting to true as required by MSQ engine."); + dropExisting = true; + } final Map autoCompactionContext = newAutoCompactionContext(config.getTaskContext()); if (candidate.getCurrentStatus() != null) { @@ -460,14 +464,27 @@ private static ClientCompactionTaskQuery compactSegments( context.put("priority", compactionTaskPriority); final String taskId = IdUtils.newTaskId(TASK_ID_PREFIX, ClientCompactionTaskQuery.TYPE, dataSource, null); + final ClientCompactionIntervalSpec clientCompactionIntervalSpec; + Preconditions.checkArgument(entry.getPolicyEligibility() != null, "Must have a policy eligibility"); + switch (entry.getPolicyEligibility().getPolicyEligibility()) { + case FULL_COMPACTION: + clientCompactionIntervalSpec = new ClientCompactionIntervalSpec(entry.getCompactionInterval(), null, null); + break; + case INCREMENTAL_COMPACTION: + clientCompactionIntervalSpec = new ClientCompactionIntervalSpec( + entry.getCompactionInterval(), + entry.getSegments().stream().map(DataSegment::toDescriptor).collect(Collectors.toList()), + null + ); + break; + default: + throw DruidException.defensive("Unexpected policy eligibility[%s]", entry.getPolicyEligibility()); + } return new ClientCompactionTaskQuery( taskId, dataSource, - new ClientCompactionIOConfig( - new ClientCompactionIntervalSpec(entry.getCompactionInterval(), null), - dropExisting - ), + new ClientCompactionIOConfig(clientCompactionIntervalSpec, dropExisting), tuningConfig, granularitySpec, dimensionsSpec, diff --git a/server/src/test/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpecTest.java b/server/src/test/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpecTest.java index 46ecc64d72d1..096a82c8c158 100644 --- a/server/src/test/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpecTest.java +++ b/server/src/test/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpecTest.java @@ -19,19 +19,25 @@ package org.apache.druid.client.indexing; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; +import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.segment.IndexIO; import org.apache.druid.server.compaction.CompactionCandidate; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.NoneShardSpec; +import org.joda.time.Interval; import org.junit.Assert; import org.junit.Test; import java.util.ArrayList; import java.util.HashMap; +import java.util.List; public class ClientCompactionIntervalSpecTest { @@ -112,4 +118,40 @@ public void testFromSegmentWithFinerSegmentGranularityAndUmbrellaIntervalNotAlig // Hence the compaction interval is modified to aling with the segmentGranularity Assert.assertEquals(Intervals.of("2015-02-09/2015-04-20"), actual.getCompactionInterval()); } + + @Test + public void testClientCompactionIntervalSpec_throwsException_whenEmptySegmentsList() + { + Interval interval = Intervals.of("2015-04-11/2015-04-12"); + List emptySegments = List.of(); + + Assert.assertThrows( + IAE.class, + () -> new ClientCompactionIntervalSpec(interval, emptySegments, null) + ); + } + + @Test + public void testClientCompactionIntervalSpec_serde() throws Exception + { + ObjectMapper mapper = new DefaultObjectMapper(); + Interval interval = Intervals.of("2015-04-11/2015-04-12"); + List segments = List.of( + new SegmentDescriptor(Intervals.of("2015-04-11/2015-04-12"), "v1", 0) + ); + + // Test with uncompactedSegments (incremental compaction) + ClientCompactionIntervalSpec withSegments = new ClientCompactionIntervalSpec(interval, segments, "sha256hash"); + String json1 = mapper.writeValueAsString(withSegments); + ClientCompactionIntervalSpec deserialized1 = mapper.readValue(json1, ClientCompactionIntervalSpec.class); + Assert.assertEquals(withSegments, deserialized1); + Assert.assertEquals(segments, deserialized1.getUncompactedSegments()); + + // Test without uncompactedSegments (full compaction) + ClientCompactionIntervalSpec withoutSegments = new ClientCompactionIntervalSpec(interval, null, null); + String json2 = mapper.writeValueAsString(withoutSegments); + ClientCompactionIntervalSpec deserialized2 = mapper.readValue(json2, ClientCompactionIntervalSpec.class); + Assert.assertEquals(withoutSegments, deserialized2); + Assert.assertNull(deserialized2.getUncompactedSegments()); + } } diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java index 1314a1a0bc79..b9bd4acf4a17 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java @@ -110,7 +110,7 @@ public void testComputeCompactionStatusForSuccessfulTask() // Verify that interval is originally eligible for compaction CompactionStatus status - = statusTracker.computeCompactionStatus(candidateSegments, policy); + = statusTracker.computeCompactionStatus(candidateSegments); Assert.assertEquals(CompactionStatus.State.PENDING, status.getState()); Assert.assertEquals("Not compacted yet", status.getReason()); @@ -119,7 +119,7 @@ public void testComputeCompactionStatusForSuccessfulTask() statusTracker.onTaskSubmitted("task1", candidateSegments); statusTracker.onTaskFinished("task1", TaskStatus.success("task1")); - status = statusTracker.computeCompactionStatus(candidateSegments, policy); + status = statusTracker.computeCompactionStatus(candidateSegments); Assert.assertEquals(CompactionStatus.State.SKIPPED, status.getState()); Assert.assertEquals( "Segment timeline not updated since last compaction task succeeded", @@ -128,7 +128,7 @@ public void testComputeCompactionStatusForSuccessfulTask() // Verify that interval becomes eligible again after timeline has been updated statusTracker.onSegmentTimelineUpdated(DateTimes.nowUtc()); - status = statusTracker.computeCompactionStatus(candidateSegments, policy); + status = statusTracker.computeCompactionStatus(candidateSegments); Assert.assertEquals(CompactionStatus.State.PENDING, status.getState()); } } diff --git a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java index 1b93bfa03a55..561986ba75eb 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java @@ -39,7 +39,8 @@ public class MostFragmentedIntervalFirstPolicyTest @Test public void test_thresholdValues_ofDefaultPolicy() { - final MostFragmentedIntervalFirstPolicy policy = new MostFragmentedIntervalFirstPolicy(null, null, null, null); + final MostFragmentedIntervalFirstPolicy policy = + new MostFragmentedIntervalFirstPolicy(null, null, null, null, null); Assertions.assertEquals(100, policy.getMinUncompactedCount()); Assertions.assertEquals(new HumanReadableBytes("10MiB"), policy.getMinUncompactedBytes()); Assertions.assertEquals(new HumanReadableBytes("2GiB"), policy.getMaxAverageUncompactedBytesPerSegment()); @@ -54,6 +55,7 @@ public void test_checkEligibilityForCompaction_fails_ifUncompactedCountLessThanC minUncompactedCount, HumanReadableBytes.valueOf(1), HumanReadableBytes.valueOf(10_000), + null, null ); @@ -64,7 +66,7 @@ public void test_checkEligibilityForCompaction_fails_ifUncompactedCountLessThanC policy.checkEligibilityForCompaction(createCandidate(1, 100L), null) ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.OK, + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, policy.checkEligibilityForCompaction(createCandidate(10_001, 100L), null) ); } @@ -77,6 +79,7 @@ public void test_checkEligibilityForCompaction_fails_ifUncompactedBytesLessThanC 1, minUncompactedBytes, HumanReadableBytes.valueOf(10_000), + null, null ); @@ -87,7 +90,7 @@ public void test_checkEligibilityForCompaction_fails_ifUncompactedBytesLessThanC policy.checkEligibilityForCompaction(createCandidate(1, 100L), null) ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.OK, + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, policy.checkEligibilityForCompaction(createCandidate(100, 10_000L), null) ); } @@ -100,6 +103,7 @@ public void test_checkEligibilityForCompaction_fails_ifAvgSegmentSizeGreaterThan 1, HumanReadableBytes.valueOf(100), maxAvgSegmentSize, + null, null ); @@ -110,7 +114,7 @@ public void test_checkEligibilityForCompaction_fails_ifAvgSegmentSizeGreaterThan policy.checkEligibilityForCompaction(createCandidate(1, 10_000L), null) ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.OK, + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, policy.checkEligibilityForCompaction(createCandidate(1, 100L), null) ); } @@ -122,14 +126,21 @@ public void test_policy_favorsIntervalWithMoreUncompactedSegments_ifTotalBytesIs 1, HumanReadableBytes.valueOf(1), HumanReadableBytes.valueOf(10_000), + null, null ); final CompactionCandidate candidateA = createCandidate(1, 1000L); final CompactionCandidate candidateB = createCandidate(2, 500L); - verifyCandidateIsEligible(candidateA, policy); - verifyCandidateIsEligible(candidateB, policy); + Assertions.assertEquals( + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + policy.checkEligibilityForCompaction(candidateA, null) + ); + Assertions.assertEquals( + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + policy.checkEligibilityForCompaction(candidateB, null) + ); Assertions.assertTrue(policy.compareCandidates(candidateA, candidateB) > 0); Assertions.assertTrue(policy.compareCandidates(candidateB, candidateA) < 0); @@ -142,14 +153,21 @@ public void test_policy_favorsIntervalWithMoreUncompactedSegments_ifAverageSizeI 1, HumanReadableBytes.valueOf(1), HumanReadableBytes.valueOf(10_000), + null, null ); final CompactionCandidate candidateA = createCandidate(1, 1000L); final CompactionCandidate candidateB = createCandidate(2, 1000L); - verifyCandidateIsEligible(candidateA, policy); - verifyCandidateIsEligible(candidateB, policy); + Assertions.assertEquals( + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + policy.checkEligibilityForCompaction(candidateA, null) + ); + Assertions.assertEquals( + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + policy.checkEligibilityForCompaction(candidateB, null) + ); Assertions.assertTrue(policy.compareCandidates(candidateA, candidateB) > 0); Assertions.assertTrue(policy.compareCandidates(candidateB, candidateA) < 0); @@ -162,14 +180,21 @@ public void test_policy_favorsIntervalWithSmallerSegments_ifCountIsEqual() 1, HumanReadableBytes.valueOf(1), HumanReadableBytes.valueOf(10_000), + null, null ); final CompactionCandidate candidateA = createCandidate(10, 500L); final CompactionCandidate candidateB = createCandidate(10, 1000L); - verifyCandidateIsEligible(candidateA, policy); - verifyCandidateIsEligible(candidateB, policy); + Assertions.assertEquals( + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + policy.checkEligibilityForCompaction(candidateA, null) + ); + Assertions.assertEquals( + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + policy.checkEligibilityForCompaction(candidateB, null) + ); Assertions.assertTrue(policy.compareCandidates(candidateA, candidateB) < 0); Assertions.assertTrue(policy.compareCandidates(candidateB, candidateA) > 0); @@ -182,14 +207,21 @@ public void test_compareCandidates_returnsZeroIfSegmentCountAndAvgSizeScaleEquiv 100, HumanReadableBytes.valueOf(1), HumanReadableBytes.valueOf(100), + null, null ); final CompactionCandidate candidateA = createCandidate(100, 25); final CompactionCandidate candidateB = createCandidate(400, 100); - verifyCandidateIsEligible(candidateA, policy); - verifyCandidateIsEligible(candidateB, policy); + Assertions.assertEquals( + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + policy.checkEligibilityForCompaction(candidateA, null) + ); + Assertions.assertEquals( + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + policy.checkEligibilityForCompaction(candidateB, null) + ); Assertions.assertEquals(0, policy.compareCandidates(candidateA, candidateB)); Assertions.assertEquals(0, policy.compareCandidates(candidateB, candidateA)); @@ -211,6 +243,7 @@ public void test_serde_allFieldsSet() throws IOException 1, HumanReadableBytes.valueOf(2), HumanReadableBytes.valueOf(3), + null, "foo" ); final DefaultObjectMapper mapper = new DefaultObjectMapper(); @@ -222,13 +255,76 @@ public void test_serde_allFieldsSet() throws IOException @Test public void test_serde_noFieldsSet() throws IOException { - final MostFragmentedIntervalFirstPolicy policy = new MostFragmentedIntervalFirstPolicy(null, null, null, null); + final MostFragmentedIntervalFirstPolicy policy = + new MostFragmentedIntervalFirstPolicy(null, null, null, null, null); final DefaultObjectMapper mapper = new DefaultObjectMapper(); final CompactionCandidateSearchPolicy policy2 = mapper.readValue(mapper.writeValueAsString(policy), CompactionCandidateSearchPolicy.class); Assertions.assertEquals(policy, policy2); } + @Test + public void test_checkEligibilityForCompaction_returnsIncrementalCompaction_whenRatioBelowThreshold() + { + // Set threshold to 0.5 (50%) + final MostFragmentedIntervalFirstPolicy policy = new MostFragmentedIntervalFirstPolicy( + 1, + HumanReadableBytes.valueOf(1), + HumanReadableBytes.valueOf(10_000), + 0.5, + null + ); + + final CompactionCandidate candidate = createCandidateWithStats(1200L, 400L, 100); + + Assertions.assertEquals( + CompactionCandidateSearchPolicy.Eligibility.incrementalCompaction( + "Uncompacted bytes ratio[0.25] is below threshold[0.50]"), + policy.checkEligibilityForCompaction(candidate, null) + ); + } + + @Test + public void test_checkEligibilityForCompaction_returnsFullCompaction_whenRatioAboveThreshold() + { + // Set threshold to 0.5 (50%) + final MostFragmentedIntervalFirstPolicy policy = new MostFragmentedIntervalFirstPolicy( + 1, + HumanReadableBytes.valueOf(1), + HumanReadableBytes.valueOf(10_000), + 0.5, + null + ); + + final CompactionCandidate candidate = createCandidateWithStats(500L, 600L, 100); + + Assertions.assertEquals( + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + policy.checkEligibilityForCompaction(candidate, null) + ); + } + + @Test + public void test_checkEligibilityForCompaction_returnsFullCompaction_whenThresholdIsDefault() + { + // Default threshold is 0.0 + final MostFragmentedIntervalFirstPolicy policy = new MostFragmentedIntervalFirstPolicy( + 1, + HumanReadableBytes.valueOf(1), + HumanReadableBytes.valueOf(10_000), + null, + null + ); + + // With default threshold 0.0, any positive ratio >= 0.0, so always FULL_COMPACTION_OK + final CompactionCandidate candidate = createCandidateWithStats(1000L, 100L, 100); + + Assertions.assertEquals( + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + policy.checkEligibilityForCompaction(candidate, null) + ); + } + private CompactionCandidate createCandidate(int numSegments, long avgSizeBytes) { final CompactionStatistics dummyCompactedStats = CompactionStatistics.create(1L, 1L, 1L); @@ -241,11 +337,20 @@ private CompactionCandidate createCandidate(int numSegments, long avgSizeBytes) .withCurrentStatus(CompactionStatus.pending(dummyCompactedStats, uncompactedStats, "")); } - private void verifyCandidateIsEligible(CompactionCandidate candidate, MostFragmentedIntervalFirstPolicy policy) + private CompactionCandidate createCandidateWithStats( + long compactedBytes, + long uncompactedBytes, + int uncompactedSegments + ) { - Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.OK, - policy.checkEligibilityForCompaction(candidate, null) + final CompactionStatistics compactedStats = CompactionStatistics.create(compactedBytes, 1L, 1L); + final CompactionStatistics uncompactedStats = CompactionStatistics.create( + uncompactedBytes, + uncompactedSegments, + 1L ); + return CompactionCandidate.from(List.of(SEGMENT), null) + .withCurrentStatus(CompactionStatus.pending(compactedStats, uncompactedStats, "")); } + } diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java index cd92e8f1999a..d03bf7f27350 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java @@ -1098,6 +1098,7 @@ public void testCompactWithGranularitySpecConflictWithActiveCompactionTask() new ClientCompactionIOConfig( new ClientCompactionIntervalSpec( Intervals.of("2000/2099"), + null, "testSha256OfSortedSegmentIds" ), null From 0a4a0dfac668d0768e4fe2c0e3bd7e5d178fb0b0 Mon Sep 17 00:00:00 2001 From: cecemei Date: Sun, 1 Feb 2026 13:05:19 -0800 Subject: [PATCH 02/19] test --- .../compaction/BaseCandidateSearchPolicy.java | 2 +- .../compaction/CompactionCandidate.java | 21 +- .../CompactionCandidateSearchPolicy.java | 6 +- .../server/compaction/CompactionStatus.java | 30 +- .../DataSourceCompactibleSegmentIterator.java | 6 +- .../compaction/FixedIntervalOrderPolicy.java | 2 +- .../MostFragmentedIntervalFirstPolicy.java | 4 +- .../CompactionRunSimulatorTest.java | 14 +- ...MostFragmentedIntervalFirstPolicyTest.java | 28 +- .../coordinator/duty/CompactSegmentsTest.java | 279 +++++++++--------- 10 files changed, 207 insertions(+), 185 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java index c707929b8f72..e0fce9c85a7f 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java @@ -73,7 +73,7 @@ public Eligibility checkEligibilityForCompaction( CompactionTaskStatus latestTaskStatus ) { - return Eligibility.FULL_COMPACTION_OK; + return Eligibility.FULL_COMPACTION_ELIGIBLE; } /** diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java index cf11eaf31703..05e3d6b67c48 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java @@ -25,6 +25,7 @@ import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.segment.SegmentUtils; +import org.apache.druid.segment.metadata.IndexingStateFingerprintMapper; import org.apache.druid.server.coordinator.DataSourceCompactionConfig; import org.apache.druid.timeline.DataSegment; import org.joda.time.Interval; @@ -223,15 +224,19 @@ public CompactionCandidate withPolicyEligibility(CompactionCandidateSearchPolicy * @param searchPolicy the policy used to determine compaction eligibility * @return a CompactionCandidate with updated status and potentially filtered segments */ - public CompactionCandidate evaluate(DataSourceCompactionConfig config, CompactionCandidateSearchPolicy searchPolicy) + public CompactionCandidate evaluate( + DataSourceCompactionConfig config, + CompactionCandidateSearchPolicy searchPolicy, + IndexingStateFingerprintMapper fingerprintMapper + ) { - CompactionStatus.Evaluator evaluator = new CompactionStatus.Evaluator(this, config, null, null); + CompactionStatus.Evaluator evaluator = new CompactionStatus.Evaluator(this, config, fingerprintMapper); Pair evaluated = evaluator.evaluate(); switch (Objects.requireNonNull(evaluated.lhs).getPolicyEligibility()) { - case NOT_ELIGIBLE: // failed evaluator check - return this.withPolicyEligibility(evaluated.lhs) - .withCurrentStatus(CompactionStatus.skipped("Rejected[%s]", evaluated.lhs.getReason())); - case FULL_COMPACTION: + case NOT_APPLICABLE: + case NOT_ELIGIBLE: + return this.withPolicyEligibility(evaluated.lhs).withCurrentStatus(evaluated.rhs); + case FULL_COMPACTION: // evaluator has decided compaction is needed, policy needs to further check final CompactionCandidateSearchPolicy.Eligibility searchPolicyEligibility = searchPolicy.checkEligibilityForCompaction(this, null); switch (searchPolicyEligibility.getPolicyEligibility()) { @@ -255,9 +260,9 @@ public CompactionCandidate evaluate(DataSourceCompactionConfig config, Compactio evaluated.rhs ); default: - throw DruidException.defensive("Unexpected eligibility[%s]", searchPolicyEligibility); + throw DruidException.defensive("Unexpected eligibility[%s] from policy", searchPolicyEligibility); } - case INCREMENTAL_COMPACTION: + case INCREMENTAL_COMPACTION: // evaluator cant decide when to perform an incremental compaction default: throw DruidException.defensive("Unexpected eligibility[%s]", evaluated.rhs); } diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java index 4d07d3bb182e..558089203c93 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java @@ -66,10 +66,12 @@ public enum PolicyEligibility { FULL_COMPACTION, INCREMENTAL_COMPACTION, - NOT_ELIGIBLE + NOT_ELIGIBLE, + NOT_APPLICABLE } - public static final Eligibility FULL_COMPACTION_OK = new Eligibility(PolicyEligibility.FULL_COMPACTION, null); + public static final Eligibility FULL_COMPACTION_ELIGIBLE = new Eligibility(PolicyEligibility.FULL_COMPACTION, null); + public static final Eligibility NOT_APPLICABLE = new Eligibility(PolicyEligibility.NOT_APPLICABLE, null); private final PolicyEligibility eligible; private final String reason; diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java index 0a10aa9b8554..1a9800bf727c 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java @@ -264,17 +264,7 @@ static CompactionStatus compute( @Nullable IndexingStateFingerprintMapper fingerprintMapper ) { - final CompactionState expectedState = config.toCompactionState(); - String expectedFingerprint; - if (fingerprintMapper == null) { - expectedFingerprint = null; - } else { - expectedFingerprint = fingerprintMapper.generateFingerprint( - config.getDataSource(), - expectedState - ); - } - return new Evaluator(candidateSegments, config, expectedFingerprint, fingerprintMapper).evaluate().rhs; + return new Evaluator(candidateSegments, config, fingerprintMapper).evaluate().rhs; } @Nullable @@ -358,13 +348,13 @@ static class Evaluator private final Map> unknownStateToSegments = new HashMap<>(); @Nullable - private final String targetFingerprint; private final IndexingStateFingerprintMapper fingerprintMapper; + @Nullable + private final String targetFingerprint; Evaluator( CompactionCandidate candidateSegments, DataSourceCompactionConfig compactionConfig, - @Nullable String targetFingerprint, @Nullable IndexingStateFingerprintMapper fingerprintMapper ) { @@ -372,8 +362,15 @@ static class Evaluator this.compactionConfig = compactionConfig; this.tuningConfig = ClientCompactionTaskQueryTuningConfig.from(compactionConfig); this.configuredGranularitySpec = compactionConfig.getGranularitySpec(); - this.targetFingerprint = targetFingerprint; this.fingerprintMapper = fingerprintMapper; + if (fingerprintMapper == null) { + targetFingerprint = null; + } else { + targetFingerprint = fingerprintMapper.generateFingerprint( + compactionConfig.getDataSource(), + compactionConfig.toCompactionState() + ); + } } Pair evaluate() @@ -423,12 +420,13 @@ Pair evaluate() if (reasonsForCompaction.isEmpty()) { return Pair.of( - CompactionCandidateSearchPolicy.Eligibility.fail("All checks are passed, no reason to compact"), + CompactionCandidateSearchPolicy.Eligibility.NOT_APPLICABLE, + //fail("All checks are passed, no reason to compact"), CompactionStatus.COMPLETE ); } else { return Pair.of( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, CompactionStatus.pending( createStats(compactedSegments), createStats(uncompactedSegments), diff --git a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java index 71125724b5a8..032f9ccdea6a 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java +++ b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java @@ -332,7 +332,8 @@ private void findAndEnqueueSegmentsToCompact(CompactibleSegmentIterator compacti } final CompactionCandidate candidatesWithStatus = - CompactionCandidate.from(segments, config.getSegmentGranularity()).evaluate(config, searchPolicy); + CompactionCandidate.from(segments, config.getSegmentGranularity()) + .evaluate(config, searchPolicy, fingerprintMapper); if (candidatesWithStatus.getCurrentStatus().isComplete()) { compactedSegments.add(candidatesWithStatus); @@ -437,7 +438,8 @@ static Interval computeLatestSkipInterval( if (configuredSegmentGranularity == null) { return new Interval(skipOffsetFromLatest, latestDataTimestamp); } else { - DateTime skipFromLastest = new DateTime(latestDataTimestamp, latestDataTimestamp.getZone()).minus(skipOffsetFromLatest); + DateTime skipFromLastest = new DateTime(latestDataTimestamp, latestDataTimestamp.getZone()).minus( + skipOffsetFromLatest); DateTime skipOffsetBucketToSegmentGranularity = configuredSegmentGranularity.bucketStart(skipFromLastest); return new Interval(skipOffsetBucketToSegmentGranularity, latestDataTimestamp); } diff --git a/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java index 2974f57881cd..afe6a52bea48 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java @@ -62,7 +62,7 @@ public Eligibility checkEligibilityForCompaction( ) { return findIndex(candidate) < Integer.MAX_VALUE - ? Eligibility.FULL_COMPACTION_OK + ? Eligibility.FULL_COMPACTION_ELIGIBLE : Eligibility.fail("Datasource/Interval is not in the list of 'eligibleCandidates'"); } diff --git a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java index 41c946c87eda..6086ff84c7fd 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java @@ -193,7 +193,7 @@ public Eligibility checkEligibilityForCompaction( { final CompactionStatistics uncompacted = candidate.getUncompactedStats(); if (uncompacted == null) { - return Eligibility.FULL_COMPACTION_OK; + return Eligibility.FULL_COMPACTION_ELIGIBLE; } else if (uncompacted.getNumSegments() < 1) { return Eligibility.fail("No uncompacted segments in interval"); } else if (uncompacted.getNumSegments() < minUncompactedCount) { @@ -225,7 +225,7 @@ public Eligibility checkEligibilityForCompaction( incrementalCompactionUncompactedBytesRatioThreshold ); } else { - return Eligibility.FULL_COMPACTION_OK; + return Eligibility.FULL_COMPACTION_ELIGIBLE; } } diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java index 56ec8525bb83..2854520b01f6 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java @@ -181,15 +181,15 @@ public void testSimulate_withFixedIntervalOrderPolicy() = "Rejected by search policy: Datasource/Interval is not in the list of 'eligibleCandidates'"; Assert.assertEquals( List.of( - List.of("wiki", Intervals.of("2013-01-02/P1D"), 10, 1_000_000_000L, 1, rejectedMessage), - List.of("wiki", Intervals.of("2013-01-03/P1D"), 10, 1_000_000_000L, 1, rejectedMessage), + List.of("wiki", Intervals.of("2013-01-10/P1D"), 10, 1_000_000_000L, 1, "skip offset from latest[P1D]"), + List.of("wiki", Intervals.of("2013-01-09/P1D"), 10, 1_000_000_000L, 1, rejectedMessage), List.of("wiki", Intervals.of("2013-01-07/P1D"), 10, 1_000_000_000L, 1, rejectedMessage), - List.of("wiki", Intervals.of("2013-01-05/P1D"), 10, 1_000_000_000L, 1, rejectedMessage), List.of("wiki", Intervals.of("2013-01-06/P1D"), 10, 1_000_000_000L, 1, rejectedMessage), - List.of("wiki", Intervals.of("2013-01-01/P1D"), 10, 1_000_000_000L, 1, rejectedMessage), - List.of("wiki", Intervals.of("2013-01-09/P1D"), 10, 1_000_000_000L, 1, rejectedMessage), - List.of("wiki", Intervals.of("2013-01-10/P1D"), 10, 1_000_000_000L, 1, "skip offset from latest[P1D]") - ), + List.of("wiki", Intervals.of("2013-01-05/P1D"), 10, 1_000_000_000L, 1, rejectedMessage), + List.of("wiki", Intervals.of("2013-01-03/P1D"), 10, 1_000_000_000L, 1, rejectedMessage), + List.of("wiki", Intervals.of("2013-01-02/P1D"), 10, 1_000_000_000L, 1, rejectedMessage), + List.of("wiki", Intervals.of("2013-01-01/P1D"), 10, 1_000_000_000L, 1, rejectedMessage) + ), skippedTable.getRows() ); } diff --git a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java index 561986ba75eb..76ea2008206a 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java @@ -66,7 +66,7 @@ public void test_checkEligibilityForCompaction_fails_ifUncompactedCountLessThanC policy.checkEligibilityForCompaction(createCandidate(1, 100L), null) ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(createCandidate(10_001, 100L), null) ); } @@ -90,7 +90,7 @@ public void test_checkEligibilityForCompaction_fails_ifUncompactedBytesLessThanC policy.checkEligibilityForCompaction(createCandidate(1, 100L), null) ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(createCandidate(100, 10_000L), null) ); } @@ -114,7 +114,7 @@ public void test_checkEligibilityForCompaction_fails_ifAvgSegmentSizeGreaterThan policy.checkEligibilityForCompaction(createCandidate(1, 10_000L), null) ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(createCandidate(1, 100L), null) ); } @@ -134,11 +134,11 @@ public void test_policy_favorsIntervalWithMoreUncompactedSegments_ifTotalBytesIs final CompactionCandidate candidateB = createCandidate(2, 500L); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidateA, null) ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidateB, null) ); @@ -161,11 +161,11 @@ public void test_policy_favorsIntervalWithMoreUncompactedSegments_ifAverageSizeI final CompactionCandidate candidateB = createCandidate(2, 1000L); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidateA, null) ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidateB, null) ); @@ -188,11 +188,11 @@ public void test_policy_favorsIntervalWithSmallerSegments_ifCountIsEqual() final CompactionCandidate candidateB = createCandidate(10, 1000L); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidateA, null) ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidateB, null) ); @@ -215,11 +215,11 @@ public void test_compareCandidates_returnsZeroIfSegmentCountAndAvgSizeScaleEquiv final CompactionCandidate candidateB = createCandidate(400, 100); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidateA, null) ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidateB, null) ); @@ -299,7 +299,7 @@ public void test_checkEligibilityForCompaction_returnsFullCompaction_whenRatioAb final CompactionCandidate candidate = createCandidateWithStats(500L, 600L, 100); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidate, null) ); } @@ -316,11 +316,11 @@ public void test_checkEligibilityForCompaction_returnsFullCompaction_whenThresho null ); - // With default threshold 0.0, any positive ratio >= 0.0, so always FULL_COMPACTION_OK + // With default threshold 0.0, any positive ratio >= 0.0, so always FULL_COMPACTION_ELIGIBLE final CompactionCandidate candidate = createCandidateWithStats(1000L, 100L, 100); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_OK, + CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidate, null) ); } diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java index d03bf7f27350..83f838f9517d 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java @@ -78,7 +78,6 @@ import org.apache.druid.rpc.indexing.OverlordClient; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.incremental.OnheapIncrementalIndex; -import org.apache.druid.segment.indexing.BatchIOConfig; import org.apache.druid.segment.transform.CompactionTransformSpec; import org.apache.druid.server.compaction.CompactionCandidate; import org.apache.druid.server.compaction.CompactionCandidateSearchPolicy; @@ -101,6 +100,7 @@ import org.apache.druid.server.coordinator.stats.Stats; import org.apache.druid.timeline.CompactionState; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.SegmentTimeline; import org.apache.druid.timeline.TimelineObjectHolder; import org.apache.druid.timeline.partition.HashBasedNumberedShardSpec; @@ -217,7 +217,10 @@ public void setup() final String dataSource = DATA_SOURCE_PREFIX + i; for (int j : new int[]{0, 1, 2, 3, 7, 8}) { for (int k = 0; k < PARTITION_PER_TIME_INTERVAL; k++) { - List segmentForDatasource = datasourceToSegments.computeIfAbsent(dataSource, key -> new ArrayList<>()); + List segmentForDatasource = datasourceToSegments.computeIfAbsent( + dataSource, + key -> new ArrayList<>() + ); DataSegment dataSegment = createSegment(dataSource, j, true, k); allSegments.add(dataSegment); segmentForDatasource.add(dataSegment); @@ -250,17 +253,10 @@ private DataSegment createSegment(String dataSource, int startDay, boolean befor startDay + 2 ) ); - return new DataSegment( - dataSource, - interval, - "version", - null, - Collections.emptyList(), - Collections.emptyList(), - shardSpec, - 0, - 10L - ); + return DataSegment.builder(SegmentId.of(dataSource, interval, "version", partition)) + .shardSpec(shardSpec) + .size(10L) + .build(); } @Test @@ -841,7 +837,11 @@ public void testCompactWithNullIOConfig() ); doCompactSegments(compactSegments, compactionConfigs); ClientCompactionTaskQuery taskPayload = (ClientCompactionTaskQuery) payloadCaptor.getValue(); - Assert.assertEquals(BatchIOConfig.DEFAULT_DROP_EXISTING, taskPayload.getIoConfig().isDropExisting()); + if (CompactionEngine.NATIVE.equals(engine)) { + Assert.assertFalse(taskPayload.getIoConfig().isDropExisting()); + } else { + Assert.assertTrue(taskPayload.getIoConfig().isDropExisting()); + } } @Test @@ -861,8 +861,12 @@ public void testCompactWithGranularitySpec() .withTuningConfig(getTuningConfig(3)) .withEngine(engine) .withGranularitySpec( - new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null) - ) + new UserCompactionTaskGranularityConfig( + Granularities.YEAR, + null, + null + ) + ) .build() ); doCompactSegments(compactSegments, compactionConfigs); @@ -897,10 +901,10 @@ public void testCompactWithDimensionSpec() .withSkipOffsetFromLatest(new Period("PT0H")) // smaller than segment interval .withTuningConfig(getTuningConfig(3)) .withDimensionsSpec( - new UserCompactionTaskDimensionsConfig( - DimensionsSpec.getDefaultSchemas(ImmutableList.of("bar", "foo")) - ) - ) + new UserCompactionTaskDimensionsConfig( + DimensionsSpec.getDefaultSchemas(ImmutableList.of("bar", "foo")) + ) + ) .withEngine(engine) .build() ); @@ -964,10 +968,10 @@ public void testCompactWithProjections() .withSkipOffsetFromLatest(new Period("PT0H")) // smaller than segment interval .withTuningConfig(getTuningConfig(3)) .withDimensionsSpec( - new UserCompactionTaskDimensionsConfig( - DimensionsSpec.getDefaultSchemas(ImmutableList.of("bar", "foo")) - ) - ) + new UserCompactionTaskDimensionsConfig( + DimensionsSpec.getDefaultSchemas(ImmutableList.of("bar", "foo")) + ) + ) .withProjections(projections) .withEngine(engine) .build() @@ -1052,8 +1056,12 @@ public void testCompactWithRollupInGranularitySpec() .withSkipOffsetFromLatest(new Period("PT0H")) // smaller than segment interval .withTuningConfig(getTuningConfig(3)) .withGranularitySpec( - new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, true) - ) + new UserCompactionTaskGranularityConfig( + Granularities.YEAR, + null, + true + ) + ) .withEngine(engine) .build() ); @@ -1143,8 +1151,12 @@ public void testCompactWithGranularitySpecConflictWithActiveCompactionTask() .withSkipOffsetFromLatest(new Period("PT0H")) // smaller than segment interval .withTuningConfig(getTuningConfig(3)) .withGranularitySpec( - new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null) - ) + new UserCompactionTaskGranularityConfig( + Granularities.YEAR, + null, + null + ) + ) .withEngine(engine) .build() ); @@ -1285,10 +1297,10 @@ public void testCompactWithTransformSpec() .withSkipOffsetFromLatest(new Period("PT0H")) // smaller than segment interval .withTuningConfig(getTuningConfig(3)) .withTransformSpec( - new CompactionTransformSpec( - new SelectorDimFilter("dim1", "foo", null) - ) - ) + new CompactionTransformSpec( + new SelectorDimFilter("dim1", "foo", null) + ) + ) .withEngine(engine) .build() ); @@ -1325,7 +1337,7 @@ public void testCompactWithoutCustomSpecs() @Test public void testCompactWithMetricsSpec() { - AggregatorFactory[] aggregatorFactories = new AggregatorFactory[] {new CountAggregatorFactory("cnt")}; + AggregatorFactory[] aggregatorFactories = new AggregatorFactory[]{new CountAggregatorFactory("cnt")}; final OverlordClient mockClient = Mockito.mock(OverlordClient.class); final ArgumentCaptor payloadCaptor = setUpMockClient(mockClient); final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient); @@ -1355,30 +1367,26 @@ public void testDetermineSegmentGranularityFromSegmentsToCompact() String dataSourceName = DATA_SOURCE_PREFIX + 1; List segments = new ArrayList<>(); segments.add( - new DataSegment( - dataSourceName, - Intervals.of("2017-01-01T00:00:00/2017-01-02T00:00:00"), - "1", - null, - ImmutableList.of(), - ImmutableList.of(), - shardSpecFactory.apply(0, 2), - 0, - 10L - ) + DataSegment.builder(SegmentId.of( + dataSourceName, + Intervals.of("2017-01-01T00:00:00/2017-01-02T00:00:00"), + "1", + 0 + )) + .shardSpec(shardSpecFactory.apply(0, 2)) + .size(10L) + .build() ); segments.add( - new DataSegment( - dataSourceName, - Intervals.of("2017-01-01T00:00:00/2017-01-02T00:00:00"), - "1", - null, - ImmutableList.of(), - ImmutableList.of(), - shardSpecFactory.apply(1, 2), - 0, - 10L - ) + DataSegment.builder(SegmentId.of( + dataSourceName, + Intervals.of("2017-01-01T00:00:00/2017-01-02T00:00:00"), + "1", + 1 + )) + .shardSpec(shardSpecFactory.apply(1, 2)) + .size(10L) + .build() ); dataSources = DataSourcesSnapshot.fromUsedSegments(segments); @@ -1415,30 +1423,26 @@ public void testDetermineSegmentGranularityFromSegmentGranularityInCompactionCon String dataSourceName = DATA_SOURCE_PREFIX + 1; List segments = new ArrayList<>(); segments.add( - new DataSegment( - dataSourceName, - Intervals.of("2017-01-01T00:00:00/2017-01-02T00:00:00"), - "1", - null, - ImmutableList.of(), - ImmutableList.of(), - shardSpecFactory.apply(0, 2), - 0, - 10L - ) + DataSegment.builder(SegmentId.of( + dataSourceName, + Intervals.of("2017-01-01T00:00:00/2017-01-02T00:00:00"), + "1", + 0 + )) + .shardSpec(shardSpecFactory.apply(0, 2)) + .size(10L) + .build() ); segments.add( - new DataSegment( - dataSourceName, - Intervals.of("2017-01-01T00:00:00/2017-01-02T00:00:00"), - "1", - null, - ImmutableList.of(), - ImmutableList.of(), - shardSpecFactory.apply(1, 2), - 0, - 10L - ) + DataSegment.builder(SegmentId.of( + dataSourceName, + Intervals.of("2017-01-01T00:00:00/2017-01-02T00:00:00"), + "1", + 1 + )) + .shardSpec(shardSpecFactory.apply(1, 2)) + .size(10L) + .build() ); dataSources = DataSourcesSnapshot.fromUsedSegments(segments); @@ -1454,8 +1458,12 @@ public void testDetermineSegmentGranularityFromSegmentGranularityInCompactionCon .withSkipOffsetFromLatest(new Period("PT0H")) // smaller than segment interval .withTuningConfig(getTuningConfig(3)) .withGranularitySpec( - new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null) - ) + new UserCompactionTaskGranularityConfig( + Granularities.YEAR, + null, + null + ) + ) .withEngine(engine) .build() ); @@ -1487,7 +1495,7 @@ public void testCompactWithMetricsSpecShouldSetPreserveExistingMetricsTrue() .withInputSegmentSizeBytes(500L) .withSkipOffsetFromLatest(new Period("PT0H")) // smaller than segment interval .withTuningConfig(getTuningConfig(3)) - .withMetricsSpec(new AggregatorFactory[] {new CountAggregatorFactory("cnt")}) + .withMetricsSpec(new AggregatorFactory[]{new CountAggregatorFactory("cnt")}) .withEngine(engine) .build() ); @@ -1542,17 +1550,20 @@ private void verifySnapshot( { Map autoCompactionSnapshots = compactSegments.getAutoCompactionSnapshot(); AutoCompactionSnapshot snapshot = autoCompactionSnapshots.get(dataSourceName); - Assert.assertEquals(dataSourceName, snapshot.getDataSource()); - Assert.assertEquals(scheduleStatus, snapshot.getScheduleStatus()); - Assert.assertEquals(expectedByteCountAwaitingCompaction, snapshot.getBytesAwaitingCompaction()); - Assert.assertEquals(expectedByteCountCompressed, snapshot.getBytesCompacted()); - Assert.assertEquals(expectedByteCountSkipped, snapshot.getBytesSkipped()); - Assert.assertEquals(expectedIntervalCountAwaitingCompaction, snapshot.getIntervalCountAwaitingCompaction()); - Assert.assertEquals(expectedIntervalCountCompressed, snapshot.getIntervalCountCompacted()); - Assert.assertEquals(expectedIntervalCountSkipped, snapshot.getIntervalCountSkipped()); - Assert.assertEquals(expectedSegmentCountAwaitingCompaction, snapshot.getSegmentCountAwaitingCompaction()); - Assert.assertEquals(expectedSegmentCountCompressed, snapshot.getSegmentCountCompacted()); - Assert.assertEquals(expectedSegmentCountSkipped, snapshot.getSegmentCountSkipped()); + Assert.assertEquals(new AutoCompactionSnapshot( + dataSourceName, + scheduleStatus, + null, + expectedByteCountAwaitingCompaction, + expectedByteCountCompressed, + expectedByteCountSkipped, + expectedSegmentCountAwaitingCompaction, + expectedSegmentCountCompressed, + expectedSegmentCountSkipped, + expectedIntervalCountAwaitingCompaction, + expectedIntervalCountCompressed, + expectedIntervalCountSkipped + ), snapshot); } private void doCompactionAndAssertCompactSegmentStatistics(CompactSegments compactSegments, int compactionRunCount) @@ -1629,7 +1640,10 @@ private CoordinatorRunStats doCompactSegments(CompactSegments compactSegments) return doCompactSegments(compactSegments, (Integer) null); } - private CoordinatorRunStats doCompactSegments(CompactSegments compactSegments, @Nullable Integer numCompactionTaskSlots) + private CoordinatorRunStats doCompactSegments( + CompactSegments compactSegments, + @Nullable Integer numCompactionTaskSlots + ) { return doCompactSegments(compactSegments, createCompactionConfigs(), numCompactionTaskSlots); } @@ -1722,7 +1736,8 @@ private void assertCompactSegments( = dataSources.getUsedSegmentsTimelinesPerDataSource(); for (int i = 0; i < 3; i++) { final String dataSource = DATA_SOURCE_PREFIX + i; - List> holders = dataSourceToTimeline.get(dataSource).lookup(expectedInterval); + List> holders = + dataSourceToTimeline.get(dataSource).lookup(expectedInterval); Assert.assertEquals(1, holders.size()); List> chunks = Lists.newArrayList(holders.get(0).getObject()); Assert.assertEquals(2, chunks.size()); @@ -1818,10 +1833,10 @@ private List createCompactionConfigs( .withTuningConfig(getTuningConfig(maxNumConcurrentSubTasksForNative)) .withEngine(engine) .withTaskContext( - maxNumTasksForMSQ == null - ? null - : ImmutableMap.of(ClientMSQContext.CTX_MAX_NUM_TASKS, maxNumTasksForMSQ) - ) + maxNumTasksForMSQ == null + ? null + : Map.of(ClientMSQContext.CTX_MAX_NUM_TASKS, maxNumTasksForMSQ) + ) .build() ); } @@ -1922,7 +1937,8 @@ private void compactSegments( if (clientCompactionTaskQuery.getTuningConfig().getPartitionsSpec() instanceof DynamicPartitionsSpec) { compactionPartitionsSpec = new DynamicPartitionsSpec( clientCompactionTaskQuery.getTuningConfig().getPartitionsSpec().getMaxRowsPerSegment(), - ((DynamicPartitionsSpec) clientCompactionTaskQuery.getTuningConfig().getPartitionsSpec()).getMaxTotalRowsOr(Long.MAX_VALUE) + ((DynamicPartitionsSpec) clientCompactionTaskQuery.getTuningConfig().getPartitionsSpec()).getMaxTotalRowsOr( + Long.MAX_VALUE) ); } else { compactionPartitionsSpec = clientCompactionTaskQuery.getTuningConfig().getPartitionsSpec(); @@ -1934,40 +1950,39 @@ private void compactSegments( } for (int i = 0; i < 2; i++) { - DataSegment compactSegment = new DataSegment( - segments.get(0).getDataSource(), - compactInterval, - version, - null, - segments.get(0).getDimensions(), - segments.get(0).getMetrics(), - shardSpecFactory.apply(i, 2), - new CompactionState( - compactionPartitionsSpec, - clientCompactionTaskQuery.getDimensionsSpec() == null ? null : new DimensionsSpec( - clientCompactionTaskQuery.getDimensionsSpec().getDimensions() - ), - metricsSpec, - clientCompactionTaskQuery.getTransformSpec(), - jsonMapper.convertValue( - ImmutableMap.of( - "bitmap", - ImmutableMap.of("type", "roaring"), - "dimensionCompression", - "lz4", - "metricCompression", - "lz4", - "longEncoding", - "longs" - ), - IndexSpec.class - ), - jsonMapper.convertValue(ImmutableMap.of(), GranularitySpec.class), - null - ), - 1, - segmentSize - ); + DataSegment compactSegment = + DataSegment.builder(SegmentId.of(segments.get(0).getDataSource(), compactInterval, version, i)) + .dimensions(segments.get(0).getDimensions()) + .metrics(segments.get(0).getMetrics()) + .shardSpec(shardSpecFactory.apply(i, 2)) + .lastCompactionState( + new CompactionState( + compactionPartitionsSpec, + clientCompactionTaskQuery.getDimensionsSpec() == null ? null : new DimensionsSpec( + clientCompactionTaskQuery.getDimensionsSpec().getDimensions() + ), + metricsSpec, + clientCompactionTaskQuery.getTransformSpec(), + jsonMapper.convertValue( + ImmutableMap.of( + "bitmap", + ImmutableMap.of("type", "roaring"), + "dimensionCompression", + "lz4", + "metricCompression", + "lz4", + "longEncoding", + "longs" + ), + IndexSpec.class + ), + jsonMapper.convertValue(ImmutableMap.of(), GranularitySpec.class), + null + ) + ) + .binaryVersion(1) + .size(segmentSize) + .build(); timeline.add( compactInterval, From ca0bc0cb6e184a101c4fbb8a6d3b06b12d52598d Mon Sep 17 00:00:00 2001 From: cecemei Date: Sun, 1 Feb 2026 21:56:08 -0800 Subject: [PATCH 03/19] eligibility --- .../compaction/CompactionCandidate.java | 12 +++++++++++- .../server/compaction/CompactionStatus.java | 19 +++++++++++++++++-- .../DataSourceCompactibleSegmentIterator.java | 4 ++-- .../MostFragmentedIntervalFirstPolicy.java | 6 ++---- .../coordinator/duty/CompactSegments.java | 2 -- ...MostFragmentedIntervalFirstPolicyTest.java | 9 +++++---- 6 files changed, 37 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java index 05e3d6b67c48..fbe9cf376052 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java @@ -237,6 +237,12 @@ public CompactionCandidate evaluate( case NOT_ELIGIBLE: return this.withPolicyEligibility(evaluated.lhs).withCurrentStatus(evaluated.rhs); case FULL_COMPACTION: // evaluator has decided compaction is needed, policy needs to further check + if (!evaluated.rhs.getState().equals(CompactionStatus.State.PENDING)) { + throw DruidException.defensive( + "Evaluated compaction status should be PENDING, got status[%s] instead.", + evaluated.rhs.getState() + ); + } final CompactionCandidateSearchPolicy.Eligibility searchPolicyEligibility = searchPolicy.checkEligibilityForCompaction(this, null); switch (searchPolicyEligibility.getPolicyEligibility()) { @@ -252,7 +258,7 @@ public CompactionCandidate evaluate( case INCREMENTAL_COMPACTION: // policy decided to perform an incremental compaction, the uncompactedSegments is a subset of the original segments. return new CompactionCandidate( - evaluator.uncompactedSegments, + evaluator.getUncompactedSegments(), umbrellaInterval, compactionInterval, numIntervals, @@ -273,8 +279,12 @@ public String toString() { return "SegmentsToCompact{" + "datasource=" + dataSource + + ", umbrellaInterval=" + umbrellaInterval + + ", compactionInterval=" + compactionInterval + + ", numIntervals=" + numIntervals + ", segments=" + SegmentUtils.commaSeparatedIdentifiers(segments) + ", totalSize=" + totalBytes + + ", policyEligiblity=" + policyEligiblity + ", currentStatus=" + currentStatus + '}'; } diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java index 1a9800bf727c..ab83a9a9cd42 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java @@ -344,7 +344,7 @@ static class Evaluator private final List fingerprintedSegments = new ArrayList<>(); private final List compactedSegments = new ArrayList<>(); - final List uncompactedSegments = new ArrayList<>(); + private final List uncompactedSegments = new ArrayList<>(); private final Map> unknownStateToSegments = new HashMap<>(); @Nullable @@ -373,6 +373,22 @@ static class Evaluator } } + List getUncompactedSegments() + { + return uncompactedSegments; + } + + /** + * Evaluates the compaction status of candidate segments through a multi-step process: + *
    + *
  1. Validates input bytes are within limits
    2. + *
  2. Categorizes segments by compaction state (fingerprinted, uncompacted, or unknown)
    3. + *
  3. Performs fingerprint-based validation if available (fast path)
    4. + *
  4. Runs detailed checks against unknown states via {@link #CHECKS}
    5. + *
+ * + * @return Pair of eligibility status and compaction status with reason for first failed check + */ Pair evaluate() { final CompactionCandidateSearchPolicy.Eligibility inputBytesCheck = inputBytesAreWithinLimit(); @@ -421,7 +437,6 @@ Pair evaluate() if (reasonsForCompaction.isEmpty()) { return Pair.of( CompactionCandidateSearchPolicy.Eligibility.NOT_APPLICABLE, - //fail("All checks are passed, no reason to compact"), CompactionStatus.COMPLETE ); } else { diff --git a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java index 032f9ccdea6a..5e77d1bf9907 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java +++ b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java @@ -438,8 +438,8 @@ static Interval computeLatestSkipInterval( if (configuredSegmentGranularity == null) { return new Interval(skipOffsetFromLatest, latestDataTimestamp); } else { - DateTime skipFromLastest = new DateTime(latestDataTimestamp, latestDataTimestamp.getZone()).minus( - skipOffsetFromLatest); + DateTime skipFromLastest = + new DateTime(latestDataTimestamp, latestDataTimestamp.getZone()).minus(skipOffsetFromLatest); DateTime skipOffsetBucketToSegmentGranularity = configuredSegmentGranularity.bucketStart(skipFromLastest); return new Interval(skipOffsetBucketToSegmentGranularity, latestDataTimestamp); } diff --git a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java index 6086ff84c7fd..c4f9b3ae3940 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java @@ -145,10 +145,8 @@ public boolean equals(Object o) } MostFragmentedIntervalFirstPolicy policy = (MostFragmentedIntervalFirstPolicy) o; return minUncompactedCount == policy.minUncompactedCount - && Double.compare( - incrementalCompactionUncompactedBytesRatioThreshold, - policy.incrementalCompactionUncompactedBytesRatioThreshold - ) == 0 + && incrementalCompactionUncompactedBytesRatioThreshold + == policy.incrementalCompactionUncompactedBytesRatioThreshold && Objects.equals(minUncompactedBytes, policy.minUncompactedBytes) && Objects.equals(maxAverageUncompactedBytesPerSegment, policy.maxAverageUncompactedBytesPerSegment); } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java index 594bcdb53b83..f912bad693c1 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java @@ -193,7 +193,6 @@ public void run( compactionSnapshotBuilder, slotManager, iterator, - policy, defaultEngine ); @@ -229,7 +228,6 @@ private int submitCompactionTasks( CompactionSnapshotBuilder snapshotBuilder, CompactionSlotManager slotManager, CompactionSegmentIterator iterator, - CompactionCandidateSearchPolicy policy, CompactionEngine defaultEngine ) { diff --git a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java index 76ea2008206a..0366c0c1ada4 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java @@ -275,7 +275,7 @@ public void test_checkEligibilityForCompaction_returnsIncrementalCompaction_when null ); - final CompactionCandidate candidate = createCandidateWithStats(1200L, 400L, 100); + final CompactionCandidate candidate = createCandidateWithStats(1200L, 10, 400L, 100); Assertions.assertEquals( CompactionCandidateSearchPolicy.Eligibility.incrementalCompaction( @@ -296,7 +296,7 @@ public void test_checkEligibilityForCompaction_returnsFullCompaction_whenRatioAb null ); - final CompactionCandidate candidate = createCandidateWithStats(500L, 600L, 100); + final CompactionCandidate candidate = createCandidateWithStats(500L, 5, 600L, 100); Assertions.assertEquals( CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, @@ -317,7 +317,7 @@ public void test_checkEligibilityForCompaction_returnsFullCompaction_whenThresho ); // With default threshold 0.0, any positive ratio >= 0.0, so always FULL_COMPACTION_ELIGIBLE - final CompactionCandidate candidate = createCandidateWithStats(1000L, 100L, 100); + final CompactionCandidate candidate = createCandidateWithStats(1000L, 10, 100L, 100); Assertions.assertEquals( CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, @@ -339,11 +339,12 @@ private CompactionCandidate createCandidate(int numSegments, long avgSizeBytes) private CompactionCandidate createCandidateWithStats( long compactedBytes, + int compactedSegments, long uncompactedBytes, int uncompactedSegments ) { - final CompactionStatistics compactedStats = CompactionStatistics.create(compactedBytes, 1L, 1L); + final CompactionStatistics compactedStats = CompactionStatistics.create(compactedBytes, compactedSegments, 1L); final CompactionStatistics uncompactedStats = CompactionStatistics.create( uncompactedBytes, uncompactedSegments, From ab7990471d4b049994ae399991f0845817a2c7db Mon Sep 17 00:00:00 2001 From: cecemei Date: Sun, 1 Feb 2026 22:40:51 -0800 Subject: [PATCH 04/19] Double.compare --- .../compaction/MostFragmentedIntervalFirstPolicy.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java index c4f9b3ae3940..62041e56b589 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java @@ -145,10 +145,13 @@ public boolean equals(Object o) } MostFragmentedIntervalFirstPolicy policy = (MostFragmentedIntervalFirstPolicy) o; return minUncompactedCount == policy.minUncompactedCount - && incrementalCompactionUncompactedBytesRatioThreshold - == policy.incrementalCompactionUncompactedBytesRatioThreshold && Objects.equals(minUncompactedBytes, policy.minUncompactedBytes) - && Objects.equals(maxAverageUncompactedBytesPerSegment, policy.maxAverageUncompactedBytesPerSegment); + && Objects.equals(maxAverageUncompactedBytesPerSegment, policy.maxAverageUncompactedBytesPerSegment) + // Use Double.compare instead of == to handle NaN correctly and keep equals() consistent with hashCode() (especially for +0.0 vs -0.0). + && Double.compare( + incrementalCompactionUncompactedBytesRatioThreshold, + policy.incrementalCompactionUncompactedBytesRatioThreshold + ) == 0; } @Override From 08d70b8e8fb272047a313ba750ceaeb20085a1d6 Mon Sep 17 00:00:00 2001 From: cecemei Date: Wed, 4 Feb 2026 15:27:54 -0800 Subject: [PATCH 05/19] CompactionEligibility --- .../compaction/BaseCandidateSearchPolicy.java | 4 +- .../compaction/CompactionCandidate.java | 16 +-- .../CompactionCandidateSearchPolicy.java | 79 +-------------- .../compaction/CompactionEligibility.java | 98 +++++++++++++++++++ .../server/compaction/CompactionStatus.java | 12 +-- .../compaction/FixedIntervalOrderPolicy.java | 6 +- .../MostFragmentedIntervalFirstPolicy.java | 16 +-- .../coordinator/duty/CompactSegments.java | 2 +- .../compaction/CompactionEligibilityTest.java | 40 ++++++++ ...MostFragmentedIntervalFirstPolicyTest.java | 34 +++---- 10 files changed, 184 insertions(+), 123 deletions(-) create mode 100644 server/src/main/java/org/apache/druid/server/compaction/CompactionEligibility.java create mode 100644 server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityTest.java diff --git a/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java index e0fce9c85a7f..7be87683dcb6 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java @@ -68,12 +68,12 @@ public final int compareCandidates(CompactionCandidate o1, CompactionCandidate o } @Override - public Eligibility checkEligibilityForCompaction( + public CompactionEligibility checkEligibilityForCompaction( CompactionCandidate candidate, CompactionTaskStatus latestTaskStatus ) { - return Eligibility.FULL_COMPACTION_ELIGIBLE; + return CompactionEligibility.FULL_COMPACTION_ELIGIBLE; } /** diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java index fbe9cf376052..a07e6b71cfd2 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java @@ -49,7 +49,7 @@ public class CompactionCandidate private final long totalBytes; private final int numIntervals; - private final CompactionCandidateSearchPolicy.Eligibility policyEligiblity; + private final CompactionEligibility policyEligiblity; private final CompactionStatus currentStatus; public static CompactionCandidate from( @@ -84,7 +84,7 @@ private CompactionCandidate( Interval umbrellaInterval, Interval compactionInterval, int numDistinctSegmentIntervals, - CompactionCandidateSearchPolicy.Eligibility policyEligiblity, + CompactionEligibility policyEligiblity, @Nullable CompactionStatus currentStatus ) { @@ -170,7 +170,7 @@ public CompactionStatus getCurrentStatus() } @Nullable - public CompactionCandidateSearchPolicy.Eligibility getPolicyEligibility() + public CompactionEligibility getPolicyEligibility() { return policyEligiblity; } @@ -190,7 +190,7 @@ public CompactionCandidate withCurrentStatus(CompactionStatus status) ); } - public CompactionCandidate withPolicyEligibility(CompactionCandidateSearchPolicy.Eligibility eligibility) + public CompactionCandidate withPolicyEligibility(CompactionEligibility eligibility) { return new CompactionCandidate( segments, @@ -231,8 +231,8 @@ public CompactionCandidate evaluate( ) { CompactionStatus.Evaluator evaluator = new CompactionStatus.Evaluator(this, config, fingerprintMapper); - Pair evaluated = evaluator.evaluate(); - switch (Objects.requireNonNull(evaluated.lhs).getPolicyEligibility()) { + Pair evaluated = evaluator.evaluate(); + switch (Objects.requireNonNull(evaluated.lhs).getState()) { case NOT_APPLICABLE: case NOT_ELIGIBLE: return this.withPolicyEligibility(evaluated.lhs).withCurrentStatus(evaluated.rhs); @@ -243,9 +243,9 @@ public CompactionCandidate evaluate( evaluated.rhs.getState() ); } - final CompactionCandidateSearchPolicy.Eligibility searchPolicyEligibility = + final CompactionEligibility searchPolicyEligibility = searchPolicy.checkEligibilityForCompaction(this, null); - switch (searchPolicyEligibility.getPolicyEligibility()) { + switch (searchPolicyEligibility.getState()) { case NOT_ELIGIBLE: // although evaluator thinks this interval qualifies for compaction, but policy decided not its turn yet. return this.withPolicyEligibility(searchPolicyEligibility) diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java index 558089203c93..14c78d1e204c 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java @@ -21,11 +21,8 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.server.coordinator.duty.CompactSegments; -import java.util.Objects; - /** * Policy used by {@link CompactSegments} duty to pick segments for compaction. */ @@ -52,82 +49,8 @@ public interface CompactionCandidateSearchPolicy * in the current iteration. A policy may implement this method to skip * compacting intervals or segments that do not fulfil some required criteria. */ - Eligibility checkEligibilityForCompaction( + CompactionEligibility checkEligibilityForCompaction( CompactionCandidate candidate, CompactionTaskStatus latestTaskStatus ); - - /** - * Describes the eligibility of an interval for compaction. - */ - class Eligibility - { - public enum PolicyEligibility - { - FULL_COMPACTION, - INCREMENTAL_COMPACTION, - NOT_ELIGIBLE, - NOT_APPLICABLE - } - - public static final Eligibility FULL_COMPACTION_ELIGIBLE = new Eligibility(PolicyEligibility.FULL_COMPACTION, null); - public static final Eligibility NOT_APPLICABLE = new Eligibility(PolicyEligibility.NOT_APPLICABLE, null); - - private final PolicyEligibility eligible; - private final String reason; - - private Eligibility(PolicyEligibility eligible, String reason) - { - this.eligible = eligible; - this.reason = reason; - } - - public PolicyEligibility getPolicyEligibility() - { - return eligible; - } - - public String getReason() - { - return reason; - } - - public static Eligibility incrementalCompaction(String messageFormat, Object... args) - { - return new Eligibility(PolicyEligibility.INCREMENTAL_COMPACTION, StringUtils.format(messageFormat, args)); - } - - public static Eligibility fail(String messageFormat, Object... args) - { - return new Eligibility(PolicyEligibility.NOT_ELIGIBLE, StringUtils.format(messageFormat, args)); - } - - @Override - public boolean equals(Object object) - { - if (this == object) { - return true; - } - if (object == null || getClass() != object.getClass()) { - return false; - } - Eligibility that = (Eligibility) object; - return eligible == that.eligible && Objects.equals(reason, that.reason); - } - - @Override - public int hashCode() - { - return Objects.hash(eligible, reason); - } - - @Override - public String toString() - { - return "Eligibility{" + - "eligible=" + eligible + - ", reason='" + reason + '\'' + - '}'; - } - } } diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionEligibility.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionEligibility.java new file mode 100644 index 000000000000..eab13ed39662 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionEligibility.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.compaction; + +import org.apache.druid.java.util.common.StringUtils; + +import java.util.Objects; + +/** + * Describes the eligibility of an interval for compaction. + */ +public class CompactionEligibility +{ + public enum State + { + FULL_COMPACTION, + INCREMENTAL_COMPACTION, + NOT_ELIGIBLE, + NOT_APPLICABLE + } + + public static final CompactionEligibility FULL_COMPACTION_ELIGIBLE = new CompactionEligibility(State.FULL_COMPACTION, null); + public static final CompactionEligibility NOT_APPLICABLE = new CompactionEligibility(State.NOT_APPLICABLE, null); + + private final State state; + private final String reason; + + private CompactionEligibility(State state, String reason) + { + this.state = state; + this.reason = reason; + } + + public State getState() + { + return state; + } + + public String getReason() + { + return reason; + } + + public static CompactionEligibility incrementalCompaction(String messageFormat, Object... args) + { + return new CompactionEligibility(State.INCREMENTAL_COMPACTION, StringUtils.format(messageFormat, args)); + } + + public static CompactionEligibility fail(String messageFormat, Object... args) + { + return new CompactionEligibility(State.NOT_ELIGIBLE, StringUtils.format(messageFormat, args)); + } + + @Override + public boolean equals(Object object) + { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + CompactionEligibility that = (CompactionEligibility) object; + return state == that.state && Objects.equals(reason, that.reason); + } + + @Override + public int hashCode() + { + return Objects.hash(state, reason); + } + + @Override + public String toString() + { + return "Eligibility{" + + "state=" + state + + ", reason='" + reason + '\'' + + '}'; + } +} diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java index ab83a9a9cd42..0c330a9f6c5f 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java @@ -389,9 +389,9 @@ List getUncompactedSegments() * * @return Pair of eligibility status and compaction status with reason for first failed check */ - Pair evaluate() + Pair evaluate() { - final CompactionCandidateSearchPolicy.Eligibility inputBytesCheck = inputBytesAreWithinLimit(); + final CompactionEligibility inputBytesCheck = inputBytesAreWithinLimit(); if (inputBytesCheck != null) { return Pair.of(inputBytesCheck, CompactionStatus.skipped(inputBytesCheck.getReason())); } @@ -436,12 +436,12 @@ Pair evaluate() if (reasonsForCompaction.isEmpty()) { return Pair.of( - CompactionCandidateSearchPolicy.Eligibility.NOT_APPLICABLE, + CompactionEligibility.NOT_APPLICABLE, CompactionStatus.COMPLETE ); } else { return Pair.of( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, + CompactionEligibility.FULL_COMPACTION_ELIGIBLE, CompactionStatus.pending( createStats(compactedSegments), createStats(uncompactedSegments), @@ -632,11 +632,11 @@ private CompactionStatus projectionsAreUpToDate(CompactionState lastCompactionSt } @Nullable - private CompactionCandidateSearchPolicy.Eligibility inputBytesAreWithinLimit() + private CompactionEligibility inputBytesAreWithinLimit() { final long inputSegmentSize = compactionConfig.getInputSegmentSizeBytes(); if (candidateSegments.getTotalBytes() > inputSegmentSize) { - return CompactionCandidateSearchPolicy.Eligibility.fail( + return CompactionEligibility.fail( "'inputSegmentSize' exceeded: Total segment size[%d] is larger than allowed inputSegmentSize[%d]", candidateSegments.getTotalBytes(), inputSegmentSize ); diff --git a/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java index afe6a52bea48..35920618801d 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java @@ -56,14 +56,14 @@ public int compareCandidates(CompactionCandidate candidateA, CompactionCandidate } @Override - public Eligibility checkEligibilityForCompaction( + public CompactionEligibility checkEligibilityForCompaction( CompactionCandidate candidate, CompactionTaskStatus latestTaskStatus ) { return findIndex(candidate) < Integer.MAX_VALUE - ? Eligibility.FULL_COMPACTION_ELIGIBLE - : Eligibility.fail("Datasource/Interval is not in the list of 'eligibleCandidates'"); + ? CompactionEligibility.FULL_COMPACTION_ELIGIBLE + : CompactionEligibility.fail("Datasource/Interval is not in the list of 'eligibleCandidates'"); } private int findIndex(CompactionCandidate candidate) diff --git a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java index 62041e56b589..48617e951861 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java @@ -187,23 +187,23 @@ private int compare(CompactionCandidate candidateA, CompactionCandidate candidat } @Override - public Eligibility checkEligibilityForCompaction( + public CompactionEligibility checkEligibilityForCompaction( CompactionCandidate candidate, CompactionTaskStatus latestTaskStatus ) { final CompactionStatistics uncompacted = candidate.getUncompactedStats(); if (uncompacted == null) { - return Eligibility.FULL_COMPACTION_ELIGIBLE; + return CompactionEligibility.FULL_COMPACTION_ELIGIBLE; } else if (uncompacted.getNumSegments() < 1) { - return Eligibility.fail("No uncompacted segments in interval"); + return CompactionEligibility.fail("No uncompacted segments in interval"); } else if (uncompacted.getNumSegments() < minUncompactedCount) { - return Eligibility.fail( + return CompactionEligibility.fail( "Uncompacted segments[%,d] in interval must be at least [%,d]", uncompacted.getNumSegments(), minUncompactedCount ); } else if (uncompacted.getTotalBytes() < minUncompactedBytes.getBytes()) { - return Eligibility.fail( + return CompactionEligibility.fail( "Uncompacted bytes[%,d] in interval must be at least [%,d]", uncompacted.getTotalBytes(), minUncompactedBytes.getBytes() ); @@ -211,7 +211,7 @@ public Eligibility checkEligibilityForCompaction( final long avgSegmentSize = (uncompacted.getTotalBytes() / uncompacted.getNumSegments()); if (avgSegmentSize > maxAverageUncompactedBytesPerSegment.getBytes()) { - return Eligibility.fail( + return CompactionEligibility.fail( "Average size[%,d] of uncompacted segments in interval must be at most [%,d]", avgSegmentSize, maxAverageUncompactedBytesPerSegment.getBytes() ); @@ -220,13 +220,13 @@ public Eligibility checkEligibilityForCompaction( final double uncompactedBytesRatio = (double) uncompacted.getTotalBytes() / (uncompacted.getTotalBytes() + candidate.getCompactedStats().getTotalBytes()); if (uncompactedBytesRatio < incrementalCompactionUncompactedBytesRatioThreshold) { - return Eligibility.incrementalCompaction( + return CompactionEligibility.incrementalCompaction( "Uncompacted bytes ratio[%.2f] is below threshold[%.2f]", uncompactedBytesRatio, incrementalCompactionUncompactedBytesRatioThreshold ); } else { - return Eligibility.FULL_COMPACTION_ELIGIBLE; + return CompactionEligibility.FULL_COMPACTION_ELIGIBLE; } } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java index f912bad693c1..07b8b23d58fa 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java @@ -464,7 +464,7 @@ private static ClientCompactionTaskQuery compactSegments( final String taskId = IdUtils.newTaskId(TASK_ID_PREFIX, ClientCompactionTaskQuery.TYPE, dataSource, null); final ClientCompactionIntervalSpec clientCompactionIntervalSpec; Preconditions.checkArgument(entry.getPolicyEligibility() != null, "Must have a policy eligibility"); - switch (entry.getPolicyEligibility().getPolicyEligibility()) { + switch (entry.getPolicyEligibility().getState()) { case FULL_COMPACTION: clientCompactionIntervalSpec = new ClientCompactionIntervalSpec(entry.getCompactionInterval(), null, null); break; diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityTest.java new file mode 100644 index 000000000000..5c96fd2e10c4 --- /dev/null +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.compaction; + +import org.junit.Assert; +import org.junit.Test; + +public class CompactionEligibilityTest +{ + @Test + public void testEqualsAndHashCode() + { + final CompactionEligibility e1 = CompactionEligibility.fail("reason"); + final CompactionEligibility e2 = CompactionEligibility.fail("reason"); + final CompactionEligibility e3 = CompactionEligibility.fail("different"); + final CompactionEligibility e4 = CompactionEligibility.incrementalCompaction("reason"); + + Assert.assertEquals(e1, e2); + Assert.assertEquals(e1.hashCode(), e2.hashCode()); + Assert.assertNotEquals(e1, e3); + Assert.assertNotEquals(e1, e4); + } +} diff --git a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java index 0366c0c1ada4..e5f92343451a 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java @@ -60,13 +60,13 @@ public void test_checkEligibilityForCompaction_fails_ifUncompactedCountLessThanC ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.fail( + CompactionEligibility.fail( "Uncompacted segments[1] in interval must be at least [10,000]" ), policy.checkEligibilityForCompaction(createCandidate(1, 100L), null) ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, + CompactionEligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(createCandidate(10_001, 100L), null) ); } @@ -84,13 +84,13 @@ public void test_checkEligibilityForCompaction_fails_ifUncompactedBytesLessThanC ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.fail( + CompactionEligibility.fail( "Uncompacted bytes[100] in interval must be at least [10,000]" ), policy.checkEligibilityForCompaction(createCandidate(1, 100L), null) ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, + CompactionEligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(createCandidate(100, 10_000L), null) ); } @@ -108,13 +108,13 @@ public void test_checkEligibilityForCompaction_fails_ifAvgSegmentSizeGreaterThan ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.fail( + CompactionEligibility.fail( "Average size[10,000] of uncompacted segments in interval must be at most [100]" ), policy.checkEligibilityForCompaction(createCandidate(1, 10_000L), null) ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, + CompactionEligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(createCandidate(1, 100L), null) ); } @@ -134,11 +134,11 @@ public void test_policy_favorsIntervalWithMoreUncompactedSegments_ifTotalBytesIs final CompactionCandidate candidateB = createCandidate(2, 500L); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, + CompactionEligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidateA, null) ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, + CompactionEligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidateB, null) ); @@ -161,11 +161,11 @@ public void test_policy_favorsIntervalWithMoreUncompactedSegments_ifAverageSizeI final CompactionCandidate candidateB = createCandidate(2, 1000L); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, + CompactionEligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidateA, null) ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, + CompactionEligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidateB, null) ); @@ -188,11 +188,11 @@ public void test_policy_favorsIntervalWithSmallerSegments_ifCountIsEqual() final CompactionCandidate candidateB = createCandidate(10, 1000L); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, + CompactionEligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidateA, null) ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, + CompactionEligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidateB, null) ); @@ -215,11 +215,11 @@ public void test_compareCandidates_returnsZeroIfSegmentCountAndAvgSizeScaleEquiv final CompactionCandidate candidateB = createCandidate(400, 100); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, + CompactionEligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidateA, null) ); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, + CompactionEligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidateB, null) ); @@ -278,7 +278,7 @@ public void test_checkEligibilityForCompaction_returnsIncrementalCompaction_when final CompactionCandidate candidate = createCandidateWithStats(1200L, 10, 400L, 100); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.incrementalCompaction( + CompactionEligibility.incrementalCompaction( "Uncompacted bytes ratio[0.25] is below threshold[0.50]"), policy.checkEligibilityForCompaction(candidate, null) ); @@ -299,7 +299,7 @@ public void test_checkEligibilityForCompaction_returnsFullCompaction_whenRatioAb final CompactionCandidate candidate = createCandidateWithStats(500L, 5, 600L, 100); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, + CompactionEligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidate, null) ); } @@ -320,7 +320,7 @@ public void test_checkEligibilityForCompaction_returnsFullCompaction_whenThresho final CompactionCandidate candidate = createCandidateWithStats(1000L, 10, 100L, 100); Assertions.assertEquals( - CompactionCandidateSearchPolicy.Eligibility.FULL_COMPACTION_ELIGIBLE, + CompactionEligibility.FULL_COMPACTION_ELIGIBLE, policy.checkEligibilityForCompaction(candidate, null) ); } From 5aba7d95daf6d0913277ee5be90bd1a8e51c793c Mon Sep 17 00:00:00 2001 From: cecemei Date: Wed, 4 Feb 2026 15:31:30 -0800 Subject: [PATCH 06/19] candidate --- .../indexing/compact/CompactionConfigBasedJobTemplate.java | 4 +--- .../apache/druid/server/compaction/CompactionCandidate.java | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionConfigBasedJobTemplate.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionConfigBasedJobTemplate.java index 24b6e1f6af40..fb0752228b6f 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionConfigBasedJobTemplate.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionConfigBasedJobTemplate.java @@ -133,9 +133,7 @@ DataSourceCompactibleSegmentIterator getCompactibleCandidates( config, timeline, Intervals.complementOf(searchInterval), - // This policy is used only while creating jobs - // The actual order of jobs is determined by the policy used in CompactionJobQueue - new NewestSegmentFirstPolicy(null), + params.getClusterCompactionConfig().getCompactionPolicy(), params.getFingerprintMapper() ); diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java index a07e6b71cfd2..4330e093ad2a 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java @@ -244,7 +244,8 @@ public CompactionCandidate evaluate( ); } final CompactionEligibility searchPolicyEligibility = - searchPolicy.checkEligibilityForCompaction(this, null); + searchPolicy.checkEligibilityForCompaction(this.withPolicyEligibility(evaluated.lhs) + .withCurrentStatus(evaluated.rhs), null); switch (searchPolicyEligibility.getState()) { case NOT_ELIGIBLE: // although evaluator thinks this interval qualifies for compaction, but policy decided not its turn yet. From 5001d313b07f51d15051121b773a7b9034e6b5a6 Mon Sep 17 00:00:00 2001 From: cecemei Date: Wed, 4 Feb 2026 15:58:12 -0800 Subject: [PATCH 07/19] format --- .../druid/indexing/compact/CompactionConfigBasedJobTemplate.java | 1 - 1 file changed, 1 deletion(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionConfigBasedJobTemplate.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionConfigBasedJobTemplate.java index fb0752228b6f..5c1b2ff16fcf 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionConfigBasedJobTemplate.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionConfigBasedJobTemplate.java @@ -28,7 +28,6 @@ import org.apache.druid.server.compaction.CompactionCandidate; import org.apache.druid.server.compaction.CompactionSlotManager; import org.apache.druid.server.compaction.DataSourceCompactibleSegmentIterator; -import org.apache.druid.server.compaction.NewestSegmentFirstPolicy; import org.apache.druid.server.coordinator.DataSourceCompactionConfig; import org.apache.druid.server.coordinator.duty.CompactSegments; import org.apache.druid.timeline.CompactionState; From e38257bd239559cf1ec8ec886bec061ebc30923a Mon Sep 17 00:00:00 2001 From: cecemei Date: Wed, 4 Feb 2026 19:11:37 -0800 Subject: [PATCH 08/19] policy-status --- .../compaction/BaseCandidateSearchPolicy.java | 5 +-- .../compaction/CompactionCandidate.java | 2 +- .../CompactionCandidateSearchPolicy.java | 5 +-- .../compaction/FixedIntervalOrderPolicy.java | 5 +-- .../MostFragmentedIntervalFirstPolicy.java | 5 +-- ...MostFragmentedIntervalFirstPolicyTest.java | 34 +++++++++---------- 6 files changed, 22 insertions(+), 34 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java index 7be87683dcb6..4d0e22d6d82e 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java @@ -68,10 +68,7 @@ public final int compareCandidates(CompactionCandidate o1, CompactionCandidate o } @Override - public CompactionEligibility checkEligibilityForCompaction( - CompactionCandidate candidate, - CompactionTaskStatus latestTaskStatus - ) + public CompactionEligibility checkEligibilityForCompaction(CompactionCandidate candidate) { return CompactionEligibility.FULL_COMPACTION_ELIGIBLE; } diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java index 4330e093ad2a..8bad730d71a9 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java @@ -245,7 +245,7 @@ public CompactionCandidate evaluate( } final CompactionEligibility searchPolicyEligibility = searchPolicy.checkEligibilityForCompaction(this.withPolicyEligibility(evaluated.lhs) - .withCurrentStatus(evaluated.rhs), null); + .withCurrentStatus(evaluated.rhs)); switch (searchPolicyEligibility.getState()) { case NOT_ELIGIBLE: // although evaluator thinks this interval qualifies for compaction, but policy decided not its turn yet. diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java index 14c78d1e204c..744de2897f6c 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java @@ -49,8 +49,5 @@ public interface CompactionCandidateSearchPolicy * in the current iteration. A policy may implement this method to skip * compacting intervals or segments that do not fulfil some required criteria. */ - CompactionEligibility checkEligibilityForCompaction( - CompactionCandidate candidate, - CompactionTaskStatus latestTaskStatus - ); + CompactionEligibility checkEligibilityForCompaction(CompactionCandidate candidate); } diff --git a/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java index 35920618801d..3d432d3fd703 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java @@ -56,10 +56,7 @@ public int compareCandidates(CompactionCandidate candidateA, CompactionCandidate } @Override - public CompactionEligibility checkEligibilityForCompaction( - CompactionCandidate candidate, - CompactionTaskStatus latestTaskStatus - ) + public CompactionEligibility checkEligibilityForCompaction(CompactionCandidate candidate) { return findIndex(candidate) < Integer.MAX_VALUE ? CompactionEligibility.FULL_COMPACTION_ELIGIBLE diff --git a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java index 48617e951861..8c3f74204feb 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java @@ -187,10 +187,7 @@ private int compare(CompactionCandidate candidateA, CompactionCandidate candidat } @Override - public CompactionEligibility checkEligibilityForCompaction( - CompactionCandidate candidate, - CompactionTaskStatus latestTaskStatus - ) + public CompactionEligibility checkEligibilityForCompaction(CompactionCandidate candidate) { final CompactionStatistics uncompacted = candidate.getUncompactedStats(); if (uncompacted == null) { diff --git a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java index e5f92343451a..8c8749645d25 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java @@ -63,11 +63,11 @@ public void test_checkEligibilityForCompaction_fails_ifUncompactedCountLessThanC CompactionEligibility.fail( "Uncompacted segments[1] in interval must be at least [10,000]" ), - policy.checkEligibilityForCompaction(createCandidate(1, 100L), null) + policy.checkEligibilityForCompaction(createCandidate(1, 100L)) ); Assertions.assertEquals( CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(createCandidate(10_001, 100L), null) + policy.checkEligibilityForCompaction(createCandidate(10_001, 100L)) ); } @@ -87,11 +87,11 @@ public void test_checkEligibilityForCompaction_fails_ifUncompactedBytesLessThanC CompactionEligibility.fail( "Uncompacted bytes[100] in interval must be at least [10,000]" ), - policy.checkEligibilityForCompaction(createCandidate(1, 100L), null) + policy.checkEligibilityForCompaction(createCandidate(1, 100L)) ); Assertions.assertEquals( CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(createCandidate(100, 10_000L), null) + policy.checkEligibilityForCompaction(createCandidate(100, 10_000L)) ); } @@ -111,11 +111,11 @@ public void test_checkEligibilityForCompaction_fails_ifAvgSegmentSizeGreaterThan CompactionEligibility.fail( "Average size[10,000] of uncompacted segments in interval must be at most [100]" ), - policy.checkEligibilityForCompaction(createCandidate(1, 10_000L), null) + policy.checkEligibilityForCompaction(createCandidate(1, 10_000L)) ); Assertions.assertEquals( CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(createCandidate(1, 100L), null) + policy.checkEligibilityForCompaction(createCandidate(1, 100L)) ); } @@ -135,11 +135,11 @@ public void test_policy_favorsIntervalWithMoreUncompactedSegments_ifTotalBytesIs Assertions.assertEquals( CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidateA, null) + policy.checkEligibilityForCompaction(candidateA) ); Assertions.assertEquals( CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidateB, null) + policy.checkEligibilityForCompaction(candidateB) ); Assertions.assertTrue(policy.compareCandidates(candidateA, candidateB) > 0); @@ -162,11 +162,11 @@ public void test_policy_favorsIntervalWithMoreUncompactedSegments_ifAverageSizeI Assertions.assertEquals( CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidateA, null) + policy.checkEligibilityForCompaction(candidateA) ); Assertions.assertEquals( CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidateB, null) + policy.checkEligibilityForCompaction(candidateB) ); Assertions.assertTrue(policy.compareCandidates(candidateA, candidateB) > 0); @@ -189,11 +189,11 @@ public void test_policy_favorsIntervalWithSmallerSegments_ifCountIsEqual() Assertions.assertEquals( CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidateA, null) + policy.checkEligibilityForCompaction(candidateA) ); Assertions.assertEquals( CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidateB, null) + policy.checkEligibilityForCompaction(candidateB) ); Assertions.assertTrue(policy.compareCandidates(candidateA, candidateB) < 0); @@ -216,11 +216,11 @@ public void test_compareCandidates_returnsZeroIfSegmentCountAndAvgSizeScaleEquiv Assertions.assertEquals( CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidateA, null) + policy.checkEligibilityForCompaction(candidateA) ); Assertions.assertEquals( CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidateB, null) + policy.checkEligibilityForCompaction(candidateB) ); Assertions.assertEquals(0, policy.compareCandidates(candidateA, candidateB)); @@ -280,7 +280,7 @@ public void test_checkEligibilityForCompaction_returnsIncrementalCompaction_when Assertions.assertEquals( CompactionEligibility.incrementalCompaction( "Uncompacted bytes ratio[0.25] is below threshold[0.50]"), - policy.checkEligibilityForCompaction(candidate, null) + policy.checkEligibilityForCompaction(candidate) ); } @@ -300,7 +300,7 @@ public void test_checkEligibilityForCompaction_returnsFullCompaction_whenRatioAb Assertions.assertEquals( CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidate, null) + policy.checkEligibilityForCompaction(candidate) ); } @@ -321,7 +321,7 @@ public void test_checkEligibilityForCompaction_returnsFullCompaction_whenThresho Assertions.assertEquals( CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidate, null) + policy.checkEligibilityForCompaction(candidate) ); } From 502854dc02edcc66286a877c6c3cc29354eeec3c Mon Sep 17 00:00:00 2001 From: cecemei Date: Thu, 5 Feb 2026 16:34:42 -0800 Subject: [PATCH 09/19] eligible --- .../compaction/BaseCandidateSearchPolicy.java | 7 +- .../compaction/CompactionCandidate.java | 309 +++--- .../CompactionCandidateSearchPolicy.java | 10 +- .../compaction/CompactionEligibility.java | 860 ++++++++++++++++- .../server/compaction/CompactionStatus.java | 751 +-------------- .../DataSourceCompactibleSegmentIterator.java | 54 +- .../compaction/FixedIntervalOrderPolicy.java | 13 +- .../MostFragmentedIntervalFirstPolicy.java | 32 +- .../DataSourceCompactionConfig.java | 4 +- .../ClientCompactionIntervalSpecTest.java | 22 +- .../compaction/CompactionCandidateTest.java | 141 +++ .../CompactionEligibilityEvaluateTest.java | 895 ++++++++++++++++++ .../compaction/CompactionEligibilityTest.java | 173 +++- .../compaction/CompactionStatusTest.java | 857 +---------------- .../CompactionStatusTrackerTest.java | 33 +- ...MostFragmentedIntervalFirstPolicyTest.java | 193 ++-- .../NewestSegmentFirstPolicyTest.java | 34 +- .../coordinator/duty/CompactSegmentsTest.java | 12 +- 18 files changed, 2442 insertions(+), 1958 deletions(-) create mode 100644 server/src/test/java/org/apache/druid/server/compaction/CompactionCandidateTest.java create mode 100644 server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityEvaluateTest.java diff --git a/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java index 4d0e22d6d82e..0400e7e5ad61 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java @@ -68,9 +68,12 @@ public final int compareCandidates(CompactionCandidate o1, CompactionCandidate o } @Override - public CompactionEligibility checkEligibilityForCompaction(CompactionCandidate candidate) + public CompactionEligibility checkEligibilityForCompaction( + CompactionCandidate.ProposedCompaction candidate, + CompactionEligibility eligibility + ) { - return CompactionEligibility.FULL_COMPACTION_ELIGIBLE; + return eligibility; } /** diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java index 8bad730d71a9..d3c12b4a858d 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java @@ -19,20 +19,16 @@ package org.apache.druid.server.compaction; -import org.apache.druid.error.DruidException; +import com.google.common.base.Preconditions; import org.apache.druid.error.InvalidInput; import org.apache.druid.java.util.common.JodaUtils; -import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.segment.SegmentUtils; -import org.apache.druid.segment.metadata.IndexingStateFingerprintMapper; -import org.apache.druid.server.coordinator.DataSourceCompactionConfig; import org.apache.druid.timeline.DataSegment; import org.joda.time.Interval; import javax.annotation.Nullable; import java.util.List; -import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -42,62 +38,25 @@ */ public class CompactionCandidate { - private final List segments; - private final Interval umbrellaInterval; - private final Interval compactionInterval; - private final String dataSource; - private final long totalBytes; - private final int numIntervals; + private final ProposedCompaction proposedCompaction; private final CompactionEligibility policyEligiblity; private final CompactionStatus currentStatus; - public static CompactionCandidate from( - List segments, - @Nullable Granularity targetSegmentGranularity + CompactionCandidate( + ProposedCompaction proposedCompaction, + CompactionEligibility policyEligiblity, + CompactionStatus currentStatus ) { - if (segments == null || segments.isEmpty()) { - throw InvalidInput.exception("Segments to compact must be non-empty"); - } - - final Set segmentIntervals = - segments.stream().map(DataSegment::getInterval).collect(Collectors.toSet()); - final Interval umbrellaInterval = JodaUtils.umbrellaInterval(segmentIntervals); - final Interval compactionInterval = - targetSegmentGranularity == null - ? umbrellaInterval - : JodaUtils.umbrellaInterval(targetSegmentGranularity.getIterable(umbrellaInterval)); - - return new CompactionCandidate( - segments, - umbrellaInterval, - compactionInterval, - segmentIntervals.size(), - null, - null - ); + this.proposedCompaction = Preconditions.checkNotNull(proposedCompaction, "proposedCompaction"); + this.policyEligiblity = Preconditions.checkNotNull(policyEligiblity, "policyEligiblity"); + this.currentStatus = Preconditions.checkNotNull(currentStatus, "currentStatus"); } - private CompactionCandidate( - List segments, - Interval umbrellaInterval, - Interval compactionInterval, - int numDistinctSegmentIntervals, - CompactionEligibility policyEligiblity, - @Nullable CompactionStatus currentStatus - ) + public ProposedCompaction getProposedCompaction() { - this.segments = segments; - this.totalBytes = segments.stream().mapToLong(DataSegment::getSize).sum(); - - this.umbrellaInterval = umbrellaInterval; - this.compactionInterval = compactionInterval; - - this.numIntervals = numDistinctSegmentIntervals; - this.dataSource = segments.get(0).getDataSource(); - this.policyEligiblity = policyEligiblity; - this.currentStatus = currentStatus; + return proposedCompaction; } /** @@ -105,17 +64,17 @@ private CompactionCandidate( */ public List getSegments() { - return segments; + return proposedCompaction.getSegments(); } public long getTotalBytes() { - return totalBytes; + return proposedCompaction.getTotalBytes(); } public int numSegments() { - return segments.size(); + return proposedCompaction.numSegments(); } /** @@ -124,40 +83,26 @@ public int numSegments() */ public Interval getUmbrellaInterval() { - return umbrellaInterval; + return proposedCompaction.getUmbrellaInterval(); } /** * Interval aligned to the target segment granularity used for the compaction - * task. This interval completely contains the {@link #umbrellaInterval}. + * task. This interval completely contains the {@link #getUmbrellaInterval()}. */ public Interval getCompactionInterval() { - return compactionInterval; + return proposedCompaction.getCompactionInterval(); } public String getDataSource() { - return dataSource; + return proposedCompaction.getDataSource(); } public CompactionStatistics getStats() { - return CompactionStatistics.create(totalBytes, numSegments(), numIntervals); - } - - @Nullable - public CompactionStatistics getCompactedStats() - { - return (currentStatus == null || currentStatus.getCompactedStats() == null) - ? null : currentStatus.getCompactedStats(); - } - - @Nullable - public CompactionStatistics getUncompactedStats() - { - return (currentStatus == null || currentStatus.getUncompactedStats() == null) - ? null : currentStatus.getUncompactedStats(); + return proposedCompaction.getStats(); } /** @@ -180,113 +125,131 @@ public CompactionEligibility getPolicyEligibility() */ public CompactionCandidate withCurrentStatus(CompactionStatus status) { - return new CompactionCandidate( - segments, - umbrellaInterval, - compactionInterval, - numIntervals, - policyEligiblity, - status - ); - } - - public CompactionCandidate withPolicyEligibility(CompactionEligibility eligibility) - { - return new CompactionCandidate( - segments, - umbrellaInterval, - compactionInterval, - numIntervals, - eligibility, - currentStatus - ); - } - - /** - * Evaluates this candidate for compaction eligibility based on the provided - * compaction configuration and search policy. - *

- * This method first evaluates the candidate against the compaction configuration - * using a {@link CompactionStatus.Evaluator} to determine if any segments need - * compaction. If segments are pending compaction, the search policy is consulted - * to determine the type of compaction: - *

    - *
  • NOT_ELIGIBLE: Returns a candidate with status SKIPPED, indicating - * the policy decided compaction should not occur at this time
    • - *
  • FULL_COMPACTION: Returns this candidate with status PENDING, - * indicating all segments should be compacted
    • - *
  • INCREMENTAL_COMPACTION: Returns a new candidate containing only - * the uncompacted segments (as determined by the evaluator), with status - * PENDING for incremental compaction
    • - *
- * - * @param config the compaction configuration for the datasource - * @param searchPolicy the policy used to determine compaction eligibility - * @return a CompactionCandidate with updated status and potentially filtered segments - */ - public CompactionCandidate evaluate( - DataSourceCompactionConfig config, - CompactionCandidateSearchPolicy searchPolicy, - IndexingStateFingerprintMapper fingerprintMapper - ) - { - CompactionStatus.Evaluator evaluator = new CompactionStatus.Evaluator(this, config, fingerprintMapper); - Pair evaluated = evaluator.evaluate(); - switch (Objects.requireNonNull(evaluated.lhs).getState()) { - case NOT_APPLICABLE: - case NOT_ELIGIBLE: - return this.withPolicyEligibility(evaluated.lhs).withCurrentStatus(evaluated.rhs); - case FULL_COMPACTION: // evaluator has decided compaction is needed, policy needs to further check - if (!evaluated.rhs.getState().equals(CompactionStatus.State.PENDING)) { - throw DruidException.defensive( - "Evaluated compaction status should be PENDING, got status[%s] instead.", - evaluated.rhs.getState() - ); - } - final CompactionEligibility searchPolicyEligibility = - searchPolicy.checkEligibilityForCompaction(this.withPolicyEligibility(evaluated.lhs) - .withCurrentStatus(evaluated.rhs)); - switch (searchPolicyEligibility.getState()) { - case - NOT_ELIGIBLE: // although evaluator thinks this interval qualifies for compaction, but policy decided not its turn yet. - return this.withPolicyEligibility(searchPolicyEligibility) - .withCurrentStatus(CompactionStatus.skipped( - "Rejected by search policy: %s", - searchPolicyEligibility.getReason() - )); - case FULL_COMPACTION: - return this.withPolicyEligibility(searchPolicyEligibility).withCurrentStatus(evaluated.rhs); - case - INCREMENTAL_COMPACTION: // policy decided to perform an incremental compaction, the uncompactedSegments is a subset of the original segments. - return new CompactionCandidate( - evaluator.getUncompactedSegments(), - umbrellaInterval, - compactionInterval, - numIntervals, - searchPolicyEligibility, - evaluated.rhs - ); - default: - throw DruidException.defensive("Unexpected eligibility[%s] from policy", searchPolicyEligibility); - } - case INCREMENTAL_COMPACTION: // evaluator cant decide when to perform an incremental compaction - default: - throw DruidException.defensive("Unexpected eligibility[%s]", evaluated.rhs); - } + return new CompactionCandidate(proposedCompaction, policyEligiblity, status); } @Override public String toString() { return "SegmentsToCompact{" + - "datasource=" + dataSource + - ", umbrellaInterval=" + umbrellaInterval + - ", compactionInterval=" + compactionInterval + - ", numIntervals=" + numIntervals + - ", segments=" + SegmentUtils.commaSeparatedIdentifiers(segments) + - ", totalSize=" + totalBytes + + ", proposedCompaction=" + proposedCompaction + ", policyEligiblity=" + policyEligiblity + ", currentStatus=" + currentStatus + '}'; } + + /** + * Non-empty list of segments of a datasource being proposed for compaction. + * A proposed compaction typically contains all the segments of a single time chunk. + */ + public static class ProposedCompaction + { + private final List segments; + private final Interval umbrellaInterval; + private final Interval compactionInterval; + private final String dataSource; + private final long totalBytes; + private final int numIntervals; + + public static ProposedCompaction from( + List segments, + @Nullable Granularity targetSegmentGranularity + ) + { + if (segments == null || segments.isEmpty()) { + throw InvalidInput.exception("Segments to compact must be non-empty"); + } + + final Set segmentIntervals = + segments.stream().map(DataSegment::getInterval).collect(Collectors.toSet()); + final Interval umbrellaInterval = JodaUtils.umbrellaInterval(segmentIntervals); + final Interval compactionInterval = + targetSegmentGranularity == null + ? umbrellaInterval + : JodaUtils.umbrellaInterval(targetSegmentGranularity.getIterable(umbrellaInterval)); + + return new ProposedCompaction( + segments, + umbrellaInterval, + compactionInterval, + segmentIntervals.size() + ); + } + + ProposedCompaction( + List segments, + Interval umbrellaInterval, + Interval compactionInterval, + int numDistinctSegmentIntervals + ) + { + this.segments = segments; + this.totalBytes = segments.stream().mapToLong(DataSegment::getSize).sum(); + + this.umbrellaInterval = umbrellaInterval; + this.compactionInterval = compactionInterval; + + this.numIntervals = numDistinctSegmentIntervals; + this.dataSource = segments.get(0).getDataSource(); + } + + /** + * @return Non-empty list of segments that make up this proposed compaction. + */ + public List getSegments() + { + return segments; + } + + public long getTotalBytes() + { + return totalBytes; + } + + public int numSegments() + { + return segments.size(); + } + + /** + * Umbrella interval of all the segments in this proposed compaction. This typically + * corresponds to a single time chunk in the segment timeline. + */ + public Interval getUmbrellaInterval() + { + return umbrellaInterval; + } + + /** + * Interval aligned to the target segment granularity used for the compaction + * task. This interval completely contains the {@link #umbrellaInterval}. + */ + public Interval getCompactionInterval() + { + return compactionInterval; + } + + public String getDataSource() + { + return dataSource; + } + + public CompactionStatistics getStats() + { + return CompactionStatistics.create(totalBytes, numSegments(), numIntervals); + } + + @Override + public String toString() + { + return "ProposedCompaction{" + + "datasource=" + dataSource + + ", umbrellaInterval=" + umbrellaInterval + + ", compactionInterval=" + compactionInterval + + ", numIntervals=" + numIntervals + + ", segments=" + SegmentUtils.commaSeparatedIdentifiers(segments) + + ", totalSize=" + totalBytes + + '}'; + } + } } diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java index 744de2897f6c..1cd65b3a53ad 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java @@ -49,5 +49,13 @@ public interface CompactionCandidateSearchPolicy * in the current iteration. A policy may implement this method to skip * compacting intervals or segments that do not fulfil some required criteria. */ - CompactionEligibility checkEligibilityForCompaction(CompactionCandidate candidate); + default CompactionEligibility checkEligibilityForCompaction(CompactionCandidate candidate) + { + return checkEligibilityForCompaction(candidate.getProposedCompaction(), candidate.getPolicyEligibility()); + } + + CompactionEligibility checkEligibilityForCompaction( + CompactionCandidate.ProposedCompaction candidate, + CompactionEligibility eligibility + ); } diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionEligibility.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionEligibility.java index eab13ed39662..30786e0ee765 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionEligibility.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionEligibility.java @@ -19,9 +19,42 @@ package org.apache.druid.server.compaction; +import com.google.common.base.Strings; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig; +import org.apache.druid.common.config.Configs; +import org.apache.druid.data.input.impl.DimensionSchema; +import org.apache.druid.error.DruidException; +import org.apache.druid.error.InvalidInput; +import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec; +import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; +import org.apache.druid.indexer.partitions.HashedPartitionsSpec; +import org.apache.druid.indexer.partitions.PartitionsSpec; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.granularity.Granularity; +import org.apache.druid.java.util.common.granularity.GranularityType; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.segment.IndexSpec; +import org.apache.druid.segment.metadata.IndexingStateFingerprintMapper; +import org.apache.druid.segment.transform.CompactionTransformSpec; +import org.apache.druid.server.coordinator.DataSourceCompactionConfig; +import org.apache.druid.server.coordinator.UserCompactionTaskGranularityConfig; +import org.apache.druid.timeline.CompactionState; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.utils.CollectionUtils; +import org.joda.time.Interval; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; /** * Describes the eligibility of an interval for compaction. @@ -36,16 +69,94 @@ public enum State NOT_APPLICABLE } - public static final CompactionEligibility FULL_COMPACTION_ELIGIBLE = new CompactionEligibility(State.FULL_COMPACTION, null); - public static final CompactionEligibility NOT_APPLICABLE = new CompactionEligibility(State.NOT_APPLICABLE, null); + static class CompactionEligibilityBuilder + { + private State state; + private CompactionStatistics compacted; + private CompactionStatistics uncompacted; + private List uncompactedSegments; + private String reason; + + CompactionEligibilityBuilder(State state, String reason) + { + this.state = state; + this.reason = reason; + } + + CompactionEligibilityBuilder compacted(CompactionStatistics compacted) + { + this.compacted = compacted; + return this; + } + + CompactionEligibilityBuilder uncompacted(CompactionStatistics uncompacted) + { + this.uncompacted = uncompacted; + return this; + } + + CompactionEligibilityBuilder uncompactedSegments(List uncompactedSegments) + { + this.uncompactedSegments = uncompactedSegments; + return this; + } + + CompactionEligibility build() + { + return new CompactionEligibility(state, reason, compacted, uncompacted, uncompactedSegments); + } + } + + public static final CompactionEligibility NOT_APPLICABLE = builder(State.NOT_APPLICABLE, "").build(); + + public static CompactionEligibility fail(String messageFormat, Object... args) + { + return builder(State.NOT_ELIGIBLE, StringUtils.format(messageFormat, args)).build(); + } private final State state; private final String reason; - private CompactionEligibility(State state, String reason) + @Nullable + private final CompactionStatistics compacted; + @Nullable + private final CompactionStatistics uncompacted; + @Nullable + private final List uncompactedSegments; + + private CompactionEligibility( + State state, + String reason, + @Nullable CompactionStatistics compacted, + @Nullable CompactionStatistics uncompacted, + @Nullable List uncompactedSegments + ) { this.state = state; this.reason = reason; + switch (state) { + case NOT_APPLICABLE: + break; + case NOT_ELIGIBLE: + InvalidInput.conditionalException(!Strings.isNullOrEmpty(reason), "must provide a reason"); + break; + case FULL_COMPACTION: + case INCREMENTAL_COMPACTION: + InvalidInput.conditionalException(compacted != null, "must provide compacted stats"); + InvalidInput.conditionalException(uncompacted != null, "must provide uncompacted stats"); + InvalidInput.conditionalException(uncompactedSegments != null, "must provide uncompactedSegments"); + break; + default: + throw DruidException.defensive("unexpected eligibility state[%s]", state); + } + this.compacted = compacted; + this.uncompacted = uncompacted; + this.uncompactedSegments = uncompactedSegments; + } + + static CompactionEligibilityBuilder builder(State state, String reason) + { + return new CompactionEligibilityBuilder(state, reason); } public State getState() @@ -58,14 +169,87 @@ public String getReason() return reason; } - public static CompactionEligibility incrementalCompaction(String messageFormat, Object... args) + @Nullable + public CompactionStatistics getUncompactedStats() { - return new CompactionEligibility(State.INCREMENTAL_COMPACTION, StringUtils.format(messageFormat, args)); + return uncompacted; } - public static CompactionEligibility fail(String messageFormat, Object... args) + @Nullable + public CompactionStatistics getCompactedStats() + { + return compacted; + } + + @Nullable + public List getUncompactedSegments() + { + return uncompactedSegments; + } + + public CompactionCandidate createCandidate(CompactionCandidate.ProposedCompaction proposedCompaction) { - return new CompactionEligibility(State.NOT_ELIGIBLE, StringUtils.format(messageFormat, args)); + switch (state) { + case NOT_APPLICABLE: + return new CompactionCandidate(proposedCompaction, this, CompactionStatus.COMPLETE); + case NOT_ELIGIBLE: + return new CompactionCandidate(proposedCompaction, this, CompactionStatus.skipped(reason)); + case FULL_COMPACTION: + return new CompactionCandidate( + proposedCompaction, + this, + CompactionStatus.pending(reason) + ); + case INCREMENTAL_COMPACTION: + CompactionCandidate.ProposedCompaction newProposed = new CompactionCandidate.ProposedCompaction( + uncompactedSegments, + proposedCompaction.getUmbrellaInterval(), + proposedCompaction.getCompactionInterval(), + Math.toIntExact(uncompactedSegments.stream().map(DataSegment::getInterval).distinct().count()) + ); + return new CompactionCandidate(newProposed, this, CompactionStatus.pending(reason)); + default: + throw DruidException.defensive("Unexpected eligibility state[%s]", state); + } + } + + /** + * Evaluates a compaction candidate to determine its eligibility and compaction status. + *

+ * This method performs a two-stage evaluation: + *

    + *
  1. First, uses {@link Evaluator} to check if the candidate needs compaction + * based on the compaction config (e.g., checking segment granularity, partitions spec, etc.)
    2. + *
  2. Then, applies the search policy to determine if this candidate should be compacted in the + * current run (e.g., checking minimum segment count, bytes, or other policy criteria)
    3. + *
+ * + * @param proposedCompaction the compaction candidate to evaluate + * @param config the compaction configuration for the datasource + * @param searchPolicy the policy that determines candidate ordering and eligibility + * @param fingerprintMapper mapper for indexing state fingerprints + * @return a new {@link CompactionCandidate} with updated eligibility and status. For incremental + * compaction, returns a candidate containing only the uncompacted segments. + */ + public static CompactionEligibility evaluate( + CompactionCandidate.ProposedCompaction proposedCompaction, + DataSourceCompactionConfig config, + CompactionCandidateSearchPolicy searchPolicy, + IndexingStateFingerprintMapper fingerprintMapper + ) + { + // ideally we should let this class only decides CompactionEligibility, and the callsite should handle recreation of candidate. + CompactionEligibility evaluatedCandidate = new Evaluator(proposedCompaction, config, fingerprintMapper).evaluate(); + switch (Objects.requireNonNull(evaluatedCandidate).getState()) { + case NOT_APPLICABLE: + case NOT_ELIGIBLE: + return evaluatedCandidate; + case FULL_COMPACTION: // evaluator has decided compaction is needed, policy needs to further check + return searchPolicy.checkEligibilityForCompaction(proposedCompaction, evaluatedCandidate); + case INCREMENTAL_COMPACTION: // evaluator cant decide when to perform an incremental compaction + default: + throw DruidException.defensive("Unexpected eligibility[%s]", evaluatedCandidate); + } } @Override @@ -78,21 +262,671 @@ public boolean equals(Object object) return false; } CompactionEligibility that = (CompactionEligibility) object; - return state == that.state && Objects.equals(reason, that.reason); + return state == that.state + && Objects.equals(reason, that.reason) + && Objects.equals(compacted, that.compacted) + && Objects.equals(uncompacted, that.uncompacted) + && Objects.equals(uncompactedSegments, that.uncompactedSegments); } @Override public int hashCode() { - return Objects.hash(state, reason); + return Objects.hash(state, reason, compacted, uncompacted, uncompactedSegments); } @Override public String toString() { - return "Eligibility{" + - "state=" + state + - ", reason='" + reason + '\'' + - '}'; + return "CompactionEligibility{" + + "state=" + state + + ", reason='" + reason + '\'' + + ", compacted=" + compacted + + ", uncompacted=" + uncompacted + + ", uncompactedSegments=" + uncompactedSegments + + '}'; + } + + /** + * List of checks performed to determine if compaction is already complete based on indexing state fingerprints. + */ + static final List> FINGERPRINT_CHECKS = List.of( + Evaluator::allFingerprintedCandidatesHaveExpectedFingerprint + ); + + /** + * List of checks performed to determine if compaction is already complete. + *

+ * The order of the checks must be honored while evaluating them. + */ + static final List> CHECKS = Arrays.asList( + Evaluator::partitionsSpecIsUpToDate, + Evaluator::indexSpecIsUpToDate, + Evaluator::segmentGranularityIsUpToDate, + Evaluator::queryGranularityIsUpToDate, + Evaluator::rollupIsUpToDate, + Evaluator::dimensionsSpecIsUpToDate, + Evaluator::metricsSpecIsUpToDate, + Evaluator::transformSpecFilterIsUpToDate, + Evaluator::projectionsAreUpToDate + ); + + /** + * Evaluates checks to determine the compaction status of a + * {@link CompactionCandidate}. + */ + private static class Evaluator + { + private static final Logger log = new Logger(Evaluator.class); + + private final DataSourceCompactionConfig compactionConfig; + private final CompactionCandidate.ProposedCompaction proposedCompaction; + private final ClientCompactionTaskQueryTuningConfig tuningConfig; + private final UserCompactionTaskGranularityConfig configuredGranularitySpec; + + private final List fingerprintedSegments = new ArrayList<>(); + private final List compactedSegments = new ArrayList<>(); + private final List uncompactedSegments = new ArrayList<>(); + private final Map> unknownStateToSegments = new HashMap<>(); + + @Nullable + private final IndexingStateFingerprintMapper fingerprintMapper; + @Nullable + private final String targetFingerprint; + + private Evaluator( + CompactionCandidate.ProposedCompaction proposedCompaction, + DataSourceCompactionConfig compactionConfig, + @Nullable IndexingStateFingerprintMapper fingerprintMapper + ) + { + this.proposedCompaction = proposedCompaction; + this.compactionConfig = compactionConfig; + this.tuningConfig = ClientCompactionTaskQueryTuningConfig.from(compactionConfig); + this.configuredGranularitySpec = compactionConfig.getGranularitySpec(); + this.fingerprintMapper = fingerprintMapper; + if (fingerprintMapper == null) { + targetFingerprint = null; + } else { + targetFingerprint = fingerprintMapper.generateFingerprint( + compactionConfig.getDataSource(), + compactionConfig.toCompactionState() + ); + } + } + + /** + * Evaluates the compaction status of candidate segments through a multi-step process: + *

    + *
  1. Validates input bytes are within limits
    2. + *
  2. Categorizes segments by compaction state (fingerprinted, uncompacted, or unknown)
    3. + *
  3. Performs fingerprint-based validation if available (fast path)
    4. + *
  4. Runs detailed checks against unknown states via {@link CompactionEligibility#CHECKS}
    5. + *
+ * + * @return Pair of eligibility status and compaction status with reason for first failed check + */ + private CompactionEligibility evaluate() + { + final String inputBytesCheck = inputBytesAreWithinLimit(); + if (inputBytesCheck != null) { + return CompactionEligibility.fail(inputBytesCheck); + } + + List reasonsForCompaction = new ArrayList<>(); + String compactedOnceCheck = segmentsHaveBeenCompactedAtLeastOnce(); + if (compactedOnceCheck != null) { + reasonsForCompaction.add(compactedOnceCheck); + } + + if (fingerprintMapper != null && targetFingerprint != null) { + // First try fingerprint-based evaluation (fast path) + FINGERPRINT_CHECKS.stream() + .map(f -> f.apply(this)) + .filter(Objects::nonNull) + .findFirst() + .ifPresent(reasonsForCompaction::add); + + } + + if (!unknownStateToSegments.isEmpty()) { + // Run CHECKS against any states with uknown compaction status + reasonsForCompaction.addAll( + CHECKS.stream() + .map(f -> f.apply(this)) + .filter(Objects::nonNull) + .collect(Collectors.toList()) + ); + + // Any segments left in unknownStateToSegments passed all checks and are considered compacted + compactedSegments.addAll( + unknownStateToSegments + .values() + .stream() + .flatMap(List::stream) + .collect(Collectors.toList()) + ); + } + + if (reasonsForCompaction.isEmpty()) { + return CompactionEligibility.NOT_APPLICABLE; + } else { + return builder(State.FULL_COMPACTION, reasonsForCompaction.get(0)).compacted(createStats(compactedSegments)) + .uncompacted(createStats(uncompactedSegments)) + .uncompactedSegments(uncompactedSegments) + .build(); + } + } + + /** + * Evaluates the fingerprints of all fingerprinted candidate segments against the expected fingerprint. + *

+ * If all fingerprinted segments have the expected fingerprint, the check can quickly pass as COMPLETE. However, + * if any fingerprinted segment has a mismatched fingerprint, we need to investigate further by adding them to + * {@link #unknownStateToSegments} where their indexing states will be analyzed. + *

+ */ + private String allFingerprintedCandidatesHaveExpectedFingerprint() + { + Map> mismatchedFingerprintToSegmentMap = new HashMap<>(); + for (DataSegment segment : fingerprintedSegments) { + String fingerprint = segment.getIndexingStateFingerprint(); + if (fingerprint == null) { + // Should not happen since we are iterating over fingerprintedSegments + } else if (fingerprint.equals(targetFingerprint)) { + compactedSegments.add(segment); + } else { + mismatchedFingerprintToSegmentMap + .computeIfAbsent(fingerprint, k -> new ArrayList<>()) + .add(segment); + } + } + + if (mismatchedFingerprintToSegmentMap.isEmpty()) { + // All fingerprinted segments have the expected fingerprint - compaction is complete + return null; + } + + if (fingerprintMapper == null) { + // Cannot evaluate further without a fingerprint mapper + uncompactedSegments.addAll( + mismatchedFingerprintToSegmentMap.values() + .stream() + .flatMap(List::stream) + .collect(Collectors.toList()) + ); + return "Segments have a mismatched fingerprint and no fingerprint mapper is available"; + } + + boolean fingerprintedSegmentWithoutCachedStateFound = false; + + for (Map.Entry> e : mismatchedFingerprintToSegmentMap.entrySet()) { + String fingerprint = e.getKey(); + CompactionState stateToValidate = fingerprintMapper.getStateForFingerprint(fingerprint).orElse(null); + if (stateToValidate == null) { + log.warn("No indexing state found for fingerprint[%s]", fingerprint); + fingerprintedSegmentWithoutCachedStateFound = true; + uncompactedSegments.addAll(e.getValue()); + } else { + // Note that this does not mean we need compaction yet - we need to validate the state further to determine this + unknownStateToSegments.compute( + stateToValidate, + (state, segments) -> { + if (segments == null) { + segments = new ArrayList<>(); + } + segments.addAll(e.getValue()); + return segments; + } + ); + } + } + + if (fingerprintedSegmentWithoutCachedStateFound) { + return "One or more fingerprinted segments do not have a cached indexing state"; + } else { + return null; + } + } + + /** + * Checks if all the segments have been compacted at least once and groups them into uncompacted, fingerprinted, or + * non-fingerprinted. + */ + private String segmentsHaveBeenCompactedAtLeastOnce() + { + for (DataSegment segment : proposedCompaction.getSegments()) { + final String fingerprint = segment.getIndexingStateFingerprint(); + final CompactionState segmentState = segment.getLastCompactionState(); + if (fingerprint != null) { + fingerprintedSegments.add(segment); + } else if (segmentState == null) { + uncompactedSegments.add(segment); + } else { + unknownStateToSegments.computeIfAbsent(segmentState, s -> new ArrayList<>()).add(segment); + } + } + + if (uncompactedSegments.isEmpty()) { + return null; + } else { + return "not compacted yet"; + } + } + + private String partitionsSpecIsUpToDate() + { + return evaluateForAllCompactionStates(this::partitionsSpecIsUpToDate); + } + + private String indexSpecIsUpToDate() + { + return evaluateForAllCompactionStates(this::indexSpecIsUpToDate); + } + + private String projectionsAreUpToDate() + { + return evaluateForAllCompactionStates(this::projectionsAreUpToDate); + } + + private String segmentGranularityIsUpToDate() + { + return evaluateForAllCompactionStates(this::segmentGranularityIsUpToDate); + } + + private String rollupIsUpToDate() + { + return evaluateForAllCompactionStates(this::rollupIsUpToDate); + } + + private String queryGranularityIsUpToDate() + { + return evaluateForAllCompactionStates(this::queryGranularityIsUpToDate); + } + + private String dimensionsSpecIsUpToDate() + { + return evaluateForAllCompactionStates(this::dimensionsSpecIsUpToDate); + } + + private String metricsSpecIsUpToDate() + { + return evaluateForAllCompactionStates(this::metricsSpecIsUpToDate); + } + + private String transformSpecFilterIsUpToDate() + { + return evaluateForAllCompactionStates(this::transformSpecFilterIsUpToDate); + } + + private String partitionsSpecIsUpToDate(CompactionState lastCompactionState) + { + PartitionsSpec existingPartionsSpec = lastCompactionState.getPartitionsSpec(); + if (existingPartionsSpec instanceof DimensionRangePartitionsSpec) { + existingPartionsSpec = getEffectiveRangePartitionsSpec((DimensionRangePartitionsSpec) existingPartionsSpec); + } else if (existingPartionsSpec instanceof DynamicPartitionsSpec) { + existingPartionsSpec = new DynamicPartitionsSpec( + existingPartionsSpec.getMaxRowsPerSegment(), + ((DynamicPartitionsSpec) existingPartionsSpec).getMaxTotalRowsOr(Long.MAX_VALUE) + ); + } + return completeIfNullOrEqual( + "partitionsSpec", + findPartitionsSpecFromConfig(tuningConfig), + existingPartionsSpec, + CompactionEligibility::asString + ); + } + + private String indexSpecIsUpToDate(CompactionState lastCompactionState) + { + return completeIfNullOrEqual( + "indexSpec", + Configs.valueOrDefault(tuningConfig.getIndexSpec(), IndexSpec.getDefault()).getEffectiveSpec(), + lastCompactionState.getIndexSpec().getEffectiveSpec(), + String::valueOf + ); + } + + private String projectionsAreUpToDate(CompactionState lastCompactionState) + { + return completeIfNullOrEqual( + "projections", + compactionConfig.getProjections(), + lastCompactionState.getProjections(), + String::valueOf + ); + } + + @Nullable + private String inputBytesAreWithinLimit() + { + final long inputSegmentSize = compactionConfig.getInputSegmentSizeBytes(); + if (proposedCompaction.getTotalBytes() > inputSegmentSize) { + return StringUtils.format( + "'inputSegmentSize' exceeded: Total segment size[%d] is larger than allowed inputSegmentSize[%d]", + proposedCompaction.getTotalBytes(), inputSegmentSize + ); + } + return null; + } + + private String segmentGranularityIsUpToDate(CompactionState lastCompactionState) + { + if (configuredGranularitySpec == null + || configuredGranularitySpec.getSegmentGranularity() == null) { + return null; + } + + final Granularity configuredSegmentGranularity = configuredGranularitySpec.getSegmentGranularity(); + final UserCompactionTaskGranularityConfig existingGranularitySpec = getGranularitySpec(lastCompactionState); + final Granularity existingSegmentGranularity + = existingGranularitySpec == null ? null : existingGranularitySpec.getSegmentGranularity(); + + if (configuredSegmentGranularity.equals(existingSegmentGranularity)) { + return null; + } else if (existingSegmentGranularity == null) { + // Candidate segments were compacted without segment granularity specified + // Check if the segments already have the desired segment granularity + final List segmentsForState = unknownStateToSegments.get(lastCompactionState); + boolean needsCompaction = segmentsForState.stream().anyMatch( + segment -> !configuredSegmentGranularity.isAligned(segment.getInterval()) + ); + if (needsCompaction) { + return StringUtils.format( + "segmentGranularity: segments do not align with target[%s]", + CompactionEligibility.asString(configuredSegmentGranularity) + ); + } + } else { + return configChanged( + "segmentGranularity", + configuredSegmentGranularity, + existingSegmentGranularity, + CompactionEligibility::asString + ); + } + + return null; + } + + private String rollupIsUpToDate(CompactionState lastCompactionState) + { + if (configuredGranularitySpec == null) { + return null; + } else { + final UserCompactionTaskGranularityConfig existingGranularitySpec + = getGranularitySpec(lastCompactionState); + return completeIfNullOrEqual( + "rollup", + configuredGranularitySpec.isRollup(), + existingGranularitySpec == null ? null : existingGranularitySpec.isRollup(), + String::valueOf + ); + } + } + + private String queryGranularityIsUpToDate(CompactionState lastCompactionState) + { + if (configuredGranularitySpec == null) { + return null; + } else { + final UserCompactionTaskGranularityConfig existingGranularitySpec + = getGranularitySpec(lastCompactionState); + return completeIfNullOrEqual( + "queryGranularity", + configuredGranularitySpec.getQueryGranularity(), + existingGranularitySpec == null ? null : existingGranularitySpec.getQueryGranularity(), + CompactionEligibility::asString + ); + } + } + + /** + * Removes partition dimensions before comparison, since they are placed in front of the sort order -- + * which can create a mismatch between expected and actual order of dimensions. Partition dimensions are separately + * covered in {@link Evaluator#partitionsSpecIsUpToDate()} check. + */ + private String dimensionsSpecIsUpToDate(CompactionState lastCompactionState) + { + if (compactionConfig.getDimensionsSpec() == null) { + return null; + } else { + List existingDimensions = getNonPartitioningDimensions( + lastCompactionState.getDimensionsSpec() == null + ? null + : lastCompactionState.getDimensionsSpec().getDimensions(), + lastCompactionState.getPartitionsSpec(), + lastCompactionState.getIndexSpec() + ); + List configuredDimensions = getNonPartitioningDimensions( + compactionConfig.getDimensionsSpec().getDimensions(), + compactionConfig.getTuningConfig() == null ? null : compactionConfig.getTuningConfig().getPartitionsSpec(), + compactionConfig.getTuningConfig() == null + ? IndexSpec.getDefault() + : compactionConfig.getTuningConfig().getIndexSpec() + ); + return completeIfNullOrEqual( + "dimensionsSpec", + configuredDimensions, + existingDimensions, + String::valueOf + ); + } + } + + private String metricsSpecIsUpToDate(CompactionState lastCompactionState) + { + final AggregatorFactory[] configuredMetricsSpec = compactionConfig.getMetricsSpec(); + if (ArrayUtils.isEmpty(configuredMetricsSpec)) { + return null; + } + + final List metricSpecList = lastCompactionState.getMetricsSpec(); + final AggregatorFactory[] existingMetricsSpec + = CollectionUtils.isNullOrEmpty(metricSpecList) + ? null : metricSpecList.toArray(new AggregatorFactory[0]); + + if (existingMetricsSpec == null || !Arrays.deepEquals(configuredMetricsSpec, existingMetricsSpec)) { + return configChanged( + "metricsSpec", + configuredMetricsSpec, + existingMetricsSpec, + Arrays::toString + ); + } else { + return null; + } + } + + private String transformSpecFilterIsUpToDate(CompactionState lastCompactionState) + { + if (compactionConfig.getTransformSpec() == null) { + return null; + } + + CompactionTransformSpec existingTransformSpec = lastCompactionState.getTransformSpec(); + return completeIfNullOrEqual( + "transformSpec filter", + compactionConfig.getTransformSpec().getFilter(), + existingTransformSpec == null ? null : existingTransformSpec.getFilter(), + String::valueOf + ); + } + + /** + * Evaluates the given check for each entry in the {@link #unknownStateToSegments}. + * If any entry fails the given check by returning a status which is not + * COMPLETE, all the segments with that state are moved to {@link #uncompactedSegments}. + * + * @return The first status which is not COMPLETE. + */ + private String evaluateForAllCompactionStates(Function check) + { + String firstIncomplete = null; + for (CompactionState state : List.copyOf(unknownStateToSegments.keySet())) { + final String eligibleReason = check.apply(state); + if (eligibleReason != null) { + uncompactedSegments.addAll(unknownStateToSegments.remove(state)); + if (firstIncomplete == null) { + firstIncomplete = eligibleReason; + } + } + } + + return firstIncomplete; + } + + private static UserCompactionTaskGranularityConfig getGranularitySpec( + CompactionState compactionState + ) + { + return UserCompactionTaskGranularityConfig.from(compactionState.getGranularitySpec()); + } + + private static CompactionStatistics createStats(List segments) + { + final Set segmentIntervals = + segments.stream().map(DataSegment::getInterval).collect(Collectors.toSet()); + final long totalBytes = segments.stream().mapToLong(DataSegment::getSize).sum(); + return CompactionStatistics.create(totalBytes, segments.size(), segmentIntervals.size()); + } + } + + + /** + * Computes compaction status for the given field. The status is assumed to be + * COMPLETE (i.e. no further compaction is required) if the configured value + * of the field is null or equal to the current value. + */ + private static String completeIfNullOrEqual( + String field, + T configured, + T current, + Function stringFunction + ) + { + if (configured == null || configured.equals(current)) { + return null; + } else { + return configChanged(field, configured, current, stringFunction); + } + } + + private static String configChanged( + String field, + T target, + T current, + Function stringFunction + ) + { + return StringUtils.format( + "'%s' mismatch: required[%s], current[%s]", + field, + target == null ? null : stringFunction.apply(target), + current == null ? null : stringFunction.apply(current) + ); + } + + private static String asString(Granularity granularity) + { + if (granularity == null) { + return null; + } + for (GranularityType type : GranularityType.values()) { + if (type.getDefaultGranularity().equals(granularity)) { + return type.toString(); + } + } + return granularity.toString(); + } + + private static String asString(PartitionsSpec partitionsSpec) + { + if (partitionsSpec instanceof DimensionRangePartitionsSpec) { + DimensionRangePartitionsSpec rangeSpec = (DimensionRangePartitionsSpec) partitionsSpec; + return StringUtils.format( + "'range' on %s with %,d rows", + rangeSpec.getPartitionDimensions(), rangeSpec.getTargetRowsPerSegment() + ); + } else if (partitionsSpec instanceof HashedPartitionsSpec) { + HashedPartitionsSpec hashedSpec = (HashedPartitionsSpec) partitionsSpec; + return StringUtils.format( + "'hashed' on %s with %,d rows", + hashedSpec.getPartitionDimensions(), hashedSpec.getTargetRowsPerSegment() + ); + } else if (partitionsSpec instanceof DynamicPartitionsSpec) { + DynamicPartitionsSpec dynamicSpec = (DynamicPartitionsSpec) partitionsSpec; + return StringUtils.format( + "'dynamic' with %,d rows", + dynamicSpec.getMaxRowsPerSegment() + ); + } else { + return partitionsSpec.toString(); + } + } + + @Nullable + public static PartitionsSpec findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig tuningConfig) + { + final PartitionsSpec partitionsSpecFromTuningConfig = tuningConfig.getPartitionsSpec(); + if (partitionsSpecFromTuningConfig == null) { + final Long maxTotalRows = tuningConfig.getMaxTotalRows(); + final Integer maxRowsPerSegment = tuningConfig.getMaxRowsPerSegment(); + + if (maxTotalRows == null && maxRowsPerSegment == null) { + // If not specified, return null so that partitionsSpec is not compared + return null; + } else { + return new DynamicPartitionsSpec(maxRowsPerSegment, maxTotalRows); + } + } else if (partitionsSpecFromTuningConfig instanceof DynamicPartitionsSpec) { + return new DynamicPartitionsSpec( + partitionsSpecFromTuningConfig.getMaxRowsPerSegment(), + ((DynamicPartitionsSpec) partitionsSpecFromTuningConfig).getMaxTotalRowsOr(Long.MAX_VALUE) + ); + } else if (partitionsSpecFromTuningConfig instanceof DimensionRangePartitionsSpec) { + return getEffectiveRangePartitionsSpec((DimensionRangePartitionsSpec) partitionsSpecFromTuningConfig); + } else { + return partitionsSpecFromTuningConfig; + } + } + + @Nullable + private static List getNonPartitioningDimensions( + @Nullable final List dimensionSchemas, + @Nullable final PartitionsSpec partitionsSpec, + @Nullable final IndexSpec indexSpec + ) + { + final IndexSpec effectiveIndexSpec = (indexSpec == null ? IndexSpec.getDefault() : indexSpec).getEffectiveSpec(); + if (dimensionSchemas == null || !(partitionsSpec instanceof DimensionRangePartitionsSpec)) { + if (dimensionSchemas != null) { + return dimensionSchemas.stream() + .map(dim -> dim.getEffectiveSchema(effectiveIndexSpec)) + .collect(Collectors.toList()); + } + return null; + } + + final List partitionsDimensions = ((DimensionRangePartitionsSpec) partitionsSpec).getPartitionDimensions(); + return dimensionSchemas.stream() + .filter(dim -> !partitionsDimensions.contains(dim.getName())) + .map(dim -> dim.getEffectiveSchema(effectiveIndexSpec)) + .collect(Collectors.toList()); + } + + /** + * Converts to have only the effective maxRowsPerSegment to avoid false positives when targetRowsPerSegment is set but + * effectively translates to the same maxRowsPerSegment. + */ + static DimensionRangePartitionsSpec getEffectiveRangePartitionsSpec(DimensionRangePartitionsSpec partitionsSpec) + { + return new DimensionRangePartitionsSpec( + null, + partitionsSpec.getMaxRowsPerSegment(), + partitionsSpec.getPartitionDimensions(), + partitionsSpec.isAssumeGrouped() + ); } } diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java index 0c330a9f6c5f..a163d4dede98 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java @@ -19,94 +19,28 @@ package org.apache.druid.server.compaction; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig; -import org.apache.druid.common.config.Configs; -import org.apache.druid.data.input.impl.DimensionSchema; -import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec; -import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; -import org.apache.druid.indexer.partitions.HashedPartitionsSpec; -import org.apache.druid.indexer.partitions.PartitionsSpec; -import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.java.util.common.granularity.GranularityType; -import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.query.aggregation.AggregatorFactory; -import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.metadata.IndexingStateFingerprintMapper; -import org.apache.druid.segment.transform.CompactionTransformSpec; -import org.apache.druid.server.coordinator.DataSourceCompactionConfig; -import org.apache.druid.server.coordinator.UserCompactionTaskGranularityConfig; -import org.apache.druid.timeline.CompactionState; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.utils.CollectionUtils; -import org.joda.time.Interval; -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; /** * Represents the status of compaction for a given {@link CompactionCandidate}. */ public class CompactionStatus { - private static final Logger log = new Logger(CompactionStatus.class); - - private static final CompactionStatus COMPLETE = new CompactionStatus(State.COMPLETE, null, null, null); + static final CompactionStatus COMPLETE = new CompactionStatus(State.COMPLETE, null); public enum State { COMPLETE, PENDING, RUNNING, SKIPPED } - /** - * List of checks performed to determine if compaction is already complete based on indexing state fingerprints. - */ - private static final List> FINGERPRINT_CHECKS = List.of( - Evaluator::allFingerprintedCandidatesHaveExpectedFingerprint - ); - - /** - * List of checks performed to determine if compaction is already complete. - *

- * The order of the checks must be honored while evaluating them. - */ - private static final List> CHECKS = Arrays.asList( - Evaluator::partitionsSpecIsUpToDate, - Evaluator::indexSpecIsUpToDate, - Evaluator::segmentGranularityIsUpToDate, - Evaluator::queryGranularityIsUpToDate, - Evaluator::rollupIsUpToDate, - Evaluator::dimensionsSpecIsUpToDate, - Evaluator::metricsSpecIsUpToDate, - Evaluator::transformSpecFilterIsUpToDate, - Evaluator::projectionsAreUpToDate - ); - private final State state; private final String reason; - private final CompactionStatistics compactedStats; - private final CompactionStatistics uncompactedStats; - private CompactionStatus( - State state, - String reason, - CompactionStatistics compactedStats, - CompactionStatistics uncompactedStats - ) + private CompactionStatus(State state, String reason) { this.state = state; this.reason = reason; - this.compactedStats = compactedStats; - this.uncompactedStats = uncompactedStats; } public boolean isComplete() @@ -129,702 +63,27 @@ public State getState() return state; } - public CompactionStatistics getCompactedStats() - { - return compactedStats; - } - - public CompactionStatistics getUncompactedStats() - { - return uncompactedStats; - } - @Override public String toString() { return "CompactionStatus{" + "state=" + state + ", reason=" + reason + - ", compactedStats=" + compactedStats + - ", uncompactedStats=" + uncompactedStats + '}'; } public static CompactionStatus pending(String reasonFormat, Object... args) { - return new CompactionStatus(State.PENDING, StringUtils.format(reasonFormat, args), null, null); - } - - public static CompactionStatus pending( - CompactionStatistics compactedStats, - CompactionStatistics uncompactedStats, - String reasonFormat, - Object... args - ) - { - return new CompactionStatus( - State.PENDING, - StringUtils.format(reasonFormat, args), - compactedStats, - uncompactedStats - ); - } - - /** - * Computes compaction status for the given field. The status is assumed to be - * COMPLETE (i.e. no further compaction is required) if the configured value - * of the field is null or equal to the current value. - */ - private static CompactionStatus completeIfNullOrEqual( - String field, - T configured, - T current, - Function stringFunction - ) - { - if (configured == null || configured.equals(current)) { - return COMPLETE; - } else { - return configChanged(field, configured, current, stringFunction); - } - } - - private static CompactionStatus configChanged( - String field, - T target, - T current, - Function stringFunction - ) - { - return CompactionStatus.pending( - "'%s' mismatch: required[%s], current[%s]", - field, - target == null ? null : stringFunction.apply(target), - current == null ? null : stringFunction.apply(current) - ); - } - - private static String asString(Granularity granularity) - { - if (granularity == null) { - return null; - } - for (GranularityType type : GranularityType.values()) { - if (type.getDefaultGranularity().equals(granularity)) { - return type.toString(); - } - } - return granularity.toString(); - } - - private static String asString(PartitionsSpec partitionsSpec) - { - if (partitionsSpec instanceof DimensionRangePartitionsSpec) { - DimensionRangePartitionsSpec rangeSpec = (DimensionRangePartitionsSpec) partitionsSpec; - return StringUtils.format( - "'range' on %s with %,d rows", - rangeSpec.getPartitionDimensions(), rangeSpec.getTargetRowsPerSegment() - ); - } else if (partitionsSpec instanceof HashedPartitionsSpec) { - HashedPartitionsSpec hashedSpec = (HashedPartitionsSpec) partitionsSpec; - return StringUtils.format( - "'hashed' on %s with %,d rows", - hashedSpec.getPartitionDimensions(), hashedSpec.getTargetRowsPerSegment() - ); - } else if (partitionsSpec instanceof DynamicPartitionsSpec) { - DynamicPartitionsSpec dynamicSpec = (DynamicPartitionsSpec) partitionsSpec; - return StringUtils.format( - "'dynamic' with %,d rows", - dynamicSpec.getMaxRowsPerSegment() - ); - } else { - return partitionsSpec.toString(); - } + return new CompactionStatus(State.PENDING, StringUtils.format(reasonFormat, args)); } public static CompactionStatus skipped(String reasonFormat, Object... args) { - return new CompactionStatus(State.SKIPPED, StringUtils.format(reasonFormat, args), null, null); + return new CompactionStatus(State.SKIPPED, StringUtils.format(reasonFormat, args)); } public static CompactionStatus running(String message) { - return new CompactionStatus(State.RUNNING, message, null, null); - } - - /** - * Determines the CompactionStatus of the given candidate segments by evaluating - * the {@link #CHECKS} one by one. If any check returns an incomplete status, - * further checks are still performed to determine the number of uncompacted - * segments but only the first incomplete status is returned. - */ - static CompactionStatus compute( - CompactionCandidate candidateSegments, - DataSourceCompactionConfig config, - @Nullable IndexingStateFingerprintMapper fingerprintMapper - ) - { - return new Evaluator(candidateSegments, config, fingerprintMapper).evaluate().rhs; - } - - @Nullable - public static PartitionsSpec findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig tuningConfig) - { - final PartitionsSpec partitionsSpecFromTuningConfig = tuningConfig.getPartitionsSpec(); - if (partitionsSpecFromTuningConfig == null) { - final Long maxTotalRows = tuningConfig.getMaxTotalRows(); - final Integer maxRowsPerSegment = tuningConfig.getMaxRowsPerSegment(); - - if (maxTotalRows == null && maxRowsPerSegment == null) { - // If not specified, return null so that partitionsSpec is not compared - return null; - } else { - return new DynamicPartitionsSpec(maxRowsPerSegment, maxTotalRows); - } - } else if (partitionsSpecFromTuningConfig instanceof DynamicPartitionsSpec) { - return new DynamicPartitionsSpec( - partitionsSpecFromTuningConfig.getMaxRowsPerSegment(), - ((DynamicPartitionsSpec) partitionsSpecFromTuningConfig).getMaxTotalRowsOr(Long.MAX_VALUE) - ); - } else if (partitionsSpecFromTuningConfig instanceof DimensionRangePartitionsSpec) { - return getEffectiveRangePartitionsSpec((DimensionRangePartitionsSpec) partitionsSpecFromTuningConfig); - } else { - return partitionsSpecFromTuningConfig; - } - } - - @Nullable - private static List getNonPartitioningDimensions( - @Nullable final List dimensionSchemas, - @Nullable final PartitionsSpec partitionsSpec, - @Nullable final IndexSpec indexSpec - ) - { - final IndexSpec effectiveIndexSpec = (indexSpec == null ? IndexSpec.getDefault() : indexSpec).getEffectiveSpec(); - if (dimensionSchemas == null || !(partitionsSpec instanceof DimensionRangePartitionsSpec)) { - if (dimensionSchemas != null) { - return dimensionSchemas.stream() - .map(dim -> dim.getEffectiveSchema(effectiveIndexSpec)) - .collect(Collectors.toList()); - } - return null; - } - - final List partitionsDimensions = ((DimensionRangePartitionsSpec) partitionsSpec).getPartitionDimensions(); - return dimensionSchemas.stream() - .filter(dim -> !partitionsDimensions.contains(dim.getName())) - .map(dim -> dim.getEffectiveSchema(effectiveIndexSpec)) - .collect(Collectors.toList()); - } - - /** - * Converts to have only the effective maxRowsPerSegment to avoid false positives when targetRowsPerSegment is set but - * effectively translates to the same maxRowsPerSegment. - */ - static DimensionRangePartitionsSpec getEffectiveRangePartitionsSpec(DimensionRangePartitionsSpec partitionsSpec) - { - return new DimensionRangePartitionsSpec( - null, - partitionsSpec.getMaxRowsPerSegment(), - partitionsSpec.getPartitionDimensions(), - partitionsSpec.isAssumeGrouped() - ); - } - - /** - * Evaluates {@link #CHECKS} to determine the compaction status of a - * {@link CompactionCandidate}. - */ - static class Evaluator - { - private final DataSourceCompactionConfig compactionConfig; - private final CompactionCandidate candidateSegments; - private final ClientCompactionTaskQueryTuningConfig tuningConfig; - private final UserCompactionTaskGranularityConfig configuredGranularitySpec; - - private final List fingerprintedSegments = new ArrayList<>(); - private final List compactedSegments = new ArrayList<>(); - private final List uncompactedSegments = new ArrayList<>(); - private final Map> unknownStateToSegments = new HashMap<>(); - - @Nullable - private final IndexingStateFingerprintMapper fingerprintMapper; - @Nullable - private final String targetFingerprint; - - Evaluator( - CompactionCandidate candidateSegments, - DataSourceCompactionConfig compactionConfig, - @Nullable IndexingStateFingerprintMapper fingerprintMapper - ) - { - this.candidateSegments = candidateSegments; - this.compactionConfig = compactionConfig; - this.tuningConfig = ClientCompactionTaskQueryTuningConfig.from(compactionConfig); - this.configuredGranularitySpec = compactionConfig.getGranularitySpec(); - this.fingerprintMapper = fingerprintMapper; - if (fingerprintMapper == null) { - targetFingerprint = null; - } else { - targetFingerprint = fingerprintMapper.generateFingerprint( - compactionConfig.getDataSource(), - compactionConfig.toCompactionState() - ); - } - } - - List getUncompactedSegments() - { - return uncompactedSegments; - } - - /** - * Evaluates the compaction status of candidate segments through a multi-step process: - *

    - *
  1. Validates input bytes are within limits
    2. - *
  2. Categorizes segments by compaction state (fingerprinted, uncompacted, or unknown)
    3. - *
  3. Performs fingerprint-based validation if available (fast path)
    4. - *
  4. Runs detailed checks against unknown states via {@link #CHECKS}
    5. - *
- * - * @return Pair of eligibility status and compaction status with reason for first failed check - */ - Pair evaluate() - { - final CompactionEligibility inputBytesCheck = inputBytesAreWithinLimit(); - if (inputBytesCheck != null) { - return Pair.of(inputBytesCheck, CompactionStatus.skipped(inputBytesCheck.getReason())); - } - - List reasonsForCompaction = new ArrayList<>(); - CompactionStatus compactedOnceCheck = segmentsHaveBeenCompactedAtLeastOnce(); - if (!compactedOnceCheck.isComplete()) { - reasonsForCompaction.add(compactedOnceCheck.getReason()); - } - - if (fingerprintMapper != null && targetFingerprint != null) { - // First try fingerprint-based evaluation (fast path) - CompactionStatus fingerprintStatus = FINGERPRINT_CHECKS.stream() - .map(f -> f.apply(this)) - .filter(status -> !status.isComplete()) - .findFirst().orElse(COMPLETE); - - if (!fingerprintStatus.isComplete()) { - reasonsForCompaction.add(fingerprintStatus.getReason()); - } - } - - if (!unknownStateToSegments.isEmpty()) { - // Run CHECKS against any states with uknown compaction status - reasonsForCompaction.addAll( - CHECKS.stream() - .map(f -> f.apply(this)) - .filter(status -> !status.isComplete()) - .map(CompactionStatus::getReason) - .collect(Collectors.toList()) - ); - - // Any segments left in unknownStateToSegments passed all checks and are considered compacted - compactedSegments.addAll( - unknownStateToSegments - .values() - .stream() - .flatMap(List::stream) - .collect(Collectors.toList()) - ); - } - - if (reasonsForCompaction.isEmpty()) { - return Pair.of( - CompactionEligibility.NOT_APPLICABLE, - CompactionStatus.COMPLETE - ); - } else { - return Pair.of( - CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - CompactionStatus.pending( - createStats(compactedSegments), - createStats(uncompactedSegments), - reasonsForCompaction.get(0) - ) - ); - } - } - - /** - * Evaluates the fingerprints of all fingerprinted candidate segments against the expected fingerprint. - *

- * If all fingerprinted segments have the expected fingerprint, the check can quickly pass as COMPLETE. However, - * if any fingerprinted segment has a mismatched fingerprint, we need to investigate further by adding them to - * {@link #unknownStateToSegments} where their indexing states will be analyzed. - *

- */ - private CompactionStatus allFingerprintedCandidatesHaveExpectedFingerprint() - { - Map> mismatchedFingerprintToSegmentMap = new HashMap<>(); - for (DataSegment segment : fingerprintedSegments) { - String fingerprint = segment.getIndexingStateFingerprint(); - if (fingerprint == null) { - // Should not happen since we are iterating over fingerprintedSegments - } else if (fingerprint.equals(targetFingerprint)) { - compactedSegments.add(segment); - } else { - mismatchedFingerprintToSegmentMap - .computeIfAbsent(fingerprint, k -> new ArrayList<>()) - .add(segment); - } - } - - if (mismatchedFingerprintToSegmentMap.isEmpty()) { - // All fingerprinted segments have the expected fingerprint - compaction is complete - return COMPLETE; - } - - if (fingerprintMapper == null) { - // Cannot evaluate further without a fingerprint mapper - uncompactedSegments.addAll( - mismatchedFingerprintToSegmentMap.values() - .stream() - .flatMap(List::stream) - .collect(Collectors.toList()) - ); - return CompactionStatus.pending("Segments have a mismatched fingerprint and no fingerprint mapper is available"); - } - - boolean fingerprintedSegmentWithoutCachedStateFound = false; - - for (Map.Entry> e : mismatchedFingerprintToSegmentMap.entrySet()) { - String fingerprint = e.getKey(); - CompactionState stateToValidate = fingerprintMapper.getStateForFingerprint(fingerprint).orElse(null); - if (stateToValidate == null) { - log.warn("No indexing state found for fingerprint[%s]", fingerprint); - fingerprintedSegmentWithoutCachedStateFound = true; - uncompactedSegments.addAll(e.getValue()); - } else { - // Note that this does not mean we need compaction yet - we need to validate the state further to determine this - unknownStateToSegments.compute( - stateToValidate, - (state, segments) -> { - if (segments == null) { - segments = new ArrayList<>(); - } - segments.addAll(e.getValue()); - return segments; - } - ); - } - } - - if (fingerprintedSegmentWithoutCachedStateFound) { - return CompactionStatus.pending("One or more fingerprinted segments do not have a cached indexing state"); - } else { - return COMPLETE; - } - } - - /** - * Checks if all the segments have been compacted at least once and groups them into uncompacted, fingerprinted, or - * non-fingerprinted. - */ - private CompactionStatus segmentsHaveBeenCompactedAtLeastOnce() - { - for (DataSegment segment : candidateSegments.getSegments()) { - final String fingerprint = segment.getIndexingStateFingerprint(); - final CompactionState segmentState = segment.getLastCompactionState(); - if (fingerprint != null) { - fingerprintedSegments.add(segment); - } else if (segmentState == null) { - uncompactedSegments.add(segment); - } else { - unknownStateToSegments.computeIfAbsent(segmentState, s -> new ArrayList<>()).add(segment); - } - } - - if (uncompactedSegments.isEmpty()) { - return COMPLETE; - } else { - return CompactionStatus.pending("not compacted yet"); - } - } - - private CompactionStatus partitionsSpecIsUpToDate() - { - return evaluateForAllCompactionStates(this::partitionsSpecIsUpToDate); - } - - private CompactionStatus indexSpecIsUpToDate() - { - return evaluateForAllCompactionStates(this::indexSpecIsUpToDate); - } - - private CompactionStatus projectionsAreUpToDate() - { - return evaluateForAllCompactionStates(this::projectionsAreUpToDate); - } - - private CompactionStatus segmentGranularityIsUpToDate() - { - return evaluateForAllCompactionStates(this::segmentGranularityIsUpToDate); - } - - private CompactionStatus rollupIsUpToDate() - { - return evaluateForAllCompactionStates(this::rollupIsUpToDate); - } - - private CompactionStatus queryGranularityIsUpToDate() - { - return evaluateForAllCompactionStates(this::queryGranularityIsUpToDate); - } - - private CompactionStatus dimensionsSpecIsUpToDate() - { - return evaluateForAllCompactionStates(this::dimensionsSpecIsUpToDate); - } - - private CompactionStatus metricsSpecIsUpToDate() - { - return evaluateForAllCompactionStates(this::metricsSpecIsUpToDate); - } - - private CompactionStatus transformSpecFilterIsUpToDate() - { - return evaluateForAllCompactionStates(this::transformSpecFilterIsUpToDate); - } - - private CompactionStatus partitionsSpecIsUpToDate(CompactionState lastCompactionState) - { - PartitionsSpec existingPartionsSpec = lastCompactionState.getPartitionsSpec(); - if (existingPartionsSpec instanceof DimensionRangePartitionsSpec) { - existingPartionsSpec = getEffectiveRangePartitionsSpec((DimensionRangePartitionsSpec) existingPartionsSpec); - } else if (existingPartionsSpec instanceof DynamicPartitionsSpec) { - existingPartionsSpec = new DynamicPartitionsSpec( - existingPartionsSpec.getMaxRowsPerSegment(), - ((DynamicPartitionsSpec) existingPartionsSpec).getMaxTotalRowsOr(Long.MAX_VALUE) - ); - } - return CompactionStatus.completeIfNullOrEqual( - "partitionsSpec", - findPartitionsSpecFromConfig(tuningConfig), - existingPartionsSpec, - CompactionStatus::asString - ); - } - - private CompactionStatus indexSpecIsUpToDate(CompactionState lastCompactionState) - { - return CompactionStatus.completeIfNullOrEqual( - "indexSpec", - Configs.valueOrDefault(tuningConfig.getIndexSpec(), IndexSpec.getDefault()).getEffectiveSpec(), - lastCompactionState.getIndexSpec().getEffectiveSpec(), - String::valueOf - ); - } - - private CompactionStatus projectionsAreUpToDate(CompactionState lastCompactionState) - { - return CompactionStatus.completeIfNullOrEqual( - "projections", - compactionConfig.getProjections(), - lastCompactionState.getProjections(), - String::valueOf - ); - } - - @Nullable - private CompactionEligibility inputBytesAreWithinLimit() - { - final long inputSegmentSize = compactionConfig.getInputSegmentSizeBytes(); - if (candidateSegments.getTotalBytes() > inputSegmentSize) { - return CompactionEligibility.fail( - "'inputSegmentSize' exceeded: Total segment size[%d] is larger than allowed inputSegmentSize[%d]", - candidateSegments.getTotalBytes(), inputSegmentSize - ); - } - return null; - } - - private CompactionStatus segmentGranularityIsUpToDate(CompactionState lastCompactionState) - { - if (configuredGranularitySpec == null - || configuredGranularitySpec.getSegmentGranularity() == null) { - return COMPLETE; - } - - final Granularity configuredSegmentGranularity = configuredGranularitySpec.getSegmentGranularity(); - final UserCompactionTaskGranularityConfig existingGranularitySpec = getGranularitySpec(lastCompactionState); - final Granularity existingSegmentGranularity - = existingGranularitySpec == null ? null : existingGranularitySpec.getSegmentGranularity(); - - if (configuredSegmentGranularity.equals(existingSegmentGranularity)) { - return COMPLETE; - } else if (existingSegmentGranularity == null) { - // Candidate segments were compacted without segment granularity specified - // Check if the segments already have the desired segment granularity - final List segmentsForState = unknownStateToSegments.get(lastCompactionState); - boolean needsCompaction = segmentsForState.stream().anyMatch( - segment -> !configuredSegmentGranularity.isAligned(segment.getInterval()) - ); - if (needsCompaction) { - return CompactionStatus.pending( - "segmentGranularity: segments do not align with target[%s]", - asString(configuredSegmentGranularity) - ); - } - } else { - return CompactionStatus.configChanged( - "segmentGranularity", - configuredSegmentGranularity, - existingSegmentGranularity, - CompactionStatus::asString - ); - } - - return COMPLETE; - } - - private CompactionStatus rollupIsUpToDate(CompactionState lastCompactionState) - { - if (configuredGranularitySpec == null) { - return COMPLETE; - } else { - final UserCompactionTaskGranularityConfig existingGranularitySpec - = getGranularitySpec(lastCompactionState); - return CompactionStatus.completeIfNullOrEqual( - "rollup", - configuredGranularitySpec.isRollup(), - existingGranularitySpec == null ? null : existingGranularitySpec.isRollup(), - String::valueOf - ); - } - } - - private CompactionStatus queryGranularityIsUpToDate(CompactionState lastCompactionState) - { - if (configuredGranularitySpec == null) { - return COMPLETE; - } else { - final UserCompactionTaskGranularityConfig existingGranularitySpec - = getGranularitySpec(lastCompactionState); - return CompactionStatus.completeIfNullOrEqual( - "queryGranularity", - configuredGranularitySpec.getQueryGranularity(), - existingGranularitySpec == null ? null : existingGranularitySpec.getQueryGranularity(), - CompactionStatus::asString - ); - } - } - - /** - * Removes partition dimensions before comparison, since they are placed in front of the sort order -- - * which can create a mismatch between expected and actual order of dimensions. Partition dimensions are separately - * covered in {@link Evaluator#partitionsSpecIsUpToDate()} check. - */ - private CompactionStatus dimensionsSpecIsUpToDate(CompactionState lastCompactionState) - { - if (compactionConfig.getDimensionsSpec() == null) { - return COMPLETE; - } else { - List existingDimensions = getNonPartitioningDimensions( - lastCompactionState.getDimensionsSpec() == null - ? null - : lastCompactionState.getDimensionsSpec().getDimensions(), - lastCompactionState.getPartitionsSpec(), - lastCompactionState.getIndexSpec() - ); - List configuredDimensions = getNonPartitioningDimensions( - compactionConfig.getDimensionsSpec().getDimensions(), - compactionConfig.getTuningConfig() == null ? null : compactionConfig.getTuningConfig().getPartitionsSpec(), - compactionConfig.getTuningConfig() == null - ? IndexSpec.getDefault() - : compactionConfig.getTuningConfig().getIndexSpec() - ); - return CompactionStatus.completeIfNullOrEqual( - "dimensionsSpec", - configuredDimensions, - existingDimensions, - String::valueOf - ); - } - } - - private CompactionStatus metricsSpecIsUpToDate(CompactionState lastCompactionState) - { - final AggregatorFactory[] configuredMetricsSpec = compactionConfig.getMetricsSpec(); - if (ArrayUtils.isEmpty(configuredMetricsSpec)) { - return COMPLETE; - } - - final List metricSpecList = lastCompactionState.getMetricsSpec(); - final AggregatorFactory[] existingMetricsSpec - = CollectionUtils.isNullOrEmpty(metricSpecList) - ? null : metricSpecList.toArray(new AggregatorFactory[0]); - - if (existingMetricsSpec == null || !Arrays.deepEquals(configuredMetricsSpec, existingMetricsSpec)) { - return CompactionStatus.configChanged( - "metricsSpec", - configuredMetricsSpec, - existingMetricsSpec, - Arrays::toString - ); - } else { - return COMPLETE; - } - } - - private CompactionStatus transformSpecFilterIsUpToDate(CompactionState lastCompactionState) - { - if (compactionConfig.getTransformSpec() == null) { - return COMPLETE; - } - - CompactionTransformSpec existingTransformSpec = lastCompactionState.getTransformSpec(); - return CompactionStatus.completeIfNullOrEqual( - "transformSpec filter", - compactionConfig.getTransformSpec().getFilter(), - existingTransformSpec == null ? null : existingTransformSpec.getFilter(), - String::valueOf - ); - } - - /** - * Evaluates the given check for each entry in the {@link #unknownStateToSegments}. - * If any entry fails the given check by returning a status which is not - * COMPLETE, all the segments with that state are moved to {@link #uncompactedSegments}. - * - * @return The first status which is not COMPLETE. - */ - private CompactionStatus evaluateForAllCompactionStates( - Function check - ) - { - CompactionStatus firstIncompleteStatus = null; - for (CompactionState state : List.copyOf(unknownStateToSegments.keySet())) { - final CompactionStatus status = check.apply(state); - if (!status.isComplete()) { - uncompactedSegments.addAll(unknownStateToSegments.remove(state)); - if (firstIncompleteStatus == null) { - firstIncompleteStatus = status; - } - } - } - - return firstIncompleteStatus == null ? COMPLETE : firstIncompleteStatus; - } - - private static UserCompactionTaskGranularityConfig getGranularitySpec( - CompactionState compactionState - ) - { - return UserCompactionTaskGranularityConfig.from(compactionState.getGranularitySpec()); - } - - private static CompactionStatistics createStats(List segments) - { - final Set segmentIntervals = - segments.stream().map(DataSegment::getInterval).collect(Collectors.toSet()); - final long totalBytes = segments.stream().mapToLong(DataSegment::getSize).sum(); - return CompactionStatistics.create(totalBytes, segments.size(), segmentIntervals.size()); - } + return new CompactionStatus(State.RUNNING, message); } } diff --git a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java index 5e77d1bf9907..e753c7a8dce3 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java +++ b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; +import org.apache.druid.error.DruidException; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.JodaUtils; @@ -56,6 +57,7 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Objects; import java.util.PriorityQueue; import java.util.Set; import java.util.stream.Collectors; @@ -123,9 +125,12 @@ private void populateQueue(SegmentTimeline timeline, List skipInterval if (!partialEternitySegments.isEmpty()) { // Do not use the target segment granularity in the CompactionCandidate // as Granularities.getIterable() will cause OOM due to the above issue - CompactionCandidate candidatesWithStatus = CompactionCandidate - .from(partialEternitySegments, null) - .withCurrentStatus(CompactionStatus.skipped("Segments have partial-eternity intervals")); + CompactionCandidate candidatesWithStatus = + CompactionEligibility.fail("Segments have partial-eternity intervals") + .createCandidate(CompactionCandidate.ProposedCompaction.from( + partialEternitySegments, + null + )); skippedSegments.add(candidatesWithStatus); return; } @@ -331,17 +336,29 @@ private void findAndEnqueueSegmentsToCompact(CompactibleSegmentIterator compacti continue; } + CompactionCandidate.ProposedCompaction candidate = + CompactionCandidate.ProposedCompaction.from(segments, config.getSegmentGranularity()); final CompactionCandidate candidatesWithStatus = - CompactionCandidate.from(segments, config.getSegmentGranularity()) - .evaluate(config, searchPolicy, fingerprintMapper); - - if (candidatesWithStatus.getCurrentStatus().isComplete()) { - compactedSegments.add(candidatesWithStatus); - } else if (candidatesWithStatus.getCurrentStatus().isSkipped()) { - skippedSegments.add(candidatesWithStatus); - } else if (!queuedIntervals.contains(candidatesWithStatus.getUmbrellaInterval())) { - queue.add(candidatesWithStatus); - queuedIntervals.add(candidatesWithStatus.getUmbrellaInterval()); + CompactionEligibility.evaluate(candidate, config, searchPolicy, fingerprintMapper).createCandidate(candidate); + switch (Objects.requireNonNull(candidatesWithStatus.getPolicyEligibility()).getState()) { + case NOT_APPLICABLE: + compactedSegments.add(candidatesWithStatus); + break; + case NOT_ELIGIBLE: + skippedSegments.add(candidatesWithStatus); + break; + case FULL_COMPACTION: + case INCREMENTAL_COMPACTION: + if (!queuedIntervals.contains(candidatesWithStatus.getProposedCompaction().getUmbrellaInterval())) { + queue.add(candidatesWithStatus); + queuedIntervals.add(candidatesWithStatus.getProposedCompaction().getUmbrellaInterval()); + } + break; + default: + throw DruidException.defensive( + "Unexpected eligibility state[%s]", + candidatesWithStatus.getPolicyEligibility().getState() + ); } } } @@ -374,16 +391,17 @@ private List findInitialSearchInterval( timeline.findNonOvershadowedObjectsInInterval(skipInterval, Partitions.ONLY_COMPLETE) ); if (!CollectionUtils.isNullOrEmpty(segments)) { - final CompactionCandidate candidates = CompactionCandidate.from(segments, config.getSegmentGranularity()); + final CompactionCandidate.ProposedCompaction candidates = + CompactionCandidate.ProposedCompaction.from(segments, config.getSegmentGranularity()); - final CompactionStatus reason; + final CompactionEligibility eligibility; if (candidates.getCompactionInterval().overlaps(latestSkipInterval)) { - reason = CompactionStatus.skipped("skip offset from latest[%s]", skipOffset); + eligibility = CompactionEligibility.fail("skip offset from latest[%s]", skipOffset); } else { - reason = CompactionStatus.skipped("interval locked by another task"); + eligibility = CompactionEligibility.fail("interval locked by another task"); } - final CompactionCandidate candidatesWithStatus = candidates.withCurrentStatus(reason); + final CompactionCandidate candidatesWithStatus = eligibility.createCandidate(candidates); skippedSegments.add(candidatesWithStatus); } } diff --git a/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java index 3d432d3fd703..04040ee0ef5b 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java @@ -52,18 +52,21 @@ public List getEligibleCandidates() @Override public int compareCandidates(CompactionCandidate candidateA, CompactionCandidate candidateB) { - return findIndex(candidateA) - findIndex(candidateB); + return findIndex(candidateA.getProposedCompaction()) - findIndex(candidateB.getProposedCompaction()); } @Override - public CompactionEligibility checkEligibilityForCompaction(CompactionCandidate candidate) + public CompactionEligibility checkEligibilityForCompaction( + CompactionCandidate.ProposedCompaction candidate, + CompactionEligibility eligibility + ) { return findIndex(candidate) < Integer.MAX_VALUE - ? CompactionEligibility.FULL_COMPACTION_ELIGIBLE - : CompactionEligibility.fail("Datasource/Interval is not in the list of 'eligibleCandidates'"); + ? eligibility + : CompactionEligibility.fail("Datasource/Interval is not in the list of 'eligibleCandidates'"); } - private int findIndex(CompactionCandidate candidate) + private int findIndex(CompactionCandidate.ProposedCompaction candidate) { int index = 0; for (Candidate eligibleCandidate : eligibleCandidates) { diff --git a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java index 8c3f74204feb..62257f6bf686 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java @@ -25,6 +25,7 @@ import org.apache.druid.error.InvalidInput; import org.apache.druid.guice.annotations.UnstableApi; import org.apache.druid.java.util.common.HumanReadableBytes; +import org.apache.druid.java.util.common.StringUtils; import javax.annotation.Nullable; import java.util.Comparator; @@ -131,7 +132,7 @@ public Double getIncrementalCompactionUncompactedRatioThreshold() @Override protected Comparator getSegmentComparator() { - return this::compare; + return Comparator.comparing(o -> Objects.requireNonNull(o.getPolicyEligibility()), this::compare); } @Override @@ -179,7 +180,7 @@ public String toString() '}'; } - private int compare(CompactionCandidate candidateA, CompactionCandidate candidateB) + private int compare(CompactionEligibility candidateA, CompactionEligibility candidateB) { final double fragmentationDiff = computeFragmentationIndex(candidateB) - computeFragmentationIndex(candidateA); @@ -187,12 +188,13 @@ private int compare(CompactionCandidate candidateA, CompactionCandidate candidat } @Override - public CompactionEligibility checkEligibilityForCompaction(CompactionCandidate candidate) + public CompactionEligibility checkEligibilityForCompaction( + CompactionCandidate.ProposedCompaction candidate, + CompactionEligibility eligibility + ) { - final CompactionStatistics uncompacted = candidate.getUncompactedStats(); - if (uncompacted == null) { - return CompactionEligibility.FULL_COMPACTION_ELIGIBLE; - } else if (uncompacted.getNumSegments() < 1) { + final CompactionStatistics uncompacted = Objects.requireNonNull(eligibility.getUncompactedStats()); + if (uncompacted.getNumSegments() < 1) { return CompactionEligibility.fail("No uncompacted segments in interval"); } else if (uncompacted.getNumSegments() < minUncompactedCount) { return CompactionEligibility.fail( @@ -215,15 +217,21 @@ public CompactionEligibility checkEligibilityForCompaction(CompactionCandidate c } final double uncompactedBytesRatio = (double) uncompacted.getTotalBytes() / - (uncompacted.getTotalBytes() + candidate.getCompactedStats().getTotalBytes()); + (uncompacted.getTotalBytes() + eligibility.getCompactedStats() + .getTotalBytes()); if (uncompactedBytesRatio < incrementalCompactionUncompactedBytesRatioThreshold) { - return CompactionEligibility.incrementalCompaction( + String reason = StringUtils.format( "Uncompacted bytes ratio[%.2f] is below threshold[%.2f]", uncompactedBytesRatio, incrementalCompactionUncompactedBytesRatioThreshold ); + return CompactionEligibility.builder(CompactionEligibility.State.INCREMENTAL_COMPACTION, reason) + .compacted(eligibility.getCompactedStats()) + .uncompacted(eligibility.getUncompactedStats()) + .uncompactedSegments(eligibility.getUncompactedSegments()) + .build(); } else { - return CompactionEligibility.FULL_COMPACTION_ELIGIBLE; + return eligibility; } } @@ -234,9 +242,9 @@ public CompactionEligibility checkEligibilityForCompaction(CompactionCandidate c * A higher fragmentation index causes the candidate to be higher in priority * for compaction. */ - private double computeFragmentationIndex(CompactionCandidate candidate) + private double computeFragmentationIndex(CompactionEligibility eligibility) { - final CompactionStatistics uncompacted = candidate.getUncompactedStats(); + final CompactionStatistics uncompacted = eligibility.getUncompactedStats(); if (uncompacted == null || uncompacted.getNumSegments() < 1 || uncompacted.getTotalBytes() < 1) { return 0; } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java index 39a24d3eb5e8..0c5a13a6682c 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java @@ -32,7 +32,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.transform.CompactionTransformSpec; -import org.apache.druid.server.compaction.CompactionStatus; +import org.apache.druid.server.compaction.CompactionEligibility; import org.apache.druid.timeline.CompactionState; import org.joda.time.Period; @@ -112,7 +112,7 @@ default CompactionState toCompactionState() { ClientCompactionTaskQueryTuningConfig tuningConfig = ClientCompactionTaskQueryTuningConfig.from(this); - PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(tuningConfig); + PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(tuningConfig); IndexSpec indexSpec = tuningConfig.getIndexSpec() == null ? IndexSpec.getDefault().getEffectiveSpec() diff --git a/server/src/test/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpecTest.java b/server/src/test/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpecTest.java index 096a82c8c158..0c4d8d5d7b45 100644 --- a/server/src/test/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpecTest.java +++ b/server/src/test/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpecTest.java @@ -20,7 +20,6 @@ package org.apache.druid.client.indexing; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableList; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.IAE; @@ -28,7 +27,7 @@ import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.segment.IndexIO; -import org.apache.druid.server.compaction.CompactionCandidate; +import org.apache.druid.server.compaction.CompactionCandidate.ProposedCompaction; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.NoneShardSpec; import org.joda.time.Interval; @@ -79,7 +78,7 @@ public class ClientCompactionIntervalSpecTest public void testFromSegmentWithNoSegmentGranularity() { // The umbrella interval of segments is 2015-02-12/2015-04-14 - CompactionCandidate actual = CompactionCandidate.from(ImmutableList.of(dataSegment1, dataSegment2, dataSegment3), null); + ProposedCompaction actual = ProposedCompaction.from(List.of(dataSegment1, dataSegment2, dataSegment3), null); Assert.assertEquals(Intervals.of("2015-02-12/2015-04-14"), actual.getCompactionInterval()); } @@ -87,7 +86,7 @@ public void testFromSegmentWithNoSegmentGranularity() public void testFromSegmentWitSegmentGranularitySameAsSegment() { // The umbrella interval of segments is 2015-04-11/2015-04-12 - CompactionCandidate actual = CompactionCandidate.from(ImmutableList.of(dataSegment1), Granularities.DAY); + ProposedCompaction actual = ProposedCompaction.from(List.of(dataSegment1), Granularities.DAY); Assert.assertEquals(Intervals.of("2015-04-11/2015-04-12"), actual.getCompactionInterval()); } @@ -95,7 +94,10 @@ public void testFromSegmentWitSegmentGranularitySameAsSegment() public void testFromSegmentWithCoarserSegmentGranularity() { // The umbrella interval of segments is 2015-02-12/2015-04-14 - CompactionCandidate actual = CompactionCandidate.from(ImmutableList.of(dataSegment1, dataSegment2, dataSegment3), Granularities.YEAR); + ProposedCompaction actual = ProposedCompaction.from( + List.of(dataSegment1, dataSegment2, dataSegment3), + Granularities.YEAR + ); // The compaction interval should be expanded to start of the year and end of the year to cover the segmentGranularity Assert.assertEquals(Intervals.of("2015-01-01/2016-01-01"), actual.getCompactionInterval()); } @@ -104,7 +106,10 @@ public void testFromSegmentWithCoarserSegmentGranularity() public void testFromSegmentWithFinerSegmentGranularityAndUmbrellaIntervalAlign() { // The umbrella interval of segments is 2015-02-12/2015-04-14 - CompactionCandidate actual = CompactionCandidate.from(ImmutableList.of(dataSegment1, dataSegment2, dataSegment3), Granularities.DAY); + ProposedCompaction actual = ProposedCompaction.from( + List.of(dataSegment1, dataSegment2, dataSegment3), + Granularities.DAY + ); // The segmentGranularity of DAY align with the umbrella interval (umbrella interval can be evenly divide into the segmentGranularity) Assert.assertEquals(Intervals.of("2015-02-12/2015-04-14"), actual.getCompactionInterval()); } @@ -113,7 +118,10 @@ public void testFromSegmentWithFinerSegmentGranularityAndUmbrellaIntervalAlign() public void testFromSegmentWithFinerSegmentGranularityAndUmbrellaIntervalNotAlign() { // The umbrella interval of segments is 2015-02-12/2015-04-14 - CompactionCandidate actual = CompactionCandidate.from(ImmutableList.of(dataSegment1, dataSegment2, dataSegment3), Granularities.WEEK); + ProposedCompaction actual = ProposedCompaction.from( + List.of(dataSegment1, dataSegment2, dataSegment3), + Granularities.WEEK + ); // The segmentGranularity of WEEK does not align with the umbrella interval (umbrella interval cannot be evenly divide into the segmentGranularity) // Hence the compaction interval is modified to aling with the segmentGranularity Assert.assertEquals(Intervals.of("2015-02-09/2015-04-20"), actual.getCompactionInterval()); diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionCandidateTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionCandidateTest.java new file mode 100644 index 000000000000..96a275681b0e --- /dev/null +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionCandidateTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.compaction; + +import org.apache.druid.error.DruidException; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.server.coordinator.CreateDataSegments; +import org.apache.druid.timeline.DataSegment; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; + +public class CompactionCandidateTest +{ + private static final String DATASOURCE = "test_datasource"; + + @Test + public void testConstructorAndGetters() + { + List segments = createTestSegments(3); + CompactionCandidate.ProposedCompaction proposed = CompactionCandidate.ProposedCompaction.from(segments, null); + CompactionEligibility eligibility = CompactionEligibility.fail("test reason"); + CompactionStatus status = CompactionStatus.COMPLETE; + + CompactionCandidate candidate = new CompactionCandidate(proposed, eligibility, status); + + Assert.assertEquals(proposed, candidate.getProposedCompaction()); + Assert.assertEquals(eligibility, candidate.getPolicyEligibility()); + Assert.assertEquals(status, candidate.getCurrentStatus()); + Assert.assertEquals(segments, candidate.getSegments()); + Assert.assertEquals(DATASOURCE, candidate.getDataSource()); + Assert.assertEquals(3, candidate.numSegments()); + } + + @Test + public void testProposedCompactionFrom() + { + List segments = createTestSegments(3); + + CompactionCandidate.ProposedCompaction proposed = + CompactionCandidate.ProposedCompaction.from(segments, null); + + Assert.assertEquals(segments, proposed.getSegments()); + Assert.assertEquals(DATASOURCE, proposed.getDataSource()); + Assert.assertEquals(3, proposed.numSegments()); + Assert.assertNotNull(proposed.getUmbrellaInterval()); + Assert.assertNotNull(proposed.getCompactionInterval()); + Assert.assertNotNull(proposed.getStats()); + } + + @Test + public void testProposedCompactionWithTargetGranularity() + { + List segments = createTestSegments(5); + + CompactionCandidate.ProposedCompaction proposed = + CompactionCandidate.ProposedCompaction.from(segments, Granularities.MONTH); + + Assert.assertEquals(segments, proposed.getSegments()); + Assert.assertEquals(5, proposed.numSegments()); + Assert.assertNotNull(proposed.getUmbrellaInterval()); + Assert.assertNotNull(proposed.getCompactionInterval()); + } + + @Test + public void testProposedCompactionThrowsOnNullOrEmptySegments() + { + Assert.assertThrows( + DruidException.class, + () -> CompactionCandidate.ProposedCompaction.from(null, null) + ); + + Assert.assertThrows( + DruidException.class, + () -> CompactionCandidate.ProposedCompaction.from(Collections.emptyList(), null) + ); + } + + @Test + public void testWithCurrentStatus() + { + List segments = createTestSegments(2); + CompactionCandidate.ProposedCompaction proposed = CompactionCandidate.ProposedCompaction.from(segments, null); + CompactionEligibility eligibility = CompactionEligibility.fail("test"); + CompactionStatus originalStatus = CompactionStatus.COMPLETE; + + CompactionCandidate candidate = new CompactionCandidate(proposed, eligibility, originalStatus); + CompactionStatus newStatus = CompactionStatus.pending("needs compaction"); + CompactionCandidate updated = candidate.withCurrentStatus(newStatus); + + Assert.assertNotSame(candidate, updated); + Assert.assertEquals(originalStatus, candidate.getCurrentStatus()); + Assert.assertEquals(newStatus, updated.getCurrentStatus()); + Assert.assertEquals(proposed, updated.getProposedCompaction()); + Assert.assertEquals(eligibility, updated.getPolicyEligibility()); + } + + @Test + public void testDelegationMethods() + { + List segments = createTestSegments(3); + CompactionCandidate.ProposedCompaction proposed = CompactionCandidate.ProposedCompaction.from(segments, null); + CompactionCandidate candidate = new CompactionCandidate( + proposed, + CompactionEligibility.fail("test"), + CompactionStatus.COMPLETE + ); + + Assert.assertEquals(proposed.getTotalBytes(), candidate.getTotalBytes()); + Assert.assertEquals(proposed.getUmbrellaInterval(), candidate.getUmbrellaInterval()); + Assert.assertEquals(proposed.getCompactionInterval(), candidate.getCompactionInterval()); + Assert.assertEquals(proposed.getStats().getTotalBytes(), candidate.getStats().getTotalBytes()); + } + + private static List createTestSegments(int count) + { + return CreateDataSegments.ofDatasource(DATASOURCE) + .forIntervals(count, Granularities.DAY) + .startingAt("2024-01-01") + .eachOfSizeInMb(100); + } +} diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityEvaluateTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityEvaluateTest.java new file mode 100644 index 000000000000..5dc42f14afc5 --- /dev/null +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityEvaluateTest.java @@ -0,0 +1,895 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.compaction; + +import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig; +import org.apache.druid.data.input.impl.AggregateProjectionSpec; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.LongDimensionSchema; +import org.apache.druid.data.input.impl.StringDimensionSchema; +import org.apache.druid.indexer.granularity.GranularitySpec; +import org.apache.druid.indexer.granularity.UniformGranularitySpec; +import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec; +import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; +import org.apache.druid.indexer.partitions.HashedPartitionsSpec; +import org.apache.druid.indexer.partitions.PartitionsSpec; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.granularity.Granularity; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.segment.AutoTypeColumnSchema; +import org.apache.druid.segment.IndexSpec; +import org.apache.druid.segment.TestDataSource; +import org.apache.druid.segment.data.CompressionStrategy; +import org.apache.druid.segment.metadata.DefaultIndexingStateFingerprintMapper; +import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; +import org.apache.druid.segment.metadata.IndexingStateCache; +import org.apache.druid.segment.metadata.IndexingStateFingerprintMapper; +import org.apache.druid.segment.nested.NestedCommonFormatColumnFormatSpec; +import org.apache.druid.server.coordinator.DataSourceCompactionConfig; +import org.apache.druid.server.coordinator.InlineSchemaDataSourceCompactionConfig; +import org.apache.druid.server.coordinator.UserCompactionTaskDimensionsConfig; +import org.apache.druid.server.coordinator.UserCompactionTaskGranularityConfig; +import org.apache.druid.server.coordinator.UserCompactionTaskQueryTuningConfig; +import org.apache.druid.timeline.CompactionState; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; + +public class CompactionEligibilityEvaluateTest +{ + private static final DataSegment WIKI_SEGMENT + = DataSegment.builder(SegmentId.of(TestDataSource.WIKI, Intervals.of("2013-01-01/PT1H"), "v1", 0)) + .size(100_000_000L) + .build(); + private static final DataSegment WIKI_SEGMENT_2 + = DataSegment.builder(SegmentId.of(TestDataSource.WIKI, Intervals.of("2013-01-01/PT1H"), "v1", 1)) + .size(100_000_000L) + .build(); + + private HeapMemoryIndexingStateStorage indexingStateStorage; + private IndexingStateCache indexingStateCache; + private IndexingStateFingerprintMapper fingerprintMapper; + + @Before + public void setUp() + { + indexingStateStorage = new HeapMemoryIndexingStateStorage(); + indexingStateCache = new IndexingStateCache(); + fingerprintMapper = new DefaultIndexingStateFingerprintMapper( + indexingStateCache, + new DefaultObjectMapper() + ); + } + + /** + * Helper to sync the cache with states stored in the manager (for tests that persist states). + */ + private void syncCacheFromManager() + { + indexingStateCache.resetIndexingStatesForPublishedSegments(indexingStateStorage.getAllStoredStates()); + } + + @Test + public void testFindPartitionsSpecWhenGivenIsNull() + { + final ClientCompactionTaskQueryTuningConfig tuningConfig + = ClientCompactionTaskQueryTuningConfig.from(null); + Assert.assertNull(CompactionEligibility.findPartitionsSpecFromConfig(tuningConfig)); + } + + @Test + public void testFindPartitionsSpecWhenGivenIsDynamicWithNullMaxTotalRows() + { + final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(null, null); + final ClientCompactionTaskQueryTuningConfig tuningConfig + = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); + Assert.assertEquals( + new DynamicPartitionsSpec(null, Long.MAX_VALUE), + CompactionEligibility.findPartitionsSpecFromConfig(tuningConfig) + ); + } + + @Test + public void testFindPartitionsSpecWhenGivenIsDynamicWithMaxTotalRows() + { + final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(null, 1000L); + final ClientCompactionTaskQueryTuningConfig tuningConfig + = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); + Assert.assertEquals( + partitionsSpec, + CompactionEligibility.findPartitionsSpecFromConfig(tuningConfig) + ); + } + + @Test + public void testFindPartitionsSpecWhenGivenIsDynamicWithMaxRowsPerSegment() + { + final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(100, 1000L); + final ClientCompactionTaskQueryTuningConfig tuningConfig + = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); + Assert.assertEquals( + partitionsSpec, + CompactionEligibility.findPartitionsSpecFromConfig(tuningConfig) + ); + } + + @Test + public void testFindPartitionsSpecFromConfigWithDeprecatedMaxRowsPerSegmentAndMaxTotalRowsReturnGivenValues() + { + final DataSourceCompactionConfig config = + InlineSchemaDataSourceCompactionConfig.builder() + .forDataSource("datasource") + .withMaxRowsPerSegment(100) + .withTuningConfig( + new UserCompactionTaskQueryTuningConfig( + null, + null, + null, + 1000L, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ) + .build(); + Assert.assertEquals( + new DynamicPartitionsSpec(100, 1000L), + CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(config)) + ); + } + + @Test + public void testFindPartitionsSpecWhenGivenIsHashed() + { + final PartitionsSpec partitionsSpec = + new HashedPartitionsSpec(null, 100, Collections.singletonList("dim")); + final ClientCompactionTaskQueryTuningConfig tuningConfig + = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); + Assert.assertEquals( + partitionsSpec, + CompactionEligibility.findPartitionsSpecFromConfig(tuningConfig) + ); + } + + @Test + public void testFindPartitionsSpecWhenGivenIsRangeWithMaxRows() + { + final PartitionsSpec partitionsSpec = + new DimensionRangePartitionsSpec(null, 10000, Collections.singletonList("dim"), false); + final ClientCompactionTaskQueryTuningConfig tuningConfig + = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); + Assert.assertEquals( + partitionsSpec, + CompactionEligibility.findPartitionsSpecFromConfig(tuningConfig) + ); + } + + @Test + public void testFindPartitionsSpecWhenGivenIsRangeWithTargetRows() + { + final PartitionsSpec partitionsSpec = + new DimensionRangePartitionsSpec(10000, null, Collections.singletonList("dim"), false); + final ClientCompactionTaskQueryTuningConfig tuningConfig + = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); + Assert.assertEquals( + new DimensionRangePartitionsSpec(null, 15000, Collections.singletonList("dim"), false), + CompactionEligibility.findPartitionsSpecFromConfig(tuningConfig) + ); + } + + @Test + public void testStatusWhenLastCompactionStateIsNull() + { + verifyCompactionIsEligibleBecause( + null, + InlineSchemaDataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(), + "not compacted yet" + ); + } + + @Test + public void testStatusWhenLastCompactionStateIsEmpty() + { + final PartitionsSpec requiredPartitionsSpec = new DynamicPartitionsSpec(5_000_000, null); + verifyCompactionIsEligibleBecause( + new CompactionState(null, null, null, null, null, null, null), + InlineSchemaDataSourceCompactionConfig + .builder() + .withTuningConfig(createTuningConfig(requiredPartitionsSpec, null)) + .forDataSource(TestDataSource.WIKI) + .build(), + "'partitionsSpec' mismatch: required['dynamic' with 5,000,000 rows], current[null]" + ); + } + + @Test + public void testStatusOnPartitionsSpecMismatch() + { + final PartitionsSpec requiredPartitionsSpec = new DynamicPartitionsSpec(5_000_000, null); + final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); + + final CompactionState lastCompactionState + = new CompactionState(currentPartitionsSpec, null, null, null, null, null, null); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .withTuningConfig(createTuningConfig(requiredPartitionsSpec, null)) + .forDataSource(TestDataSource.WIKI) + .build(); + + verifyCompactionIsEligibleBecause( + lastCompactionState, + compactionConfig, + "'partitionsSpec' mismatch: required['dynamic' with 5,000,000 rows]," + + " current['dynamic' with 100 rows]" + ); + } + + @Test + public void testStatusOnIndexSpecMismatch() + { + final IndexSpec currentIndexSpec + = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); + + final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); + final CompactionState lastCompactionState = new CompactionState( + currentPartitionsSpec, + null, + null, + null, + currentIndexSpec, + null, + null + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withTuningConfig(createTuningConfig(currentPartitionsSpec, null)) + .build(); + + verifyCompactionIsEligibleBecause( + lastCompactionState, + compactionConfig, + "'indexSpec' mismatch: " + + "required[IndexSpec{bitmapSerdeFactory=RoaringBitmapSerdeFactory{}," + + " metadataCompression=none," + + " dimensionCompression=lz4, stringDictionaryEncoding=Utf8{}," + + " metricCompression=lz4, longEncoding=longs, complexMetricCompression=null," + + " autoColumnFormatSpec=null, jsonCompression=null, segmentLoader=null}], " + + "current[IndexSpec{bitmapSerdeFactory=RoaringBitmapSerdeFactory{}," + + " metadataCompression=none," + + " dimensionCompression=zstd, stringDictionaryEncoding=Utf8{}," + + " metricCompression=lz4, longEncoding=longs, complexMetricCompression=null," + + " autoColumnFormatSpec=null, jsonCompression=null, segmentLoader=null}]" + ); + } + + @Test + public void testStatusOnSegmentGranularityMismatch() + { + final GranularitySpec currentGranularitySpec + = new UniformGranularitySpec(Granularities.HOUR, null, null); + + final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); + final IndexSpec currentIndexSpec + = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); + final CompactionState lastCompactionState = new CompactionState( + currentPartitionsSpec, + null, + null, + null, + currentIndexSpec, + currentGranularitySpec, + null + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec)) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + verifyCompactionIsEligibleBecause( + lastCompactionState, + compactionConfig, + "'segmentGranularity' mismatch: required[DAY], current[HOUR]" + ); + } + + @Test + public void testStatusWhenLastCompactionStateSameAsRequired() + { + final GranularitySpec currentGranularitySpec + = new UniformGranularitySpec(Granularities.HOUR, null, null); + final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); + final IndexSpec currentIndexSpec + = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); + final CompactionState lastCompactionState = new CompactionState( + currentPartitionsSpec, + null, + null, + null, + currentIndexSpec, + currentGranularitySpec, + null + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec)) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .build(); + + final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); + final CompactionEligibility status = CompactionEligibility.evaluate( + CompactionCandidate.ProposedCompaction.from(List.of(segment), Granularities.HOUR), + compactionConfig, + new NewestSegmentFirstPolicy(null), + fingerprintMapper + ); + Assert.assertEquals(CompactionEligibility.State.NOT_APPLICABLE, status.getState()); + } + + @Test + public void testStatusWhenProjectionsMatch() + { + final GranularitySpec currentGranularitySpec + = new UniformGranularitySpec(Granularities.HOUR, null, null); + final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); + final IndexSpec currentIndexSpec + = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); + final AggregateProjectionSpec projection1 = + AggregateProjectionSpec.builder("foo") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns( + new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + new StringDimensionSchema("a") + ) + .aggregators( + new LongSumAggregatorFactory("sum_long", "long") + ) + .build(); + final CompactionState lastCompactionState = new CompactionState( + currentPartitionsSpec, + null, + null, + null, + currentIndexSpec, + currentGranularitySpec, + List.of(projection1) + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec)) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .withProjections(List.of(projection1)) + .build(); + + final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); + final CompactionEligibility status = CompactionEligibility.evaluate( + CompactionCandidate.ProposedCompaction.from(List.of(segment), Granularities.HOUR), + compactionConfig, + new NewestSegmentFirstPolicy(null), + fingerprintMapper + ); + Assert.assertEquals(CompactionEligibility.NOT_APPLICABLE, status); + } + + @Test + public void testStatusWhenProjectionsMismatch() + { + final GranularitySpec currentGranularitySpec + = new UniformGranularitySpec(Granularities.HOUR, null, null); + final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); + final IndexSpec currentIndexSpec + = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); + final AggregateProjectionSpec projection1 = + AggregateProjectionSpec.builder("1") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns( + new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + new StringDimensionSchema("a") + ) + .aggregators( + new LongSumAggregatorFactory("sum_long", "long") + ) + .build(); + final AggregateProjectionSpec projection2 = + AggregateProjectionSpec.builder("2") + .aggregators(new LongSumAggregatorFactory("sum_long", "long")) + .build(); + + final CompactionState lastCompactionState = new CompactionState( + currentPartitionsSpec, + null, + null, + null, + currentIndexSpec, + currentGranularitySpec, + List.of(projection1) + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec)) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .withProjections(List.of(projection1, projection2)) + .build(); + + final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); + final CompactionEligibility status = CompactionEligibility.evaluate( + CompactionCandidate.ProposedCompaction.from(List.of(segment), Granularities.HOUR), + compactionConfig, + new NewestSegmentFirstPolicy(null), + fingerprintMapper + ); + Assert.assertEquals(CompactionEligibility.State.FULL_COMPACTION, status.getState()); + Assert.assertTrue(status.getReason().contains("'projections' mismatch")); + } + + @Test + public void testStatusWhenAutoSchemaMatch() + { + final GranularitySpec currentGranularitySpec + = new UniformGranularitySpec(Granularities.HOUR, null, null); + final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); + + final CompactionState lastCompactionState = new CompactionState( + currentPartitionsSpec, + DimensionsSpec.builder() + .setDimensions( + List.of( + AutoTypeColumnSchema.of("x").getEffectiveSchema(IndexSpec.getDefault().getEffectiveSpec()), + AutoTypeColumnSchema.of("y").getEffectiveSchema(IndexSpec.getDefault().getEffectiveSpec()) + ) + ) + .build(), + null, + null, + IndexSpec.getDefault().getEffectiveSpec(), + currentGranularitySpec, + Collections.emptyList() + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withDimensionsSpec( + new UserCompactionTaskDimensionsConfig( + List.of( + new AutoTypeColumnSchema( + "x", + null, + NestedCommonFormatColumnFormatSpec.builder() + .setDoubleColumnCompression(CompressionStrategy.LZ4) + .build() + ), + AutoTypeColumnSchema.of("y") + ) + ) + ) + .withTuningConfig(createTuningConfig(currentPartitionsSpec, IndexSpec.getDefault())) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .withProjections(Collections.emptyList()) + .build(); + + final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); + final CompactionEligibility status = CompactionEligibility.evaluate( + CompactionCandidate.ProposedCompaction.from(List.of(segment), null), + compactionConfig, + new NewestSegmentFirstPolicy(null), + fingerprintMapper + ); + Assert.assertEquals(CompactionEligibility.NOT_APPLICABLE, status); + } + + @Test + public void testStatusWhenAutoSchemaMismatch() + { + final GranularitySpec currentGranularitySpec + = new UniformGranularitySpec(Granularities.HOUR, null, null); + final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); + + final CompactionState lastCompactionState = new CompactionState( + currentPartitionsSpec, + DimensionsSpec.builder() + .setDimensions( + List.of( + AutoTypeColumnSchema.of("x").getEffectiveSchema(IndexSpec.getDefault()), + AutoTypeColumnSchema.of("y").getEffectiveSchema(IndexSpec.getDefault()) + ) + ) + .build(), + null, + null, + IndexSpec.getDefault(), + currentGranularitySpec, + Collections.emptyList() + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withDimensionsSpec( + new UserCompactionTaskDimensionsConfig( + List.of( + new AutoTypeColumnSchema( + "x", + null, + NestedCommonFormatColumnFormatSpec.builder() + .setDoubleColumnCompression(CompressionStrategy.ZSTD) + .build() + ), + AutoTypeColumnSchema.of("y") + ) + ) + ) + .withTuningConfig(createTuningConfig(currentPartitionsSpec, IndexSpec.getDefault())) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .withProjections(Collections.emptyList()) + .build(); + + final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); + final CompactionEligibility status = CompactionEligibility.evaluate( + CompactionCandidate.ProposedCompaction.from(List.of(segment), null), + compactionConfig, + new NewestSegmentFirstPolicy(null), + fingerprintMapper + ); + Assert.assertEquals(CompactionEligibility.State.FULL_COMPACTION, status.getState()); + Assert.assertTrue(status.getReason().contains("'dimensionsSpec' mismatch")); + } + + @Test + public void test_evaluate_needsCompactionWhenAllSegmentsHaveUnexpectedIndexingStateFingerprint() + { + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint("wrongFingerprint").build(), + DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint("wrongFingerprint").build() + ); + + final DataSourceCompactionConfig oldCompactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .build(); + CompactionState wrongState = oldCompactionConfig.toCompactionState(); + + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, "wrongFingerprint", wrongState, DateTimes.nowUtc()); + syncCacheFromManager(); + + verifyEvaluationNeedsCompactionBecauseWithCustomSegments( + CompactionCandidate.ProposedCompaction.from(segments, null), + compactionConfig, + "'segmentGranularity' mismatch: required[DAY], current[HOUR]" + ); + } + + @Test + public void test_evaluate_needsCompactionWhenSomeSegmentsHaveUnexpectedIndexingStateFingerprint() + { + final DataSourceCompactionConfig oldCompactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .build(); + CompactionState wrongState = oldCompactionConfig.toCompactionState(); + + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + CompactionState expectedState = compactionConfig.toCompactionState(); + + String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); + + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), + DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint("wrongFingerprint").build() + ); + + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, expectedFingerprint, expectedState, DateTimes.nowUtc()); + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, "wrongFingerprint", wrongState, DateTimes.nowUtc()); + syncCacheFromManager(); + + verifyEvaluationNeedsCompactionBecauseWithCustomSegments( + CompactionCandidate.ProposedCompaction.from(segments, null), + compactionConfig, + "'segmentGranularity' mismatch: required[DAY], current[HOUR]" + ); + } + + @Test + public void test_evaluate_noCompacationIfUnexpectedFingerprintHasExpectedIndexingState() + { + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint("wrongFingerprint").build() + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .build(); + + CompactionState expectedState = compactionConfig.toCompactionState(); + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, "wrongFingerprint", expectedState, DateTimes.nowUtc()); + syncCacheFromManager(); + + final CompactionEligibility status = CompactionEligibility.evaluate( + CompactionCandidate.ProposedCompaction.from(segments, null), + compactionConfig, + new NewestSegmentFirstPolicy(null), + fingerprintMapper + ); + Assert.assertEquals(CompactionEligibility.NOT_APPLICABLE, status); + } + + @Test + public void test_evaluate_needsCompactionWhenUnexpectedFingerprintAndNoFingerprintInMetadataStore() + { + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint("wrongFingerprint").build() + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + verifyEvaluationNeedsCompactionBecauseWithCustomSegments( + CompactionCandidate.ProposedCompaction.from(segments, null), + compactionConfig, + "One or more fingerprinted segments do not have a cached indexing state" + ); + } + + @Test + public void test_evaluate_noCompactionWhenAllSegmentsHaveExpectedIndexingStateFingerprint() + { + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + CompactionState expectedState = compactionConfig.toCompactionState(); + + String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); + + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), + DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint(expectedFingerprint).build() + ); + + final CompactionEligibility status = CompactionEligibility.evaluate( + CompactionCandidate.ProposedCompaction.from(segments, null), + compactionConfig, + new NewestSegmentFirstPolicy(null), + fingerprintMapper + ); + Assert.assertEquals(CompactionEligibility.NOT_APPLICABLE, status); + } + + @Test + public void test_evaluate_needsCompactionWhenNonFingerprintedSegmentsFailChecksOnLastCompactionState() + { + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + CompactionState expectedState = compactionConfig.toCompactionState(); + String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); + + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, expectedFingerprint, expectedState, DateTimes.nowUtc()); + syncCacheFromManager(); + + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), + DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint(null).lastCompactionState(createCompactionStateWithGranularity(Granularities.HOUR)).build() + ); + + + verifyEvaluationNeedsCompactionBecauseWithCustomSegments( + CompactionCandidate.ProposedCompaction.from(segments, null), + compactionConfig, + "'segmentGranularity' mismatch: required[DAY], current[HOUR]" + ); + } + + @Test + public void test_evaluate_noCompactionWhenNonFingerprintedSegmentsPassChecksOnLastCompactionState() + { + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + CompactionState expectedState = compactionConfig.toCompactionState(); + + String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); + + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), + DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint(null).lastCompactionState(createCompactionStateWithGranularity(Granularities.DAY)).build() + ); + + final CompactionEligibility status = CompactionEligibility.evaluate( + CompactionCandidate.ProposedCompaction.from(segments, null), + compactionConfig, + new NewestSegmentFirstPolicy(null), + fingerprintMapper + ); + Assert.assertEquals(CompactionEligibility.NOT_APPLICABLE, status); + } + + // ============================ + // SKIPPED status tests + // ============================ + + @Test + public void test_evaluate_isSkippedWhenInputBytesExceedLimit() + { + // Two segments with 100MB each = 200MB total + // inputSegmentSizeBytes is 150MB, so should be skipped + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withInputSegmentSizeBytes(150_000_000L) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + final CompactionState lastCompactionState = createCompactionStateWithGranularity(Granularities.HOUR); + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(), + DataSegment.builder(WIKI_SEGMENT_2).lastCompactionState(lastCompactionState).build() + ); + + final CompactionEligibility status = CompactionEligibility.evaluate( + CompactionCandidate.ProposedCompaction.from(segments, null), + compactionConfig, + new NewestSegmentFirstPolicy(null), + fingerprintMapper + ); + + Assert.assertFalse(status.getState().equals(CompactionEligibility.State.FULL_COMPACTION)); + Assert.assertTrue(status.getReason().contains("'inputSegmentSize' exceeded")); + Assert.assertTrue(status.getReason().contains("200000000")); + Assert.assertTrue(status.getReason().contains("150000000")); + } + + /** + * Verify that the evaluation indicates compaction is needed for the expected reason. + * Allows customization of the segments in the compaction candidate. + */ + private void verifyEvaluationNeedsCompactionBecauseWithCustomSegments( + CompactionCandidate.ProposedCompaction proposedCompaction, + DataSourceCompactionConfig compactionConfig, + String expectedReason + ) + { + final CompactionEligibility status = CompactionEligibility.evaluate( + proposedCompaction, + compactionConfig, + new NewestSegmentFirstPolicy(null), + fingerprintMapper + ); + + Assert.assertEquals(CompactionEligibility.State.FULL_COMPACTION, status.getState()); + Assert.assertEquals(expectedReason, status.getReason()); + } + + private void verifyCompactionIsEligibleBecause( + CompactionState lastCompactionState, + DataSourceCompactionConfig compactionConfig, + String expectedReason + ) + { + final DataSegment segment + = DataSegment.builder(WIKI_SEGMENT) + .lastCompactionState(lastCompactionState) + .build(); + final CompactionEligibility status = CompactionEligibility.evaluate( + CompactionCandidate.ProposedCompaction.from(List.of(segment), null), + compactionConfig, + new NewestSegmentFirstPolicy(null), + fingerprintMapper + ); + + Assert.assertEquals(CompactionEligibility.State.FULL_COMPACTION, status.getState()); + Assert.assertEquals(expectedReason, status.getReason()); + } + + private static DataSourceCompactionConfig createCompactionConfig( + PartitionsSpec partitionsSpec + ) + { + return InlineSchemaDataSourceCompactionConfig.builder() + .forDataSource(TestDataSource.WIKI) + .withTuningConfig(createTuningConfig(partitionsSpec, null)) + .build(); + } + + private static UserCompactionTaskQueryTuningConfig createTuningConfig( + PartitionsSpec partitionsSpec, + IndexSpec indexSpec + ) + { + return new UserCompactionTaskQueryTuningConfig( + null, + null, null, null, null, partitionsSpec, indexSpec, null, null, + null, null, null, null, null, null, null, null, null, null + ); + } + + /** + * Simple helper to create a CompactionState with only segmentGranularity set + */ + private static CompactionState createCompactionStateWithGranularity(Granularity segmentGranularity) + { + return new CompactionState( + null, + null, + null, + null, + IndexSpec.getDefault(), + new UniformGranularitySpec(segmentGranularity, null, null, null), + null + ); + } +} diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityTest.java index 5c96fd2e10c4..4aed19cb4668 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityTest.java @@ -19,22 +19,177 @@ package org.apache.druid.server.compaction; +import org.apache.druid.error.DruidException; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.server.coordinator.CreateDataSegments; +import org.apache.druid.timeline.DataSegment; import org.junit.Assert; import org.junit.Test; +import java.util.Collections; +import java.util.List; + public class CompactionEligibilityTest { + private static final String DATASOURCE = "test_datasource"; + + @Test + public void testNotApplicable() + { + CompactionEligibility eligibility = CompactionEligibility.NOT_APPLICABLE; + + Assert.assertEquals(CompactionEligibility.State.NOT_APPLICABLE, eligibility.getState()); + Assert.assertEquals("", eligibility.getReason()); + Assert.assertNull(eligibility.getCompactedStats()); + Assert.assertNull(eligibility.getUncompactedStats()); + Assert.assertNull(eligibility.getUncompactedSegments()); + } + + @Test + public void testFail() + { + CompactionEligibility eligibility = CompactionEligibility.fail("test reason: %s", "failure"); + + Assert.assertEquals(CompactionEligibility.State.NOT_ELIGIBLE, eligibility.getState()); + Assert.assertEquals("test reason: failure", eligibility.getReason()); + Assert.assertNull(eligibility.getCompactedStats()); + Assert.assertNull(eligibility.getUncompactedStats()); + Assert.assertNull(eligibility.getUncompactedSegments()); + } + + @Test + public void testBuilderWithCompactionStats() + { + CompactionStatistics compactedStats = CompactionStatistics.create(1000, 5, 2); + CompactionStatistics uncompactedStats = CompactionStatistics.create(500, 3, 1); + List uncompactedSegments = createTestSegments(3); + + CompactionEligibility eligibility = + CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "needs full compaction") + .compacted(compactedStats) + .uncompacted(uncompactedStats) + .uncompactedSegments(uncompactedSegments) + .build(); + + Assert.assertEquals(CompactionEligibility.State.FULL_COMPACTION, eligibility.getState()); + Assert.assertEquals("needs full compaction", eligibility.getReason()); + Assert.assertEquals(compactedStats, eligibility.getCompactedStats()); + Assert.assertEquals(uncompactedStats, eligibility.getUncompactedStats()); + Assert.assertEquals(uncompactedSegments, eligibility.getUncompactedSegments()); + } + @Test public void testEqualsAndHashCode() { - final CompactionEligibility e1 = CompactionEligibility.fail("reason"); - final CompactionEligibility e2 = CompactionEligibility.fail("reason"); - final CompactionEligibility e3 = CompactionEligibility.fail("different"); - final CompactionEligibility e4 = CompactionEligibility.incrementalCompaction("reason"); - - Assert.assertEquals(e1, e2); - Assert.assertEquals(e1.hashCode(), e2.hashCode()); - Assert.assertNotEquals(e1, e3); - Assert.assertNotEquals(e1, e4); + // Test with simple eligibility objects (same state and reason) + CompactionEligibility simple1 = CompactionEligibility.fail("reason"); + CompactionEligibility simple2 = CompactionEligibility.fail("reason"); + Assert.assertEquals(simple1, simple2); + Assert.assertEquals(simple1.hashCode(), simple2.hashCode()); + + // Test with different reasons + CompactionEligibility differentReason = CompactionEligibility.fail("different"); + Assert.assertNotEquals(simple1, differentReason); + + // Test with different states + CompactionEligibility differentState = CompactionEligibility.NOT_APPLICABLE; + Assert.assertNotEquals(simple1, differentState); + + // Test with full compaction eligibility (with stats and segments) + CompactionStatistics stats1 = CompactionStatistics.create(1000, 5, 2); + CompactionStatistics stats2 = CompactionStatistics.create(500, 3, 1); + List segments = createTestSegments(3); + + CompactionEligibility withStats1 = + CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "reason") + .compacted(stats1) + .uncompacted(stats2) + .uncompactedSegments(segments) + .build(); + + CompactionEligibility withStats2 = + CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "reason") + .compacted(stats1) + .uncompacted(stats2) + .uncompactedSegments(segments) + .build(); + + // Same values - should be equal + Assert.assertEquals(withStats1, withStats2); + Assert.assertEquals(withStats1.hashCode(), withStats2.hashCode()); + + // Test with different compacted stats + CompactionStatistics differentStats = CompactionStatistics.create(2000, 10, 5); + CompactionEligibility differentCompactedStats = + CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "reason") + .compacted(differentStats) + .uncompacted(stats2) + .uncompactedSegments(segments) + .build(); + Assert.assertNotEquals(withStats1, differentCompactedStats); + + // Test with different uncompacted stats + CompactionEligibility differentUncompactedStats = + CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "reason") + .compacted(stats1) + .uncompacted(differentStats) + .uncompactedSegments(segments) + .build(); + Assert.assertNotEquals(withStats1, differentUncompactedStats); + + // Test with different segment lists + List differentSegments = createTestSegments(5); + CompactionEligibility differentSegmentList = + CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "reason") + .compacted(stats1) + .uncompacted(stats2) + .uncompactedSegments(differentSegments) + .build(); + Assert.assertNotEquals(withStats1, differentSegmentList); + } + + @Test + public void testBuilderRequiresReasonForNotEligible() + { + Assert.assertThrows( + DruidException.class, + () -> CompactionEligibility.builder(CompactionEligibility.State.NOT_ELIGIBLE, null).build() + ); + } + + @Test + public void testBuilderRequiresStatsForFullCompaction() + { + Assert.assertThrows( + DruidException.class, + () -> CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "reason").build() + ); + + Assert.assertThrows( + DruidException.class, + () -> CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "reason") + .compacted(CompactionStatistics.create(1000, 5, 2)) + .build() + ); + + Assert.assertThrows( + DruidException.class, + () -> CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "reason") + .compacted(CompactionStatistics.create(1000, 5, 2)) + .uncompacted(CompactionStatistics.create(500, 3, 1)) + .build() + ); + } + + private static List createTestSegments(int count) + { + if (count == 0) { + return Collections.emptyList(); + } + + return CreateDataSegments.ofDatasource(DATASOURCE) + .forIntervals(count, Granularities.DAY) + .startingAt("2024-01-01") + .eachOfSizeInMb(100); } } diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java index 2b274e805d4c..5f44f9968d9a 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java @@ -19,869 +19,52 @@ package org.apache.druid.server.compaction; -import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig; -import org.apache.druid.data.input.impl.AggregateProjectionSpec; -import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.impl.LongDimensionSchema; -import org.apache.druid.data.input.impl.StringDimensionSchema; -import org.apache.druid.indexer.granularity.GranularitySpec; -import org.apache.druid.indexer.granularity.UniformGranularitySpec; -import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec; -import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; -import org.apache.druid.indexer.partitions.HashedPartitionsSpec; -import org.apache.druid.indexer.partitions.PartitionsSpec; -import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.query.aggregation.LongSumAggregatorFactory; -import org.apache.druid.segment.AutoTypeColumnSchema; -import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.TestDataSource; -import org.apache.druid.segment.data.CompressionStrategy; -import org.apache.druid.segment.metadata.DefaultIndexingStateFingerprintMapper; -import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; -import org.apache.druid.segment.metadata.IndexingStateCache; -import org.apache.druid.segment.metadata.IndexingStateFingerprintMapper; -import org.apache.druid.segment.nested.NestedCommonFormatColumnFormatSpec; -import org.apache.druid.server.coordinator.DataSourceCompactionConfig; -import org.apache.druid.server.coordinator.InlineSchemaDataSourceCompactionConfig; -import org.apache.druid.server.coordinator.UserCompactionTaskDimensionsConfig; -import org.apache.druid.server.coordinator.UserCompactionTaskGranularityConfig; -import org.apache.druid.server.coordinator.UserCompactionTaskQueryTuningConfig; -import org.apache.druid.timeline.CompactionState; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.SegmentId; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; -import java.util.Collections; -import java.util.List; - public class CompactionStatusTest { - private static final DataSegment WIKI_SEGMENT - = DataSegment.builder(SegmentId.of(TestDataSource.WIKI, Intervals.of("2013-01-01/PT1H"), "v1", 0)) - .size(100_000_000L) - .build(); - private static final DataSegment WIKI_SEGMENT_2 - = DataSegment.builder(SegmentId.of(TestDataSource.WIKI, Intervals.of("2013-01-01/PT1H"), "v1", 1)) - .size(100_000_000L) - .build(); - - private HeapMemoryIndexingStateStorage indexingStateStorage; - private IndexingStateCache indexingStateCache; - private IndexingStateFingerprintMapper fingerprintMapper; - - @Before - public void setUp() - { - indexingStateStorage = new HeapMemoryIndexingStateStorage(); - indexingStateCache = new IndexingStateCache(); - fingerprintMapper = new DefaultIndexingStateFingerprintMapper( - indexingStateCache, - new DefaultObjectMapper() - ); - } - - /** - * Helper to sync the cache with states stored in the manager (for tests that persist states). - */ - private void syncCacheFromManager() - { - indexingStateCache.resetIndexingStatesForPublishedSegments(indexingStateStorage.getAllStoredStates()); - } - - @Test - public void testFindPartitionsSpecWhenGivenIsNull() - { - final ClientCompactionTaskQueryTuningConfig tuningConfig - = ClientCompactionTaskQueryTuningConfig.from(null); - Assert.assertNull( - CompactionStatus.findPartitionsSpecFromConfig(tuningConfig) - ); - } - - @Test - public void testFindPartitionsSpecWhenGivenIsDynamicWithNullMaxTotalRows() - { - final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(null, null); - final ClientCompactionTaskQueryTuningConfig tuningConfig - = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); - Assert.assertEquals( - new DynamicPartitionsSpec(null, Long.MAX_VALUE), - CompactionStatus.findPartitionsSpecFromConfig(tuningConfig) - ); - } - - @Test - public void testFindPartitionsSpecWhenGivenIsDynamicWithMaxTotalRows() - { - final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(null, 1000L); - final ClientCompactionTaskQueryTuningConfig tuningConfig - = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); - Assert.assertEquals( - partitionsSpec, - CompactionStatus.findPartitionsSpecFromConfig(tuningConfig) - ); - } - - @Test - public void testFindPartitionsSpecWhenGivenIsDynamicWithMaxRowsPerSegment() - { - final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(100, 1000L); - final ClientCompactionTaskQueryTuningConfig tuningConfig - = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); - Assert.assertEquals( - partitionsSpec, - CompactionStatus.findPartitionsSpecFromConfig(tuningConfig) - ); - } - - @Test - public void testFindPartitionsSpecFromConfigWithDeprecatedMaxRowsPerSegmentAndMaxTotalRowsReturnGivenValues() - { - final DataSourceCompactionConfig config = - InlineSchemaDataSourceCompactionConfig.builder() - .forDataSource("datasource") - .withMaxRowsPerSegment(100) - .withTuningConfig( - new UserCompactionTaskQueryTuningConfig( - null, - null, - null, - 1000L, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null - ) - ) - .build(); - Assert.assertEquals( - new DynamicPartitionsSpec(100, 1000L), - CompactionStatus.findPartitionsSpecFromConfig( - ClientCompactionTaskQueryTuningConfig.from(config) - ) - ); - } - - @Test - public void testFindPartitionsSpecWhenGivenIsHashed() - { - final PartitionsSpec partitionsSpec = - new HashedPartitionsSpec(null, 100, Collections.singletonList("dim")); - final ClientCompactionTaskQueryTuningConfig tuningConfig - = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); - Assert.assertEquals( - partitionsSpec, - CompactionStatus.findPartitionsSpecFromConfig(tuningConfig) - ); - } - - @Test - public void testFindPartitionsSpecWhenGivenIsRangeWithMaxRows() - { - final PartitionsSpec partitionsSpec = - new DimensionRangePartitionsSpec(null, 10000, Collections.singletonList("dim"), false); - final ClientCompactionTaskQueryTuningConfig tuningConfig - = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); - Assert.assertEquals( - partitionsSpec, - CompactionStatus.findPartitionsSpecFromConfig(tuningConfig) - ); - } - - @Test - public void testFindPartitionsSpecWhenGivenIsRangeWithTargetRows() - { - final PartitionsSpec partitionsSpec = - new DimensionRangePartitionsSpec(10000, null, Collections.singletonList("dim"), false); - final ClientCompactionTaskQueryTuningConfig tuningConfig - = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); - Assert.assertEquals( - new DimensionRangePartitionsSpec(null, 15000, Collections.singletonList("dim"), false), - CompactionStatus.findPartitionsSpecFromConfig(tuningConfig) - ); - } - @Test - public void testStatusWhenLastCompactionStateIsNull() + public void testCompleteConstant() { - verifyCompactionStatusIsPendingBecause( - null, - InlineSchemaDataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(), - "not compacted yet" - ); - } - - @Test - public void testStatusWhenLastCompactionStateIsEmpty() - { - final PartitionsSpec requiredPartitionsSpec = new DynamicPartitionsSpec(5_000_000, null); - verifyCompactionStatusIsPendingBecause( - new CompactionState(null, null, null, null, null, null, null), - InlineSchemaDataSourceCompactionConfig - .builder() - .withTuningConfig(createTuningConfig(requiredPartitionsSpec, null)) - .forDataSource(TestDataSource.WIKI) - .build(), - "'partitionsSpec' mismatch: required['dynamic' with 5,000,000 rows], current[null]" - ); - } - - @Test - public void testStatusOnPartitionsSpecMismatch() - { - final PartitionsSpec requiredPartitionsSpec = new DynamicPartitionsSpec(5_000_000, null); - final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); - - final CompactionState lastCompactionState - = new CompactionState(currentPartitionsSpec, null, null, null, null, null, null); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .withTuningConfig(createTuningConfig(requiredPartitionsSpec, null)) - .forDataSource(TestDataSource.WIKI) - .build(); - - verifyCompactionStatusIsPendingBecause( - lastCompactionState, - compactionConfig, - "'partitionsSpec' mismatch: required['dynamic' with 5,000,000 rows]," - + " current['dynamic' with 100 rows]" - ); - } - - @Test - public void testStatusOnIndexSpecMismatch() - { - final IndexSpec currentIndexSpec - = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); - - final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); - final CompactionState lastCompactionState = new CompactionState( - currentPartitionsSpec, - null, - null, - null, - currentIndexSpec, - null, - null - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withTuningConfig(createTuningConfig(currentPartitionsSpec, null)) - .build(); - - verifyCompactionStatusIsPendingBecause( - lastCompactionState, - compactionConfig, - "'indexSpec' mismatch: " - + "required[IndexSpec{bitmapSerdeFactory=RoaringBitmapSerdeFactory{}," - + " metadataCompression=none," - + " dimensionCompression=lz4, stringDictionaryEncoding=Utf8{}," - + " metricCompression=lz4, longEncoding=longs, complexMetricCompression=null," - + " autoColumnFormatSpec=null, jsonCompression=null, segmentLoader=null}], " - + "current[IndexSpec{bitmapSerdeFactory=RoaringBitmapSerdeFactory{}," - + " metadataCompression=none," - + " dimensionCompression=zstd, stringDictionaryEncoding=Utf8{}," - + " metricCompression=lz4, longEncoding=longs, complexMetricCompression=null," - + " autoColumnFormatSpec=null, jsonCompression=null, segmentLoader=null}]" - ); - } - - @Test - public void testStatusOnSegmentGranularityMismatch() - { - final GranularitySpec currentGranularitySpec - = new UniformGranularitySpec(Granularities.HOUR, null, null); - - final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); - final IndexSpec currentIndexSpec - = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); - final CompactionState lastCompactionState = new CompactionState( - currentPartitionsSpec, - null, - null, - null, - currentIndexSpec, - currentGranularitySpec, - null - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec)) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) - .build(); + CompactionStatus status = CompactionStatus.COMPLETE; - verifyCompactionStatusIsPendingBecause( - lastCompactionState, - compactionConfig, - "'segmentGranularity' mismatch: required[DAY], current[HOUR]" - ); - } - - @Test - public void testStatusWhenLastCompactionStateSameAsRequired() - { - final GranularitySpec currentGranularitySpec - = new UniformGranularitySpec(Granularities.HOUR, null, null); - final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); - final IndexSpec currentIndexSpec - = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); - final CompactionState lastCompactionState = new CompactionState( - currentPartitionsSpec, - null, - null, - null, - currentIndexSpec, - currentGranularitySpec, - null - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec)) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) - .build(); - - final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); - final CompactionStatus status = CompactionStatus.compute( - CompactionCandidate.from(List.of(segment), Granularities.HOUR), - compactionConfig, - fingerprintMapper - ); - Assert.assertTrue(status.isComplete()); - } - - @Test - public void testStatusWhenProjectionsMatch() - { - final GranularitySpec currentGranularitySpec - = new UniformGranularitySpec(Granularities.HOUR, null, null); - final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); - final IndexSpec currentIndexSpec - = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); - final AggregateProjectionSpec projection1 = - AggregateProjectionSpec.builder("foo") - .virtualColumns( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ) - .groupingColumns( - new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - new StringDimensionSchema("a") - ) - .aggregators( - new LongSumAggregatorFactory("sum_long", "long") - ) - .build(); - final CompactionState lastCompactionState = new CompactionState( - currentPartitionsSpec, - null, - null, - null, - currentIndexSpec, - currentGranularitySpec, - List.of(projection1) - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec)) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) - .withProjections(List.of(projection1)) - .build(); - - final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); - final CompactionStatus status = CompactionStatus.compute( - CompactionCandidate.from(List.of(segment), Granularities.HOUR), - compactionConfig, - fingerprintMapper - ); + Assert.assertEquals(CompactionStatus.State.COMPLETE, status.getState()); + Assert.assertNull(status.getReason()); Assert.assertTrue(status.isComplete()); + Assert.assertFalse(status.isSkipped()); } @Test - public void testStatusWhenProjectionsMismatch() + public void testPendingFactoryMethod() { - final GranularitySpec currentGranularitySpec - = new UniformGranularitySpec(Granularities.HOUR, null, null); - final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); - final IndexSpec currentIndexSpec - = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); - final AggregateProjectionSpec projection1 = - AggregateProjectionSpec.builder("1") - .virtualColumns( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ) - .groupingColumns( - new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - new StringDimensionSchema("a") - ) - .aggregators( - new LongSumAggregatorFactory("sum_long", "long") - ) - .build(); - final AggregateProjectionSpec projection2 = - AggregateProjectionSpec.builder("2") - .aggregators(new LongSumAggregatorFactory("sum_long", "long")) - .build(); - - final CompactionState lastCompactionState = new CompactionState( - currentPartitionsSpec, - null, - null, - null, - currentIndexSpec, - currentGranularitySpec, - List.of(projection1) - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec)) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) - .withProjections(List.of(projection1, projection2)) - .build(); + CompactionStatus status = CompactionStatus.pending("needs compaction: %d segments", 5); - final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); - final CompactionStatus status = CompactionStatus.compute( - CompactionCandidate.from(List.of(segment), Granularities.HOUR), - compactionConfig, - fingerprintMapper - ); + Assert.assertEquals(CompactionStatus.State.PENDING, status.getState()); + Assert.assertEquals("needs compaction: 5 segments", status.getReason()); Assert.assertFalse(status.isComplete()); + Assert.assertFalse(status.isSkipped()); } @Test - public void testStatusWhenAutoSchemaMatch() - { - final GranularitySpec currentGranularitySpec - = new UniformGranularitySpec(Granularities.HOUR, null, null); - final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); - - final CompactionState lastCompactionState = new CompactionState( - currentPartitionsSpec, - DimensionsSpec.builder() - .setDimensions( - List.of( - AutoTypeColumnSchema.of("x").getEffectiveSchema(IndexSpec.getDefault().getEffectiveSpec()), - AutoTypeColumnSchema.of("y").getEffectiveSchema(IndexSpec.getDefault().getEffectiveSpec()) - ) - ) - .build(), - null, - null, - IndexSpec.getDefault().getEffectiveSpec(), - currentGranularitySpec, - Collections.emptyList() - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withDimensionsSpec( - new UserCompactionTaskDimensionsConfig( - List.of( - new AutoTypeColumnSchema( - "x", - null, - NestedCommonFormatColumnFormatSpec.builder() - .setDoubleColumnCompression(CompressionStrategy.LZ4) - .build() - ), - AutoTypeColumnSchema.of("y") - ) - ) - ) - .withTuningConfig(createTuningConfig(currentPartitionsSpec, IndexSpec.getDefault())) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) - .withProjections(Collections.emptyList()) - .build(); - - final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); - final CompactionStatus status = CompactionStatus.compute( - CompactionCandidate.from(List.of(segment), null), - compactionConfig, - fingerprintMapper - ); - Assert.assertTrue(status.isComplete()); - } - - @Test - public void testStatusWhenAutoSchemaMismatch() + public void testSkippedFactoryMethod() { - final GranularitySpec currentGranularitySpec - = new UniformGranularitySpec(Granularities.HOUR, null, null); - final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); - - final CompactionState lastCompactionState = new CompactionState( - currentPartitionsSpec, - DimensionsSpec.builder() - .setDimensions( - List.of( - AutoTypeColumnSchema.of("x").getEffectiveSchema(IndexSpec.getDefault()), - AutoTypeColumnSchema.of("y").getEffectiveSchema(IndexSpec.getDefault()) - ) - ) - .build(), - null, - null, - IndexSpec.getDefault(), - currentGranularitySpec, - Collections.emptyList() - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withDimensionsSpec( - new UserCompactionTaskDimensionsConfig( - List.of( - new AutoTypeColumnSchema( - "x", - null, - NestedCommonFormatColumnFormatSpec.builder() - .setDoubleColumnCompression(CompressionStrategy.ZSTD) - .build() - ), - AutoTypeColumnSchema.of("y") - ) - ) - ) - .withTuningConfig(createTuningConfig(currentPartitionsSpec, IndexSpec.getDefault())) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) - .withProjections(Collections.emptyList()) - .build(); - - final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); - final CompactionStatus status = CompactionStatus.compute( - CompactionCandidate.from(List.of(segment), null), - compactionConfig, - fingerprintMapper - ); - Assert.assertFalse(status.isComplete()); - } - - @Test - public void test_evaluate_needsCompactionWhenAllSegmentsHaveUnexpectedIndexingStateFingerprint() - { - List segments = List.of( - DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint("wrongFingerprint").build(), - DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint("wrongFingerprint").build() - ); - - final DataSourceCompactionConfig oldCompactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) - .build(); - CompactionState wrongState = oldCompactionConfig.toCompactionState(); - - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) - .build(); - - indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, "wrongFingerprint", wrongState, DateTimes.nowUtc()); - syncCacheFromManager(); - - verifyEvaluationNeedsCompactionBecauseWithCustomSegments( - CompactionCandidate.from(segments, null), - compactionConfig, - "'segmentGranularity' mismatch: required[DAY], current[HOUR]" - ); - } - - @Test - public void test_evaluate_needsCompactionWhenSomeSegmentsHaveUnexpectedIndexingStateFingerprint() - { - final DataSourceCompactionConfig oldCompactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) - .build(); - CompactionState wrongState = oldCompactionConfig.toCompactionState(); - - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) - .build(); - - CompactionState expectedState = compactionConfig.toCompactionState(); - - String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); - - List segments = List.of( - DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), - DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint("wrongFingerprint").build() - ); - - indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, expectedFingerprint, expectedState, DateTimes.nowUtc()); - indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, "wrongFingerprint", wrongState, DateTimes.nowUtc()); - syncCacheFromManager(); - - verifyEvaluationNeedsCompactionBecauseWithCustomSegments( - CompactionCandidate.from(segments, null), - compactionConfig, - "'segmentGranularity' mismatch: required[DAY], current[HOUR]" - ); - } - - @Test - public void test_evaluate_noCompacationIfUnexpectedFingerprintHasExpectedIndexingState() - { - List segments = List.of( - DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint("wrongFingerprint").build() - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) - .build(); - - CompactionState expectedState = compactionConfig.toCompactionState(); - indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, "wrongFingerprint", expectedState, DateTimes.nowUtc()); - syncCacheFromManager(); - - final CompactionStatus status = CompactionStatus.compute( - CompactionCandidate.from(segments, null), - compactionConfig, - fingerprintMapper - ); - Assert.assertTrue(status.isComplete()); - } - - @Test - public void test_evaluate_needsCompactionWhenUnexpectedFingerprintAndNoFingerprintInMetadataStore() - { - List segments = List.of( - DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint("wrongFingerprint").build() - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) - .build(); - - verifyEvaluationNeedsCompactionBecauseWithCustomSegments( - CompactionCandidate.from(segments, null), - compactionConfig, - "One or more fingerprinted segments do not have a cached indexing state" - ); - } - - @Test - public void test_evaluate_noCompactionWhenAllSegmentsHaveExpectedIndexingStateFingerprint() - { - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) - .build(); - - CompactionState expectedState = compactionConfig.toCompactionState(); - - String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); - - List segments = List.of( - DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), - DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint(expectedFingerprint).build() - ); - - final CompactionStatus status = CompactionStatus.compute( - CompactionCandidate.from(segments, null), - compactionConfig, - fingerprintMapper - ); - Assert.assertTrue(status.isComplete()); - } - - @Test - public void test_evaluate_needsCompactionWhenNonFingerprintedSegmentsFailChecksOnLastCompactionState() - { - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) - .build(); - - CompactionState expectedState = compactionConfig.toCompactionState(); - String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); - - indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, expectedFingerprint, expectedState, DateTimes.nowUtc()); - syncCacheFromManager(); - - List segments = List.of( - DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), - DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint(null).lastCompactionState(createCompactionStateWithGranularity(Granularities.HOUR)).build() - ); - - - verifyEvaluationNeedsCompactionBecauseWithCustomSegments( - CompactionCandidate.from(segments, null), - compactionConfig, - "'segmentGranularity' mismatch: required[DAY], current[HOUR]" - ); - } - - @Test - public void test_evaluate_noCompactionWhenNonFingerprintedSegmentsPassChecksOnLastCompactionState() - { - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) - .build(); - - CompactionState expectedState = compactionConfig.toCompactionState(); - - String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); - - List segments = List.of( - DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), - DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint(null).lastCompactionState(createCompactionStateWithGranularity(Granularities.DAY)).build() - ); - - final CompactionStatus status = CompactionStatus.compute( - CompactionCandidate.from(segments, null), - compactionConfig, - fingerprintMapper - ); - Assert.assertTrue(status.isComplete()); - } - - // ============================ - // SKIPPED status tests - // ============================ - - @Test - public void test_evaluate_isSkippedWhenInputBytesExceedLimit() - { - // Two segments with 100MB each = 200MB total - // inputSegmentSizeBytes is 150MB, so should be skipped - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withInputSegmentSizeBytes(150_000_000L) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) - .build(); - - final CompactionState lastCompactionState = createCompactionStateWithGranularity(Granularities.HOUR); - List segments = List.of( - DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(), - DataSegment.builder(WIKI_SEGMENT_2).lastCompactionState(lastCompactionState).build() - ); - - final CompactionStatus status = CompactionStatus.compute( - CompactionCandidate.from(segments, null), - compactionConfig, - fingerprintMapper - ); + CompactionStatus status = CompactionStatus.skipped("already compacted"); + Assert.assertEquals(CompactionStatus.State.SKIPPED, status.getState()); + Assert.assertEquals("already compacted", status.getReason()); Assert.assertFalse(status.isComplete()); Assert.assertTrue(status.isSkipped()); - Assert.assertTrue(status.getReason().contains("'inputSegmentSize' exceeded")); - Assert.assertTrue(status.getReason().contains("200000000")); - Assert.assertTrue(status.getReason().contains("150000000")); - } - - /** - * Verify that the evaluation indicates compaction is needed for the expected reason. - * Allows customization of the segments in the compaction candidate. - */ - private void verifyEvaluationNeedsCompactionBecauseWithCustomSegments( - CompactionCandidate candidate, - DataSourceCompactionConfig compactionConfig, - String expectedReason - ) - { - final CompactionStatus status = CompactionStatus.compute( - candidate, - compactionConfig, - fingerprintMapper - ); - - Assert.assertFalse(status.isComplete()); - Assert.assertEquals(expectedReason, status.getReason()); } - private void verifyCompactionStatusIsPendingBecause( - CompactionState lastCompactionState, - DataSourceCompactionConfig compactionConfig, - String expectedReason - ) + @Test + public void testRunningFactoryMethod() { - final DataSegment segment - = DataSegment.builder(WIKI_SEGMENT) - .lastCompactionState(lastCompactionState) - .build(); - final CompactionStatus status = CompactionStatus.compute( - CompactionCandidate.from(List.of(segment), null), - compactionConfig, - fingerprintMapper - ); + CompactionStatus status = CompactionStatus.running("task-123"); + Assert.assertEquals(CompactionStatus.State.RUNNING, status.getState()); + Assert.assertEquals("task-123", status.getReason()); Assert.assertFalse(status.isComplete()); - Assert.assertEquals(expectedReason, status.getReason()); - } - - private static DataSourceCompactionConfig createCompactionConfig( - PartitionsSpec partitionsSpec - ) - { - return InlineSchemaDataSourceCompactionConfig.builder() - .forDataSource(TestDataSource.WIKI) - .withTuningConfig(createTuningConfig(partitionsSpec, null)) - .build(); - } - - private static UserCompactionTaskQueryTuningConfig createTuningConfig( - PartitionsSpec partitionsSpec, - IndexSpec indexSpec - ) - { - return new UserCompactionTaskQueryTuningConfig( - null, - null, null, null, null, partitionsSpec, indexSpec, null, null, - null, null, null, null, null, null, null, null, null, null - ); - } - - /** - * Simple helper to create a CompactionState with only segmentGranularity set - */ - private static CompactionState createCompactionStateWithGranularity(Granularity segmentGranularity) - { - return new CompactionState( - null, - null, - null, - null, - IndexSpec.getDefault(), - new UniformGranularitySpec(segmentGranularity, null, null, null), - null - ); + Assert.assertFalse(status.isSkipped()); } } diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java index b9bd4acf4a17..e08fb2c9184f 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java @@ -22,6 +22,7 @@ import org.apache.druid.indexer.TaskState; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.segment.TestDataSource; import org.apache.druid.server.coordinator.CreateDataSegments; import org.apache.druid.timeline.DataSegment; @@ -29,6 +30,7 @@ import org.junit.Before; import org.junit.Test; +import javax.annotation.Nullable; import java.util.List; public class CompactionStatusTrackerTest @@ -47,8 +49,7 @@ public void setup() @Test public void testGetLatestTaskStatusForSubmittedTask() { - final CompactionCandidate candidateSegments - = CompactionCandidate.from(List.of(WIKI_SEGMENT), null); + final CompactionCandidate candidateSegments = createCandidate(List.of(WIKI_SEGMENT), null); statusTracker.onTaskSubmitted("task1", candidateSegments); CompactionTaskStatus status = statusTracker.getLatestTaskStatus(candidateSegments); @@ -58,8 +59,7 @@ public void testGetLatestTaskStatusForSubmittedTask() @Test public void testGetLatestTaskStatusForSuccessfulTask() { - final CompactionCandidate candidateSegments - = CompactionCandidate.from(List.of(WIKI_SEGMENT), null); + final CompactionCandidate candidateSegments = createCandidate(List.of(WIKI_SEGMENT), null); statusTracker.onTaskSubmitted("task1", candidateSegments); statusTracker.onTaskFinished("task1", TaskStatus.success("task1")); @@ -70,8 +70,7 @@ public void testGetLatestTaskStatusForSuccessfulTask() @Test public void testGetLatestTaskStatusForFailedTask() { - final CompactionCandidate candidateSegments - = CompactionCandidate.from(List.of(WIKI_SEGMENT), null); + final CompactionCandidate candidateSegments = createCandidate(List.of(WIKI_SEGMENT), null); statusTracker.onTaskSubmitted("task1", candidateSegments); statusTracker.onTaskFinished("task1", TaskStatus.failure("task1", "some failure")); @@ -83,8 +82,7 @@ public void testGetLatestTaskStatusForFailedTask() @Test public void testGetLatestTaskStatusForRepeatedlyFailingTask() { - final CompactionCandidate candidateSegments - = CompactionCandidate.from(List.of(WIKI_SEGMENT), null); + final CompactionCandidate candidateSegments = createCandidate(List.of(WIKI_SEGMENT), null); statusTracker.onTaskSubmitted("task1", candidateSegments); statusTracker.onTaskFinished("task1", TaskStatus.failure("task1", "some failure")); @@ -105,8 +103,7 @@ public void testGetLatestTaskStatusForRepeatedlyFailingTask() public void testComputeCompactionStatusForSuccessfulTask() { final NewestSegmentFirstPolicy policy = new NewestSegmentFirstPolicy(null); - final CompactionCandidate candidateSegments - = CompactionCandidate.from(List.of(WIKI_SEGMENT), null); + final CompactionCandidate candidateSegments = createCandidate(List.of(WIKI_SEGMENT), null); // Verify that interval is originally eligible for compaction CompactionStatus status @@ -131,4 +128,20 @@ public void testComputeCompactionStatusForSuccessfulTask() status = statusTracker.computeCompactionStatus(candidateSegments); Assert.assertEquals(CompactionStatus.State.PENDING, status.getState()); } + + public static CompactionCandidate createCandidate( + List segments, + @Nullable Granularity targetSegmentGranularity + ) + { + CompactionCandidate.ProposedCompaction proposedCompaction = CompactionCandidate.ProposedCompaction.from( + segments, + targetSegmentGranularity + ); + return CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "approve without check") + .compacted(CompactionStatistics.create(1, 1, 1)) + .uncompacted(CompactionStatistics.create(1, 1, 1)) + .uncompactedSegments(List.of()).build() + .createCandidate(proposedCompaction); + } } diff --git a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java index 8c8749645d25..d3ce932efeb7 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java @@ -35,6 +35,10 @@ public class MostFragmentedIntervalFirstPolicyTest { private static final DataSegment SEGMENT = CreateDataSegments.ofDatasource(TestDataSource.WIKI).eachOfSizeInMb(100).get(0); + private static final CompactionCandidate.ProposedCompaction PROPOSED_COMPACTION = + CompactionCandidate.ProposedCompaction.from(List.of(SEGMENT), null); + + private static final CompactionStatistics DUMMY_COMPACTION_STATS = CompactionStatistics.create(1L, 1L, 1L); @Test public void test_thresholdValues_ofDefaultPolicy() @@ -59,16 +63,16 @@ public void test_checkEligibilityForCompaction_fails_ifUncompactedCountLessThanC null ); + final CompactionEligibility eligibility1 = + eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(1, 100L)).build(); Assertions.assertEquals( - CompactionEligibility.fail( - "Uncompacted segments[1] in interval must be at least [10,000]" - ), - policy.checkEligibilityForCompaction(createCandidate(1, 100L)) - ); - Assertions.assertEquals( - CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(createCandidate(10_001, 100L)) + CompactionEligibility.fail("Uncompacted segments[1] in interval must be at least [10,000]"), + policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility1) ); + + final CompactionEligibility eligibility2 = + eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(10_001, 100L)).build(); + Assertions.assertEquals(eligibility2, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility2)); } @Test @@ -83,16 +87,16 @@ public void test_checkEligibilityForCompaction_fails_ifUncompactedBytesLessThanC null ); + final CompactionEligibility eligibility1 = + eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(1, 100L)).build(); Assertions.assertEquals( - CompactionEligibility.fail( - "Uncompacted bytes[100] in interval must be at least [10,000]" - ), - policy.checkEligibilityForCompaction(createCandidate(1, 100L)) - ); - Assertions.assertEquals( - CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(createCandidate(100, 10_000L)) + CompactionEligibility.fail("Uncompacted bytes[100] in interval must be at least [10,000]"), + policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility1) ); + + final CompactionEligibility eligibility2 = + eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(100, 10_000L)).build(); + Assertions.assertEquals(eligibility2, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility2)); } @Test @@ -107,16 +111,15 @@ public void test_checkEligibilityForCompaction_fails_ifAvgSegmentSizeGreaterThan null ); + final CompactionEligibility eligibility1 = + eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(1, 10_000L)).build(); Assertions.assertEquals( - CompactionEligibility.fail( - "Average size[10,000] of uncompacted segments in interval must be at most [100]" - ), - policy.checkEligibilityForCompaction(createCandidate(1, 10_000L)) - ); - Assertions.assertEquals( - CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(createCandidate(1, 100L)) + CompactionEligibility.fail("Average size[10,000] of uncompacted segments in interval must be at most [100]"), + policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility1) ); + final CompactionEligibility eligibility2 = + eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(1, 100L)).build(); + Assertions.assertEquals(eligibility2, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility2)); } @Test @@ -130,18 +133,16 @@ public void test_policy_favorsIntervalWithMoreUncompactedSegments_ifTotalBytesIs null ); - final CompactionCandidate candidateA = createCandidate(1, 1000L); - final CompactionCandidate candidateB = createCandidate(2, 500L); + final CompactionEligibility eligibility1 = + eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(1, 1_000L)).build(); + final CompactionEligibility eligibility2 = + eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(2, 500L)).build(); - Assertions.assertEquals( - CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidateA) - ); - Assertions.assertEquals( - CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidateB) - ); + Assertions.assertEquals(eligibility1, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility1)); + Assertions.assertEquals(eligibility2, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility2)); + final CompactionCandidate candidateA = eligibility1.createCandidate(PROPOSED_COMPACTION); + final CompactionCandidate candidateB = eligibility2.createCandidate(PROPOSED_COMPACTION); Assertions.assertTrue(policy.compareCandidates(candidateA, candidateB) > 0); Assertions.assertTrue(policy.compareCandidates(candidateB, candidateA) < 0); } @@ -157,18 +158,17 @@ public void test_policy_favorsIntervalWithMoreUncompactedSegments_ifAverageSizeI null ); - final CompactionCandidate candidateA = createCandidate(1, 1000L); - final CompactionCandidate candidateB = createCandidate(2, 1000L); + final CompactionEligibility eligibility1 = + eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(1, 1000L)).build(); + final CompactionEligibility eligibility2 = + eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(2, 1000L)).build(); - Assertions.assertEquals( - CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidateA) - ); - Assertions.assertEquals( - CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidateB) - ); + Assertions.assertEquals(eligibility1, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility1)); + Assertions.assertEquals(eligibility2, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility2)); + + final CompactionCandidate candidateA = eligibility1.createCandidate(PROPOSED_COMPACTION); + final CompactionCandidate candidateB = eligibility2.createCandidate(PROPOSED_COMPACTION); Assertions.assertTrue(policy.compareCandidates(candidateA, candidateB) > 0); Assertions.assertTrue(policy.compareCandidates(candidateB, candidateA) < 0); } @@ -184,18 +184,16 @@ public void test_policy_favorsIntervalWithSmallerSegments_ifCountIsEqual() null ); - final CompactionCandidate candidateA = createCandidate(10, 500L); - final CompactionCandidate candidateB = createCandidate(10, 1000L); + final CompactionEligibility eligibility1 = + eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(10, 500L)).build(); + final CompactionEligibility eligibility2 = + eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(10, 1000L)).build(); - Assertions.assertEquals( - CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidateA) - ); - Assertions.assertEquals( - CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidateB) - ); + Assertions.assertEquals(eligibility1, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility1)); + Assertions.assertEquals(eligibility2, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility2)); + final CompactionCandidate candidateA = eligibility1.createCandidate(PROPOSED_COMPACTION); + final CompactionCandidate candidateB = eligibility2.createCandidate(PROPOSED_COMPACTION); Assertions.assertTrue(policy.compareCandidates(candidateA, candidateB) < 0); Assertions.assertTrue(policy.compareCandidates(candidateB, candidateA) > 0); } @@ -211,18 +209,16 @@ public void test_compareCandidates_returnsZeroIfSegmentCountAndAvgSizeScaleEquiv null ); - final CompactionCandidate candidateA = createCandidate(100, 25); - final CompactionCandidate candidateB = createCandidate(400, 100); + final CompactionEligibility eligibility1 = + eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(100, 25)).build(); + final CompactionEligibility eligibility2 = + eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(400, 100)).build(); - Assertions.assertEquals( - CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidateA) - ); - Assertions.assertEquals( - CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidateB) - ); + Assertions.assertEquals(eligibility1, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility1)); + Assertions.assertEquals(eligibility2, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility2)); + final CompactionCandidate candidateA = eligibility1.createCandidate(PROPOSED_COMPACTION); + final CompactionCandidate candidateB = eligibility2.createCandidate(PROPOSED_COMPACTION); Assertions.assertEquals(0, policy.compareCandidates(candidateA, candidateB)); Assertions.assertEquals(0, policy.compareCandidates(candidateB, candidateA)); } @@ -275,12 +271,23 @@ public void test_checkEligibilityForCompaction_returnsIncrementalCompaction_when null ); - final CompactionCandidate candidate = createCandidateWithStats(1200L, 10, 400L, 100); + final CompactionEligibility candidate = eligibilityBuilder() + .compacted(CompactionStatistics.create(1200L, 10, 1L)) + .uncompacted(CompactionStatistics.create(400L, 100, 1L)) + .uncompactedSegments(List.of(SEGMENT)) + .build(); + Assertions.assertEquals( - CompactionEligibility.incrementalCompaction( - "Uncompacted bytes ratio[0.25] is below threshold[0.50]"), - policy.checkEligibilityForCompaction(candidate) + CompactionEligibility.builder( + CompactionEligibility.State.INCREMENTAL_COMPACTION, + "Uncompacted bytes ratio[0.25] is below threshold[0.50]" + ) + .compacted(CompactionStatistics.create(1200L, 10, 1L)) + .uncompacted(CompactionStatistics.create(400L, 100, 1L)) + .uncompactedSegments(List.of(SEGMENT)) + .build(), + policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, candidate) ); } @@ -296,11 +303,15 @@ public void test_checkEligibilityForCompaction_returnsFullCompaction_whenRatioAb null ); - final CompactionCandidate candidate = createCandidateWithStats(500L, 5, 600L, 100); + final CompactionEligibility eligibility = + eligibilityBuilder() + .compacted(CompactionStatistics.create(500L, 5, 1)) + .uncompacted(CompactionStatistics.create(600L, 100, 1)) + .build(); Assertions.assertEquals( - CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidate) + eligibility, + policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility) ); } @@ -317,41 +328,23 @@ public void test_checkEligibilityForCompaction_returnsFullCompaction_whenThresho ); // With default threshold 0.0, any positive ratio >= 0.0, so always FULL_COMPACTION_ELIGIBLE - final CompactionCandidate candidate = createCandidateWithStats(1000L, 10, 100L, 100); + final CompactionEligibility eligibility = + eligibilityBuilder() + .compacted(CompactionStatistics.create(1_000L, 10, 1)) + .uncompacted(CompactionStatistics.create(100L, 100, 1)) + .build(); - Assertions.assertEquals( - CompactionEligibility.FULL_COMPACTION_ELIGIBLE, - policy.checkEligibilityForCompaction(candidate) - ); + Assertions.assertEquals(eligibility, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility)); } - private CompactionCandidate createCandidate(int numSegments, long avgSizeBytes) + private CompactionStatistics createStats(int numSegments, long avgSizeBytes) { - final CompactionStatistics dummyCompactedStats = CompactionStatistics.create(1L, 1L, 1L); - final CompactionStatistics uncompactedStats = CompactionStatistics.create( - avgSizeBytes * numSegments, - numSegments, - 1L - ); - return CompactionCandidate.from(List.of(SEGMENT), null) - .withCurrentStatus(CompactionStatus.pending(dummyCompactedStats, uncompactedStats, "")); + return CompactionStatistics.create(avgSizeBytes * numSegments, numSegments, 1L); } - private CompactionCandidate createCandidateWithStats( - long compactedBytes, - int compactedSegments, - long uncompactedBytes, - int uncompactedSegments - ) + private static CompactionEligibility.CompactionEligibilityBuilder eligibilityBuilder() { - final CompactionStatistics compactedStats = CompactionStatistics.create(compactedBytes, compactedSegments, 1L); - final CompactionStatistics uncompactedStats = CompactionStatistics.create( - uncompactedBytes, - uncompactedSegments, - 1L - ); - return CompactionCandidate.from(List.of(SEGMENT), null) - .withCurrentStatus(CompactionStatus.pending(compactedStats, uncompactedStats, "")); + return CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "approve") + .uncompactedSegments(List.of()); } - } diff --git a/server/src/test/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicyTest.java b/server/src/test/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicyTest.java index 4e86abcd2468..91309f56a409 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicyTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicyTest.java @@ -740,7 +740,7 @@ public void testIteratorReturnsNothingAsSegmentsWasCompactedAndHaveSameSegmentGr // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY @@ -769,7 +769,7 @@ public void testIteratorReturnsNothingAsSegmentsWasCompactedAndHaveSameSegmentGr // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY @@ -805,7 +805,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentSeg // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY @@ -853,7 +853,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentSeg // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY @@ -902,7 +902,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentTim // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY @@ -952,7 +952,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentOri // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY @@ -1001,7 +1001,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentRol // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have @@ -1090,7 +1090,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentQue // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have @@ -1185,7 +1185,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentDim // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have @@ -1414,7 +1414,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentFil // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have @@ -1536,7 +1536,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentMet // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have @@ -1688,7 +1688,7 @@ public void testIteratorReturnsSegmentsAsCompactionStateChangedWithCompactedStat { // Different indexSpec as what is set in the auto compaction config IndexSpec newIndexSpec = IndexSpec.builder().withBitmapSerdeFactory(new ConciseBitmapSerdeFactory()).build(); - PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY @@ -1734,7 +1734,7 @@ public void testIteratorReturnsSegmentsAsCompactionStateChangedWithCompactedStat @Test public void testIteratorDoesNotReturnSegmentWithChangingAppendableIndexSpec() { - PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); final SegmentTimeline timeline = createTimeline( createSegments() @@ -2066,13 +2066,13 @@ TestDataSource.KOALA, configBuilder().forDataSource(TestDataSource.KOALA).build( // Verify that the segments of WIKI are preferred even though they are older Assert.assertTrue(iterator.hasNext()); CompactionCandidate next = iterator.next(); - Assert.assertEquals(TestDataSource.WIKI, next.getDataSource()); - Assert.assertEquals(Intervals.of("2012-01-01/P1D"), next.getUmbrellaInterval()); + Assert.assertEquals(TestDataSource.WIKI, next.getProposedCompaction().getDataSource()); + Assert.assertEquals(Intervals.of("2012-01-01/P1D"), next.getProposedCompaction().getUmbrellaInterval()); Assert.assertTrue(iterator.hasNext()); next = iterator.next(); - Assert.assertEquals(TestDataSource.KOALA, next.getDataSource()); - Assert.assertEquals(Intervals.of("2013-01-01/P1D"), next.getUmbrellaInterval()); + Assert.assertEquals(TestDataSource.KOALA, next.getProposedCompaction().getDataSource()); + Assert.assertEquals(Intervals.of("2013-01-01/P1D"), next.getProposedCompaction().getUmbrellaInterval()); } private CompactionSegmentIterator createIterator(DataSourceCompactionConfig config, SegmentTimeline timeline) diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java index 83f838f9517d..5da9b66d1471 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java @@ -79,7 +79,7 @@ import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.incremental.OnheapIncrementalIndex; import org.apache.druid.segment.transform.CompactionTransformSpec; -import org.apache.druid.server.compaction.CompactionCandidate; +import org.apache.druid.server.compaction.CompactionCandidate.ProposedCompaction; import org.apache.druid.server.compaction.CompactionCandidateSearchPolicy; import org.apache.druid.server.compaction.CompactionSlotManager; import org.apache.druid.server.compaction.CompactionStatusTracker; @@ -876,7 +876,7 @@ public void testCompactWithGranularitySpec() // All segments is compact at the same time since we changed the segment granularity to YEAR and all segment // are within the same year Assert.assertEquals( - CompactionCandidate.from(datasourceToSegments.get(dataSource), Granularities.YEAR).getCompactionInterval(), + ProposedCompaction.from(datasourceToSegments.get(dataSource), Granularities.YEAR).getCompactionInterval(), taskPayload.getIoConfig().getInputSpec().getInterval() ); @@ -1071,7 +1071,7 @@ public void testCompactWithRollupInGranularitySpec() // All segments is compact at the same time since we changed the segment granularity to YEAR and all segment // are within the same year Assert.assertEquals( - CompactionCandidate.from(datasourceToSegments.get(dataSource), Granularities.YEAR).getCompactionInterval(), + ProposedCompaction.from(datasourceToSegments.get(dataSource), Granularities.YEAR).getCompactionInterval(), taskPayload.getIoConfig().getInputSpec().getInterval() ); @@ -1171,7 +1171,7 @@ public void testCompactWithGranularitySpecConflictWithActiveCompactionTask() // All segments is compact at the same time since we changed the segment granularity to YEAR and all segment // are within the same year Assert.assertEquals( - CompactionCandidate.from(datasourceToSegments.get(dataSource), Granularities.YEAR).getCompactionInterval(), + ProposedCompaction.from(datasourceToSegments.get(dataSource), Granularities.YEAR).getCompactionInterval(), taskPayload.getIoConfig().getInputSpec().getInterval() ); @@ -1408,7 +1408,7 @@ public void testDetermineSegmentGranularityFromSegmentsToCompact() ClientCompactionTaskQuery taskPayload = (ClientCompactionTaskQuery) payloadCaptor.getValue(); Assert.assertEquals( - CompactionCandidate.from(segments, Granularities.DAY).getCompactionInterval(), + ProposedCompaction.from(segments, Granularities.DAY).getCompactionInterval(), taskPayload.getIoConfig().getInputSpec().getInterval() ); @@ -1471,7 +1471,7 @@ public void testDetermineSegmentGranularityFromSegmentGranularityInCompactionCon ClientCompactionTaskQuery taskPayload = (ClientCompactionTaskQuery) payloadCaptor.getValue(); Assert.assertEquals( - CompactionCandidate.from(segments, Granularities.YEAR).getCompactionInterval(), + ProposedCompaction.from(segments, Granularities.YEAR).getCompactionInterval(), taskPayload.getIoConfig().getInputSpec().getInterval() ); From 3ec2d340b84acd31cfbe03a4d74e9f728f69c23b Mon Sep 17 00:00:00 2001 From: cecemei Date: Thu, 5 Feb 2026 16:42:42 -0800 Subject: [PATCH 10/19] revert drop existing change --- .../druid/server/coordinator/duty/CompactSegments.java | 4 ---- .../druid/server/coordinator/duty/CompactSegmentsTest.java | 7 ++----- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java index 07b8b23d58fa..661458462fab 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java @@ -364,10 +364,6 @@ public static ClientCompactionTaskQuery createCompactionTask( } final CompactionEngine compactionEngine = config.getEngine() == null ? defaultEngine : config.getEngine(); - if (CompactionEngine.MSQ.equals(compactionEngine) && !Boolean.TRUE.equals(dropExisting)) { - LOG.info("Forcing dropExisting to true as required by MSQ engine."); - dropExisting = true; - } final Map autoCompactionContext = newAutoCompactionContext(config.getTaskContext()); if (candidate.getCurrentStatus() != null) { diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java index 5da9b66d1471..02669d6384c4 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java @@ -78,6 +78,7 @@ import org.apache.druid.rpc.indexing.OverlordClient; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.incremental.OnheapIncrementalIndex; +import org.apache.druid.segment.indexing.BatchIOConfig; import org.apache.druid.segment.transform.CompactionTransformSpec; import org.apache.druid.server.compaction.CompactionCandidate.ProposedCompaction; import org.apache.druid.server.compaction.CompactionCandidateSearchPolicy; @@ -837,11 +838,7 @@ public void testCompactWithNullIOConfig() ); doCompactSegments(compactSegments, compactionConfigs); ClientCompactionTaskQuery taskPayload = (ClientCompactionTaskQuery) payloadCaptor.getValue(); - if (CompactionEngine.NATIVE.equals(engine)) { - Assert.assertFalse(taskPayload.getIoConfig().isDropExisting()); - } else { - Assert.assertTrue(taskPayload.getIoConfig().isDropExisting()); - } + Assert.assertEquals(BatchIOConfig.DEFAULT_DROP_EXISTING, taskPayload.getIoConfig().isDropExisting()); } @Test From f9f1bc9c0abfeef09a328b2ab2be48150bddb272 Mon Sep 17 00:00:00 2001 From: cecemei Date: Thu, 5 Feb 2026 16:48:38 -0800 Subject: [PATCH 11/19] javadoc --- .../compaction/CompactionCandidateSearchPolicy.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java index 1cd65b3a53ad..66fe33ff39d4 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java @@ -45,15 +45,12 @@ public interface CompactionCandidateSearchPolicy int compareCandidates(CompactionCandidate candidateA, CompactionCandidate candidateB); /** - * Checks if the given {@link CompactionCandidate} is eligible for compaction - * in the current iteration. A policy may implement this method to skip - * compacting intervals or segments that do not fulfil some required criteria. + * Applies policy-specific eligibility checks to the proposed compaction. + * + * @param candidate the proposed compaction + * @param eligibility initial eligibility from compaction config checks + * @return final eligibility after applying policy checks */ - default CompactionEligibility checkEligibilityForCompaction(CompactionCandidate candidate) - { - return checkEligibilityForCompaction(candidate.getProposedCompaction(), candidate.getPolicyEligibility()); - } - CompactionEligibility checkEligibilityForCompaction( CompactionCandidate.ProposedCompaction candidate, CompactionEligibility eligibility From 5bfcf0c41ae765c5a4a0b6054de99ca29686d691 Mon Sep 17 00:00:00 2001 From: cecemei Date: Thu, 5 Feb 2026 18:38:15 -0800 Subject: [PATCH 12/19] test --- .../druid/indexing/compact/CompactionJobQueue.java | 6 ++---- .../server/compaction/CompactionRunSimulatorTest.java | 3 +-- .../MostFragmentedIntervalFirstPolicyTest.java | 11 ++++++----- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobQueue.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobQueue.java index cdfb41c5f129..e8cb8ada8edc 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobQueue.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobQueue.java @@ -379,10 +379,8 @@ private void persistPendingIndexingState(CompactionJob job) public CompactionStatus getCurrentStatusForJob(CompactionJob job) { - final CompactionStatus compactionStatus = statusTracker.computeCompactionStatus(job.getCandidate()); - final CompactionCandidate candidatesWithStatus = job.getCandidate().withCurrentStatus(null); - statusTracker.onCompactionStatusComputed(candidatesWithStatus, null); - return compactionStatus; + statusTracker.onCompactionStatusComputed(job.getCandidate(), null); + return statusTracker.computeCompactionStatus(job.getCandidate()); } public static CompactionConfigValidationResult validateCompactionJob(BatchIndexingJob job) diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java index 2854520b01f6..afed6f00abcc 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java @@ -177,8 +177,7 @@ public void testSimulate_withFixedIntervalOrderPolicy() List.of("dataSource", "interval", "numSegments", "bytes", "reasonToSkip"), skippedTable.getColumnNames() ); - final String rejectedMessage - = "Rejected by search policy: Datasource/Interval is not in the list of 'eligibleCandidates'"; + final String rejectedMessage = "Datasource/Interval is not in the list of 'eligibleCandidates'"; Assert.assertEquals( List.of( List.of("wiki", Intervals.of("2013-01-10/P1D"), 10, 1_000_000_000L, 1, "skip offset from latest[P1D]"), diff --git a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java index d3ce932efeb7..869eabb1662b 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java @@ -271,20 +271,21 @@ public void test_checkEligibilityForCompaction_returnsIncrementalCompaction_when null ); + final CompactionStatistics compacted = CompactionStatistics.create(1200L, 10, 1L); + final CompactionStatistics uncompacted = CompactionStatistics.create(400L, 100, 1L); final CompactionEligibility candidate = eligibilityBuilder() - .compacted(CompactionStatistics.create(1200L, 10, 1L)) - .uncompacted(CompactionStatistics.create(400L, 100, 1L)) + .compacted(compacted) + .uncompacted(uncompacted) .uncompactedSegments(List.of(SEGMENT)) .build(); - Assertions.assertEquals( CompactionEligibility.builder( CompactionEligibility.State.INCREMENTAL_COMPACTION, "Uncompacted bytes ratio[0.25] is below threshold[0.50]" ) - .compacted(CompactionStatistics.create(1200L, 10, 1L)) - .uncompacted(CompactionStatistics.create(400L, 100, 1L)) + .compacted(compacted) + .uncompacted(uncompacted) .uncompactedSegments(List.of(SEGMENT)) .build(), policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, candidate) From a39a35ee3fc7f130de6705967bcdd70ef858f6a9 Mon Sep 17 00:00:00 2001 From: cecemei Date: Fri, 6 Feb 2026 20:47:33 -0800 Subject: [PATCH 13/19] compaction mode --- .../indexing/compact/CompactionJobQueue.java | 23 +- .../compact/OverlordCompactionScheduler.java | 2 +- .../OverlordCompactionSchedulerTest.java | 4 +- .../compaction/BaseCandidateSearchPolicy.java | 8 +- .../compaction/CompactionCandidate.java | 237 +++-- .../CompactionCandidateSearchPolicy.java | 9 +- .../compaction/CompactionEligibility.java | 932 ------------------ .../server/compaction/CompactionMode.java | 107 ++ .../compaction/CompactionRunSimulator.java | 106 +- .../compaction/CompactionSimulateResult.java | 15 +- .../server/compaction/CompactionStatus.java | 838 +++++++++++++++- .../compaction/CompactionStatusTracker.java | 29 +- .../DataSourceCompactibleSegmentIterator.java | 57 +- .../compaction/FixedIntervalOrderPolicy.java | 12 +- .../MostFragmentedIntervalFirstPolicy.java | 44 +- .../DataSourceCompactionConfig.java | 4 +- .../coordinator/duty/CompactSegments.java | 39 +- .../compaction/CompactionCandidateTest.java | 32 +- .../CompactionEligibilityEvaluateTest.java | 895 ----------------- .../CompactionRunSimulatorTest.java | 20 +- ....java => CompactionStatusBuilderTest.java} | 62 +- .../compaction/CompactionStatusTest.java | 860 +++++++++++++++- .../CompactionStatusTrackerTest.java | 33 +- ...MostFragmentedIntervalFirstPolicyTest.java | 142 ++- .../NewestSegmentFirstPolicyTest.java | 34 +- 25 files changed, 2252 insertions(+), 2292 deletions(-) delete mode 100644 server/src/main/java/org/apache/druid/server/compaction/CompactionEligibility.java create mode 100644 server/src/main/java/org/apache/druid/server/compaction/CompactionMode.java delete mode 100644 server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityEvaluateTest.java rename server/src/test/java/org/apache/druid/server/compaction/{CompactionEligibilityTest.java => CompactionStatusBuilderTest.java} (71%) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobQueue.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobQueue.java index e8cb8ada8edc..53697b0ae9c1 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobQueue.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobQueue.java @@ -25,6 +25,7 @@ import org.apache.druid.client.indexing.ClientCompactionTaskQuery; import org.apache.druid.client.indexing.ClientTaskQuery; import org.apache.druid.common.guava.FutureUtils; +import org.apache.druid.error.DruidException; import org.apache.druid.indexer.TaskState; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexing.common.actions.TaskActionClientFactory; @@ -43,7 +44,6 @@ import org.apache.druid.server.compaction.CompactionCandidateSearchPolicy; import org.apache.druid.server.compaction.CompactionSlotManager; import org.apache.druid.server.compaction.CompactionSnapshotBuilder; -import org.apache.druid.server.compaction.CompactionStatus; import org.apache.druid.server.compaction.CompactionStatusTracker; import org.apache.druid.server.coordinator.AutoCompactionSnapshot; import org.apache.druid.server.coordinator.ClusterCompactionConfig; @@ -281,18 +281,17 @@ private boolean startJobIfPendingAndReady( } // Check if the job is already running, completed or skipped - final CompactionStatus compactionStatus = getCurrentStatusForJob(job); - switch (compactionStatus.getState()) { - case RUNNING: + final CompactionCandidate.TaskState candidateState = getCurrentTaskStateForJob(job); + switch (candidateState) { + case TASK_IN_PROGRESS: return false; - case COMPLETE: + case RECENTLY_COMPLETED: snapshotBuilder.moveFromPendingToCompleted(candidate); return false; - case SKIPPED: - snapshotBuilder.moveFromPendingToSkipped(candidate); - return false; - default: + case READY: break; + default: + throw DruidException.defensive("unknown compaction candidate state[%s]", candidateState); } // Check if enough compaction task slots are available @@ -377,10 +376,10 @@ private void persistPendingIndexingState(CompactionJob job) } } - public CompactionStatus getCurrentStatusForJob(CompactionJob job) + public CompactionCandidate.TaskState getCurrentTaskStateForJob(CompactionJob job) { - statusTracker.onCompactionStatusComputed(job.getCandidate(), null); - return statusTracker.computeCompactionStatus(job.getCandidate()); + statusTracker.onCompactionCandidates(job.getCandidate(), null); + return statusTracker.computeCompactionTaskState(job.getCandidate()); } public static CompactionConfigValidationResult validateCompactionJob(BatchIndexingJob job) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java index 11709e616c71..6e06c962046a 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java @@ -499,7 +499,7 @@ public CompactionSimulateResult simulateRunWithConfigUpdate(ClusterCompactionCon updateRequest.getEngine() ); } else { - return new CompactionSimulateResult(Collections.emptyMap()); + return new CompactionSimulateResult(Collections.emptyMap(), null); } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/compact/OverlordCompactionSchedulerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/compact/OverlordCompactionSchedulerTest.java index 4c98b52f48c1..78389263b5fd 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/compact/OverlordCompactionSchedulerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/compact/OverlordCompactionSchedulerTest.java @@ -66,9 +66,9 @@ import org.apache.druid.segment.TestIndex; import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; import org.apache.druid.segment.metadata.IndexingStateCache; +import org.apache.druid.server.compaction.CompactionCandidate; import org.apache.druid.server.compaction.CompactionSimulateResult; import org.apache.druid.server.compaction.CompactionStatistics; -import org.apache.druid.server.compaction.CompactionStatus; import org.apache.druid.server.compaction.CompactionStatusTracker; import org.apache.druid.server.compaction.Table; import org.apache.druid.server.coordinator.AutoCompactionSnapshot; @@ -451,7 +451,7 @@ public void test_simulateRunWithConfigUpdate() new ClusterCompactionConfig(null, null, null, null, null, null) ); Assert.assertEquals(1, simulateResult.getCompactionStates().size()); - final Table pendingCompactionTable = simulateResult.getCompactionStates().get(CompactionStatus.State.PENDING); + final Table pendingCompactionTable = simulateResult.getCompactionStates().get(CompactionCandidate.TaskState.READY); Assert.assertEquals( Arrays.asList("dataSource", "interval", "numSegments", "bytes", "maxTaskSlots", "reasonToCompact"), pendingCompactionTable.getColumnNames() diff --git a/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java index 0400e7e5ad61..951f00ba788f 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java @@ -68,12 +68,12 @@ public final int compareCandidates(CompactionCandidate o1, CompactionCandidate o } @Override - public CompactionEligibility checkEligibilityForCompaction( - CompactionCandidate.ProposedCompaction candidate, - CompactionEligibility eligibility + public CompactionCandidate createCandidate( + CompactionCandidate.ProposedCompaction proposedCompaction, + CompactionStatus eligibility ) { - return eligibility; + return CompactionMode.FULL_COMPACTION.createCandidate(proposedCompaction, eligibility); } /** diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java index d3c12b4a858d..c2845fa46f7b 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidate.java @@ -29,6 +29,7 @@ import javax.annotation.Nullable; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -38,106 +39,6 @@ */ public class CompactionCandidate { - private final ProposedCompaction proposedCompaction; - - private final CompactionEligibility policyEligiblity; - private final CompactionStatus currentStatus; - - CompactionCandidate( - ProposedCompaction proposedCompaction, - CompactionEligibility policyEligiblity, - CompactionStatus currentStatus - ) - { - this.proposedCompaction = Preconditions.checkNotNull(proposedCompaction, "proposedCompaction"); - this.policyEligiblity = Preconditions.checkNotNull(policyEligiblity, "policyEligiblity"); - this.currentStatus = Preconditions.checkNotNull(currentStatus, "currentStatus"); - } - - public ProposedCompaction getProposedCompaction() - { - return proposedCompaction; - } - - /** - * @return Non-empty list of segments that make up this candidate. - */ - public List getSegments() - { - return proposedCompaction.getSegments(); - } - - public long getTotalBytes() - { - return proposedCompaction.getTotalBytes(); - } - - public int numSegments() - { - return proposedCompaction.numSegments(); - } - - /** - * Umbrella interval of all the segments in this candidate. This typically - * corresponds to a single time chunk in the segment timeline. - */ - public Interval getUmbrellaInterval() - { - return proposedCompaction.getUmbrellaInterval(); - } - - /** - * Interval aligned to the target segment granularity used for the compaction - * task. This interval completely contains the {@link #getUmbrellaInterval()}. - */ - public Interval getCompactionInterval() - { - return proposedCompaction.getCompactionInterval(); - } - - public String getDataSource() - { - return proposedCompaction.getDataSource(); - } - - public CompactionStatistics getStats() - { - return proposedCompaction.getStats(); - } - - /** - * Current compaction status of the time chunk corresponding to this candidate. - */ - @Nullable - public CompactionStatus getCurrentStatus() - { - return currentStatus; - } - - @Nullable - public CompactionEligibility getPolicyEligibility() - { - return policyEligiblity; - } - - /** - * Creates a copy of this CompactionCandidate object with the given status. - */ - public CompactionCandidate withCurrentStatus(CompactionStatus status) - { - return new CompactionCandidate(proposedCompaction, policyEligiblity, status); - } - - @Override - public String toString() - { - return "SegmentsToCompact{" + - ", proposedCompaction=" + proposedCompaction + - ", policyEligiblity=" + policyEligiblity + - ", currentStatus=" + currentStatus + - '}'; - } - /** * Non-empty list of segments of a datasource being proposed for compaction. * A proposed compaction typically contains all the segments of a single time chunk. @@ -239,6 +140,30 @@ public CompactionStatistics getStats() return CompactionStatistics.create(totalBytes, numSegments(), numIntervals); } + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ProposedCompaction that = (ProposedCompaction) o; + return totalBytes == that.totalBytes + && numIntervals == that.numIntervals + && segments.equals(that.segments) + && umbrellaInterval.equals(that.umbrellaInterval) + && compactionInterval.equals(that.compactionInterval) + && dataSource.equals(that.dataSource); + } + + @Override + public int hashCode() + { + return Objects.hash(segments, umbrellaInterval, compactionInterval, dataSource, totalBytes, numIntervals); + } + @Override public String toString() { @@ -252,4 +177,116 @@ public String toString() '}'; } } + + /** + * Used by {@link CompactionStatusTracker#computeCompactionTaskState(CompactionCandidate)}. + * The callsite then determines whether to launch compaction task or not. + */ + public enum TaskState + { + // no other compaction candidate is running, we can start a new task + READY, + // compaction candidate is already running under a task + TASK_IN_PROGRESS, + // compaction candidate has recently been completed, and the segment timeline has not yet updated after that + RECENTLY_COMPLETED + } + + private final ProposedCompaction proposedCompaction; + + private final CompactionStatus eligibility; + @Nullable + private final String policyNote; + private final CompactionMode mode; + + CompactionCandidate( + ProposedCompaction proposedCompaction, + CompactionStatus eligibility, + @Nullable String policyNote, + CompactionMode mode + ) + { + this.proposedCompaction = Preconditions.checkNotNull(proposedCompaction, "proposedCompaction"); + this.eligibility = Preconditions.checkNotNull(eligibility, "eligibility"); + this.policyNote = policyNote; + this.mode = Preconditions.checkNotNull(mode, "mode"); + } + + public ProposedCompaction getProposedCompaction() + { + return proposedCompaction; + } + + /** + * @return Non-empty list of segments that make up this candidate. + */ + public List getSegments() + { + return proposedCompaction.getSegments(); + } + + public long getTotalBytes() + { + return proposedCompaction.getTotalBytes(); + } + + public int numSegments() + { + return proposedCompaction.numSegments(); + } + + /** + * Umbrella interval of all the segments in this candidate. This typically + * corresponds to a single time chunk in the segment timeline. + */ + public Interval getUmbrellaInterval() + { + return proposedCompaction.getUmbrellaInterval(); + } + + /** + * Interval aligned to the target segment granularity used for the compaction + * task. This interval completely contains the {@link #getUmbrellaInterval()}. + */ + public Interval getCompactionInterval() + { + return proposedCompaction.getCompactionInterval(); + } + + public String getDataSource() + { + return proposedCompaction.getDataSource(); + } + + public CompactionStatistics getStats() + { + return proposedCompaction.getStats(); + } + + @Nullable + public String getPolicyNote() + { + return policyNote; + } + + public CompactionMode getMode() + { + return mode; + } + + public CompactionStatus getEligibility() + { + return eligibility; + } + + @Override + public String toString() + { + return "SegmentsToCompact{" + + ", proposedCompaction=" + proposedCompaction + + ", eligibility=" + eligibility + + ", policyNote=" + policyNote + + ", mode=" + mode + + '}'; + } } diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java index 66fe33ff39d4..3afe296297b1 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionCandidateSearchPolicy.java @@ -45,14 +45,11 @@ public interface CompactionCandidateSearchPolicy int compareCandidates(CompactionCandidate candidateA, CompactionCandidate candidateB); /** - * Applies policy-specific eligibility checks to the proposed compaction. + * Creates a {@link CompactionCandidate} after applying policy-specific checks to the proposed compaction candidate. * * @param candidate the proposed compaction * @param eligibility initial eligibility from compaction config checks - * @return final eligibility after applying policy checks + * @return final compaction candidate */ - CompactionEligibility checkEligibilityForCompaction( - CompactionCandidate.ProposedCompaction candidate, - CompactionEligibility eligibility - ); + CompactionCandidate createCandidate(CompactionCandidate.ProposedCompaction candidate, CompactionStatus eligibility); } diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionEligibility.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionEligibility.java deleted file mode 100644 index 30786e0ee765..000000000000 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionEligibility.java +++ /dev/null @@ -1,932 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.server.compaction; - -import com.google.common.base.Strings; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig; -import org.apache.druid.common.config.Configs; -import org.apache.druid.data.input.impl.DimensionSchema; -import org.apache.druid.error.DruidException; -import org.apache.druid.error.InvalidInput; -import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec; -import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; -import org.apache.druid.indexer.partitions.HashedPartitionsSpec; -import org.apache.druid.indexer.partitions.PartitionsSpec; -import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.java.util.common.granularity.GranularityType; -import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.query.aggregation.AggregatorFactory; -import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.metadata.IndexingStateFingerprintMapper; -import org.apache.druid.segment.transform.CompactionTransformSpec; -import org.apache.druid.server.coordinator.DataSourceCompactionConfig; -import org.apache.druid.server.coordinator.UserCompactionTaskGranularityConfig; -import org.apache.druid.timeline.CompactionState; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.utils.CollectionUtils; -import org.joda.time.Interval; - -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * Describes the eligibility of an interval for compaction. - */ -public class CompactionEligibility -{ - public enum State - { - FULL_COMPACTION, - INCREMENTAL_COMPACTION, - NOT_ELIGIBLE, - NOT_APPLICABLE - } - - static class CompactionEligibilityBuilder - { - private State state; - private CompactionStatistics compacted; - private CompactionStatistics uncompacted; - private List uncompactedSegments; - private String reason; - - CompactionEligibilityBuilder(State state, String reason) - { - this.state = state; - this.reason = reason; - } - - CompactionEligibilityBuilder compacted(CompactionStatistics compacted) - { - this.compacted = compacted; - return this; - } - - CompactionEligibilityBuilder uncompacted(CompactionStatistics uncompacted) - { - this.uncompacted = uncompacted; - return this; - } - - CompactionEligibilityBuilder uncompactedSegments(List uncompactedSegments) - { - this.uncompactedSegments = uncompactedSegments; - return this; - } - - CompactionEligibility build() - { - return new CompactionEligibility(state, reason, compacted, uncompacted, uncompactedSegments); - } - } - - public static final CompactionEligibility NOT_APPLICABLE = builder(State.NOT_APPLICABLE, "").build(); - - public static CompactionEligibility fail(String messageFormat, Object... args) - { - return builder(State.NOT_ELIGIBLE, StringUtils.format(messageFormat, args)).build(); - } - - private final State state; - private final String reason; - - @Nullable - private final CompactionStatistics compacted; - @Nullable - private final CompactionStatistics uncompacted; - @Nullable - private final List uncompactedSegments; - - private CompactionEligibility( - State state, - String reason, - @Nullable CompactionStatistics compacted, - @Nullable CompactionStatistics uncompacted, - @Nullable List uncompactedSegments - ) - { - this.state = state; - this.reason = reason; - switch (state) { - case NOT_APPLICABLE: - break; - case NOT_ELIGIBLE: - InvalidInput.conditionalException(!Strings.isNullOrEmpty(reason), "must provide a reason"); - break; - case FULL_COMPACTION: - case INCREMENTAL_COMPACTION: - InvalidInput.conditionalException(compacted != null, "must provide compacted stats"); - InvalidInput.conditionalException(uncompacted != null, "must provide uncompacted stats"); - InvalidInput.conditionalException(uncompactedSegments != null, "must provide uncompactedSegments"); - break; - default: - throw DruidException.defensive("unexpected eligibility state[%s]", state); - } - this.compacted = compacted; - this.uncompacted = uncompacted; - this.uncompactedSegments = uncompactedSegments; - } - - static CompactionEligibilityBuilder builder(State state, String reason) - { - return new CompactionEligibilityBuilder(state, reason); - } - - public State getState() - { - return state; - } - - public String getReason() - { - return reason; - } - - @Nullable - public CompactionStatistics getUncompactedStats() - { - return uncompacted; - } - - @Nullable - public CompactionStatistics getCompactedStats() - { - return compacted; - } - - @Nullable - public List getUncompactedSegments() - { - return uncompactedSegments; - } - - public CompactionCandidate createCandidate(CompactionCandidate.ProposedCompaction proposedCompaction) - { - switch (state) { - case NOT_APPLICABLE: - return new CompactionCandidate(proposedCompaction, this, CompactionStatus.COMPLETE); - case NOT_ELIGIBLE: - return new CompactionCandidate(proposedCompaction, this, CompactionStatus.skipped(reason)); - case FULL_COMPACTION: - return new CompactionCandidate( - proposedCompaction, - this, - CompactionStatus.pending(reason) - ); - case INCREMENTAL_COMPACTION: - CompactionCandidate.ProposedCompaction newProposed = new CompactionCandidate.ProposedCompaction( - uncompactedSegments, - proposedCompaction.getUmbrellaInterval(), - proposedCompaction.getCompactionInterval(), - Math.toIntExact(uncompactedSegments.stream().map(DataSegment::getInterval).distinct().count()) - ); - return new CompactionCandidate(newProposed, this, CompactionStatus.pending(reason)); - default: - throw DruidException.defensive("Unexpected eligibility state[%s]", state); - } - } - - /** - * Evaluates a compaction candidate to determine its eligibility and compaction status. - *

- * This method performs a two-stage evaluation: - *

    - *
  1. First, uses {@link Evaluator} to check if the candidate needs compaction - * based on the compaction config (e.g., checking segment granularity, partitions spec, etc.)
    2. - *
  2. Then, applies the search policy to determine if this candidate should be compacted in the - * current run (e.g., checking minimum segment count, bytes, or other policy criteria)
    3. - *
- * - * @param proposedCompaction the compaction candidate to evaluate - * @param config the compaction configuration for the datasource - * @param searchPolicy the policy that determines candidate ordering and eligibility - * @param fingerprintMapper mapper for indexing state fingerprints - * @return a new {@link CompactionCandidate} with updated eligibility and status. For incremental - * compaction, returns a candidate containing only the uncompacted segments. - */ - public static CompactionEligibility evaluate( - CompactionCandidate.ProposedCompaction proposedCompaction, - DataSourceCompactionConfig config, - CompactionCandidateSearchPolicy searchPolicy, - IndexingStateFingerprintMapper fingerprintMapper - ) - { - // ideally we should let this class only decides CompactionEligibility, and the callsite should handle recreation of candidate. - CompactionEligibility evaluatedCandidate = new Evaluator(proposedCompaction, config, fingerprintMapper).evaluate(); - switch (Objects.requireNonNull(evaluatedCandidate).getState()) { - case NOT_APPLICABLE: - case NOT_ELIGIBLE: - return evaluatedCandidate; - case FULL_COMPACTION: // evaluator has decided compaction is needed, policy needs to further check - return searchPolicy.checkEligibilityForCompaction(proposedCompaction, evaluatedCandidate); - case INCREMENTAL_COMPACTION: // evaluator cant decide when to perform an incremental compaction - default: - throw DruidException.defensive("Unexpected eligibility[%s]", evaluatedCandidate); - } - } - - @Override - public boolean equals(Object object) - { - if (this == object) { - return true; - } - if (object == null || getClass() != object.getClass()) { - return false; - } - CompactionEligibility that = (CompactionEligibility) object; - return state == that.state - && Objects.equals(reason, that.reason) - && Objects.equals(compacted, that.compacted) - && Objects.equals(uncompacted, that.uncompacted) - && Objects.equals(uncompactedSegments, that.uncompactedSegments); - } - - @Override - public int hashCode() - { - return Objects.hash(state, reason, compacted, uncompacted, uncompactedSegments); - } - - @Override - public String toString() - { - return "CompactionEligibility{" - + "state=" + state - + ", reason='" + reason + '\'' - + ", compacted=" + compacted - + ", uncompacted=" + uncompacted - + ", uncompactedSegments=" + uncompactedSegments - + '}'; - } - - /** - * List of checks performed to determine if compaction is already complete based on indexing state fingerprints. - */ - static final List> FINGERPRINT_CHECKS = List.of( - Evaluator::allFingerprintedCandidatesHaveExpectedFingerprint - ); - - /** - * List of checks performed to determine if compaction is already complete. - *

- * The order of the checks must be honored while evaluating them. - */ - static final List> CHECKS = Arrays.asList( - Evaluator::partitionsSpecIsUpToDate, - Evaluator::indexSpecIsUpToDate, - Evaluator::segmentGranularityIsUpToDate, - Evaluator::queryGranularityIsUpToDate, - Evaluator::rollupIsUpToDate, - Evaluator::dimensionsSpecIsUpToDate, - Evaluator::metricsSpecIsUpToDate, - Evaluator::transformSpecFilterIsUpToDate, - Evaluator::projectionsAreUpToDate - ); - - /** - * Evaluates checks to determine the compaction status of a - * {@link CompactionCandidate}. - */ - private static class Evaluator - { - private static final Logger log = new Logger(Evaluator.class); - - private final DataSourceCompactionConfig compactionConfig; - private final CompactionCandidate.ProposedCompaction proposedCompaction; - private final ClientCompactionTaskQueryTuningConfig tuningConfig; - private final UserCompactionTaskGranularityConfig configuredGranularitySpec; - - private final List fingerprintedSegments = new ArrayList<>(); - private final List compactedSegments = new ArrayList<>(); - private final List uncompactedSegments = new ArrayList<>(); - private final Map> unknownStateToSegments = new HashMap<>(); - - @Nullable - private final IndexingStateFingerprintMapper fingerprintMapper; - @Nullable - private final String targetFingerprint; - - private Evaluator( - CompactionCandidate.ProposedCompaction proposedCompaction, - DataSourceCompactionConfig compactionConfig, - @Nullable IndexingStateFingerprintMapper fingerprintMapper - ) - { - this.proposedCompaction = proposedCompaction; - this.compactionConfig = compactionConfig; - this.tuningConfig = ClientCompactionTaskQueryTuningConfig.from(compactionConfig); - this.configuredGranularitySpec = compactionConfig.getGranularitySpec(); - this.fingerprintMapper = fingerprintMapper; - if (fingerprintMapper == null) { - targetFingerprint = null; - } else { - targetFingerprint = fingerprintMapper.generateFingerprint( - compactionConfig.getDataSource(), - compactionConfig.toCompactionState() - ); - } - } - - /** - * Evaluates the compaction status of candidate segments through a multi-step process: - *

    - *
  1. Validates input bytes are within limits
    2. - *
  2. Categorizes segments by compaction state (fingerprinted, uncompacted, or unknown)
    3. - *
  3. Performs fingerprint-based validation if available (fast path)
    4. - *
  4. Runs detailed checks against unknown states via {@link CompactionEligibility#CHECKS}
    5. - *
- * - * @return Pair of eligibility status and compaction status with reason for first failed check - */ - private CompactionEligibility evaluate() - { - final String inputBytesCheck = inputBytesAreWithinLimit(); - if (inputBytesCheck != null) { - return CompactionEligibility.fail(inputBytesCheck); - } - - List reasonsForCompaction = new ArrayList<>(); - String compactedOnceCheck = segmentsHaveBeenCompactedAtLeastOnce(); - if (compactedOnceCheck != null) { - reasonsForCompaction.add(compactedOnceCheck); - } - - if (fingerprintMapper != null && targetFingerprint != null) { - // First try fingerprint-based evaluation (fast path) - FINGERPRINT_CHECKS.stream() - .map(f -> f.apply(this)) - .filter(Objects::nonNull) - .findFirst() - .ifPresent(reasonsForCompaction::add); - - } - - if (!unknownStateToSegments.isEmpty()) { - // Run CHECKS against any states with uknown compaction status - reasonsForCompaction.addAll( - CHECKS.stream() - .map(f -> f.apply(this)) - .filter(Objects::nonNull) - .collect(Collectors.toList()) - ); - - // Any segments left in unknownStateToSegments passed all checks and are considered compacted - compactedSegments.addAll( - unknownStateToSegments - .values() - .stream() - .flatMap(List::stream) - .collect(Collectors.toList()) - ); - } - - if (reasonsForCompaction.isEmpty()) { - return CompactionEligibility.NOT_APPLICABLE; - } else { - return builder(State.FULL_COMPACTION, reasonsForCompaction.get(0)).compacted(createStats(compactedSegments)) - .uncompacted(createStats(uncompactedSegments)) - .uncompactedSegments(uncompactedSegments) - .build(); - } - } - - /** - * Evaluates the fingerprints of all fingerprinted candidate segments against the expected fingerprint. - *

- * If all fingerprinted segments have the expected fingerprint, the check can quickly pass as COMPLETE. However, - * if any fingerprinted segment has a mismatched fingerprint, we need to investigate further by adding them to - * {@link #unknownStateToSegments} where their indexing states will be analyzed. - *

- */ - private String allFingerprintedCandidatesHaveExpectedFingerprint() - { - Map> mismatchedFingerprintToSegmentMap = new HashMap<>(); - for (DataSegment segment : fingerprintedSegments) { - String fingerprint = segment.getIndexingStateFingerprint(); - if (fingerprint == null) { - // Should not happen since we are iterating over fingerprintedSegments - } else if (fingerprint.equals(targetFingerprint)) { - compactedSegments.add(segment); - } else { - mismatchedFingerprintToSegmentMap - .computeIfAbsent(fingerprint, k -> new ArrayList<>()) - .add(segment); - } - } - - if (mismatchedFingerprintToSegmentMap.isEmpty()) { - // All fingerprinted segments have the expected fingerprint - compaction is complete - return null; - } - - if (fingerprintMapper == null) { - // Cannot evaluate further without a fingerprint mapper - uncompactedSegments.addAll( - mismatchedFingerprintToSegmentMap.values() - .stream() - .flatMap(List::stream) - .collect(Collectors.toList()) - ); - return "Segments have a mismatched fingerprint and no fingerprint mapper is available"; - } - - boolean fingerprintedSegmentWithoutCachedStateFound = false; - - for (Map.Entry> e : mismatchedFingerprintToSegmentMap.entrySet()) { - String fingerprint = e.getKey(); - CompactionState stateToValidate = fingerprintMapper.getStateForFingerprint(fingerprint).orElse(null); - if (stateToValidate == null) { - log.warn("No indexing state found for fingerprint[%s]", fingerprint); - fingerprintedSegmentWithoutCachedStateFound = true; - uncompactedSegments.addAll(e.getValue()); - } else { - // Note that this does not mean we need compaction yet - we need to validate the state further to determine this - unknownStateToSegments.compute( - stateToValidate, - (state, segments) -> { - if (segments == null) { - segments = new ArrayList<>(); - } - segments.addAll(e.getValue()); - return segments; - } - ); - } - } - - if (fingerprintedSegmentWithoutCachedStateFound) { - return "One or more fingerprinted segments do not have a cached indexing state"; - } else { - return null; - } - } - - /** - * Checks if all the segments have been compacted at least once and groups them into uncompacted, fingerprinted, or - * non-fingerprinted. - */ - private String segmentsHaveBeenCompactedAtLeastOnce() - { - for (DataSegment segment : proposedCompaction.getSegments()) { - final String fingerprint = segment.getIndexingStateFingerprint(); - final CompactionState segmentState = segment.getLastCompactionState(); - if (fingerprint != null) { - fingerprintedSegments.add(segment); - } else if (segmentState == null) { - uncompactedSegments.add(segment); - } else { - unknownStateToSegments.computeIfAbsent(segmentState, s -> new ArrayList<>()).add(segment); - } - } - - if (uncompactedSegments.isEmpty()) { - return null; - } else { - return "not compacted yet"; - } - } - - private String partitionsSpecIsUpToDate() - { - return evaluateForAllCompactionStates(this::partitionsSpecIsUpToDate); - } - - private String indexSpecIsUpToDate() - { - return evaluateForAllCompactionStates(this::indexSpecIsUpToDate); - } - - private String projectionsAreUpToDate() - { - return evaluateForAllCompactionStates(this::projectionsAreUpToDate); - } - - private String segmentGranularityIsUpToDate() - { - return evaluateForAllCompactionStates(this::segmentGranularityIsUpToDate); - } - - private String rollupIsUpToDate() - { - return evaluateForAllCompactionStates(this::rollupIsUpToDate); - } - - private String queryGranularityIsUpToDate() - { - return evaluateForAllCompactionStates(this::queryGranularityIsUpToDate); - } - - private String dimensionsSpecIsUpToDate() - { - return evaluateForAllCompactionStates(this::dimensionsSpecIsUpToDate); - } - - private String metricsSpecIsUpToDate() - { - return evaluateForAllCompactionStates(this::metricsSpecIsUpToDate); - } - - private String transformSpecFilterIsUpToDate() - { - return evaluateForAllCompactionStates(this::transformSpecFilterIsUpToDate); - } - - private String partitionsSpecIsUpToDate(CompactionState lastCompactionState) - { - PartitionsSpec existingPartionsSpec = lastCompactionState.getPartitionsSpec(); - if (existingPartionsSpec instanceof DimensionRangePartitionsSpec) { - existingPartionsSpec = getEffectiveRangePartitionsSpec((DimensionRangePartitionsSpec) existingPartionsSpec); - } else if (existingPartionsSpec instanceof DynamicPartitionsSpec) { - existingPartionsSpec = new DynamicPartitionsSpec( - existingPartionsSpec.getMaxRowsPerSegment(), - ((DynamicPartitionsSpec) existingPartionsSpec).getMaxTotalRowsOr(Long.MAX_VALUE) - ); - } - return completeIfNullOrEqual( - "partitionsSpec", - findPartitionsSpecFromConfig(tuningConfig), - existingPartionsSpec, - CompactionEligibility::asString - ); - } - - private String indexSpecIsUpToDate(CompactionState lastCompactionState) - { - return completeIfNullOrEqual( - "indexSpec", - Configs.valueOrDefault(tuningConfig.getIndexSpec(), IndexSpec.getDefault()).getEffectiveSpec(), - lastCompactionState.getIndexSpec().getEffectiveSpec(), - String::valueOf - ); - } - - private String projectionsAreUpToDate(CompactionState lastCompactionState) - { - return completeIfNullOrEqual( - "projections", - compactionConfig.getProjections(), - lastCompactionState.getProjections(), - String::valueOf - ); - } - - @Nullable - private String inputBytesAreWithinLimit() - { - final long inputSegmentSize = compactionConfig.getInputSegmentSizeBytes(); - if (proposedCompaction.getTotalBytes() > inputSegmentSize) { - return StringUtils.format( - "'inputSegmentSize' exceeded: Total segment size[%d] is larger than allowed inputSegmentSize[%d]", - proposedCompaction.getTotalBytes(), inputSegmentSize - ); - } - return null; - } - - private String segmentGranularityIsUpToDate(CompactionState lastCompactionState) - { - if (configuredGranularitySpec == null - || configuredGranularitySpec.getSegmentGranularity() == null) { - return null; - } - - final Granularity configuredSegmentGranularity = configuredGranularitySpec.getSegmentGranularity(); - final UserCompactionTaskGranularityConfig existingGranularitySpec = getGranularitySpec(lastCompactionState); - final Granularity existingSegmentGranularity - = existingGranularitySpec == null ? null : existingGranularitySpec.getSegmentGranularity(); - - if (configuredSegmentGranularity.equals(existingSegmentGranularity)) { - return null; - } else if (existingSegmentGranularity == null) { - // Candidate segments were compacted without segment granularity specified - // Check if the segments already have the desired segment granularity - final List segmentsForState = unknownStateToSegments.get(lastCompactionState); - boolean needsCompaction = segmentsForState.stream().anyMatch( - segment -> !configuredSegmentGranularity.isAligned(segment.getInterval()) - ); - if (needsCompaction) { - return StringUtils.format( - "segmentGranularity: segments do not align with target[%s]", - CompactionEligibility.asString(configuredSegmentGranularity) - ); - } - } else { - return configChanged( - "segmentGranularity", - configuredSegmentGranularity, - existingSegmentGranularity, - CompactionEligibility::asString - ); - } - - return null; - } - - private String rollupIsUpToDate(CompactionState lastCompactionState) - { - if (configuredGranularitySpec == null) { - return null; - } else { - final UserCompactionTaskGranularityConfig existingGranularitySpec - = getGranularitySpec(lastCompactionState); - return completeIfNullOrEqual( - "rollup", - configuredGranularitySpec.isRollup(), - existingGranularitySpec == null ? null : existingGranularitySpec.isRollup(), - String::valueOf - ); - } - } - - private String queryGranularityIsUpToDate(CompactionState lastCompactionState) - { - if (configuredGranularitySpec == null) { - return null; - } else { - final UserCompactionTaskGranularityConfig existingGranularitySpec - = getGranularitySpec(lastCompactionState); - return completeIfNullOrEqual( - "queryGranularity", - configuredGranularitySpec.getQueryGranularity(), - existingGranularitySpec == null ? null : existingGranularitySpec.getQueryGranularity(), - CompactionEligibility::asString - ); - } - } - - /** - * Removes partition dimensions before comparison, since they are placed in front of the sort order -- - * which can create a mismatch between expected and actual order of dimensions. Partition dimensions are separately - * covered in {@link Evaluator#partitionsSpecIsUpToDate()} check. - */ - private String dimensionsSpecIsUpToDate(CompactionState lastCompactionState) - { - if (compactionConfig.getDimensionsSpec() == null) { - return null; - } else { - List existingDimensions = getNonPartitioningDimensions( - lastCompactionState.getDimensionsSpec() == null - ? null - : lastCompactionState.getDimensionsSpec().getDimensions(), - lastCompactionState.getPartitionsSpec(), - lastCompactionState.getIndexSpec() - ); - List configuredDimensions = getNonPartitioningDimensions( - compactionConfig.getDimensionsSpec().getDimensions(), - compactionConfig.getTuningConfig() == null ? null : compactionConfig.getTuningConfig().getPartitionsSpec(), - compactionConfig.getTuningConfig() == null - ? IndexSpec.getDefault() - : compactionConfig.getTuningConfig().getIndexSpec() - ); - return completeIfNullOrEqual( - "dimensionsSpec", - configuredDimensions, - existingDimensions, - String::valueOf - ); - } - } - - private String metricsSpecIsUpToDate(CompactionState lastCompactionState) - { - final AggregatorFactory[] configuredMetricsSpec = compactionConfig.getMetricsSpec(); - if (ArrayUtils.isEmpty(configuredMetricsSpec)) { - return null; - } - - final List metricSpecList = lastCompactionState.getMetricsSpec(); - final AggregatorFactory[] existingMetricsSpec - = CollectionUtils.isNullOrEmpty(metricSpecList) - ? null : metricSpecList.toArray(new AggregatorFactory[0]); - - if (existingMetricsSpec == null || !Arrays.deepEquals(configuredMetricsSpec, existingMetricsSpec)) { - return configChanged( - "metricsSpec", - configuredMetricsSpec, - existingMetricsSpec, - Arrays::toString - ); - } else { - return null; - } - } - - private String transformSpecFilterIsUpToDate(CompactionState lastCompactionState) - { - if (compactionConfig.getTransformSpec() == null) { - return null; - } - - CompactionTransformSpec existingTransformSpec = lastCompactionState.getTransformSpec(); - return completeIfNullOrEqual( - "transformSpec filter", - compactionConfig.getTransformSpec().getFilter(), - existingTransformSpec == null ? null : existingTransformSpec.getFilter(), - String::valueOf - ); - } - - /** - * Evaluates the given check for each entry in the {@link #unknownStateToSegments}. - * If any entry fails the given check by returning a status which is not - * COMPLETE, all the segments with that state are moved to {@link #uncompactedSegments}. - * - * @return The first status which is not COMPLETE. - */ - private String evaluateForAllCompactionStates(Function check) - { - String firstIncomplete = null; - for (CompactionState state : List.copyOf(unknownStateToSegments.keySet())) { - final String eligibleReason = check.apply(state); - if (eligibleReason != null) { - uncompactedSegments.addAll(unknownStateToSegments.remove(state)); - if (firstIncomplete == null) { - firstIncomplete = eligibleReason; - } - } - } - - return firstIncomplete; - } - - private static UserCompactionTaskGranularityConfig getGranularitySpec( - CompactionState compactionState - ) - { - return UserCompactionTaskGranularityConfig.from(compactionState.getGranularitySpec()); - } - - private static CompactionStatistics createStats(List segments) - { - final Set segmentIntervals = - segments.stream().map(DataSegment::getInterval).collect(Collectors.toSet()); - final long totalBytes = segments.stream().mapToLong(DataSegment::getSize).sum(); - return CompactionStatistics.create(totalBytes, segments.size(), segmentIntervals.size()); - } - } - - - /** - * Computes compaction status for the given field. The status is assumed to be - * COMPLETE (i.e. no further compaction is required) if the configured value - * of the field is null or equal to the current value. - */ - private static String completeIfNullOrEqual( - String field, - T configured, - T current, - Function stringFunction - ) - { - if (configured == null || configured.equals(current)) { - return null; - } else { - return configChanged(field, configured, current, stringFunction); - } - } - - private static String configChanged( - String field, - T target, - T current, - Function stringFunction - ) - { - return StringUtils.format( - "'%s' mismatch: required[%s], current[%s]", - field, - target == null ? null : stringFunction.apply(target), - current == null ? null : stringFunction.apply(current) - ); - } - - private static String asString(Granularity granularity) - { - if (granularity == null) { - return null; - } - for (GranularityType type : GranularityType.values()) { - if (type.getDefaultGranularity().equals(granularity)) { - return type.toString(); - } - } - return granularity.toString(); - } - - private static String asString(PartitionsSpec partitionsSpec) - { - if (partitionsSpec instanceof DimensionRangePartitionsSpec) { - DimensionRangePartitionsSpec rangeSpec = (DimensionRangePartitionsSpec) partitionsSpec; - return StringUtils.format( - "'range' on %s with %,d rows", - rangeSpec.getPartitionDimensions(), rangeSpec.getTargetRowsPerSegment() - ); - } else if (partitionsSpec instanceof HashedPartitionsSpec) { - HashedPartitionsSpec hashedSpec = (HashedPartitionsSpec) partitionsSpec; - return StringUtils.format( - "'hashed' on %s with %,d rows", - hashedSpec.getPartitionDimensions(), hashedSpec.getTargetRowsPerSegment() - ); - } else if (partitionsSpec instanceof DynamicPartitionsSpec) { - DynamicPartitionsSpec dynamicSpec = (DynamicPartitionsSpec) partitionsSpec; - return StringUtils.format( - "'dynamic' with %,d rows", - dynamicSpec.getMaxRowsPerSegment() - ); - } else { - return partitionsSpec.toString(); - } - } - - @Nullable - public static PartitionsSpec findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig tuningConfig) - { - final PartitionsSpec partitionsSpecFromTuningConfig = tuningConfig.getPartitionsSpec(); - if (partitionsSpecFromTuningConfig == null) { - final Long maxTotalRows = tuningConfig.getMaxTotalRows(); - final Integer maxRowsPerSegment = tuningConfig.getMaxRowsPerSegment(); - - if (maxTotalRows == null && maxRowsPerSegment == null) { - // If not specified, return null so that partitionsSpec is not compared - return null; - } else { - return new DynamicPartitionsSpec(maxRowsPerSegment, maxTotalRows); - } - } else if (partitionsSpecFromTuningConfig instanceof DynamicPartitionsSpec) { - return new DynamicPartitionsSpec( - partitionsSpecFromTuningConfig.getMaxRowsPerSegment(), - ((DynamicPartitionsSpec) partitionsSpecFromTuningConfig).getMaxTotalRowsOr(Long.MAX_VALUE) - ); - } else if (partitionsSpecFromTuningConfig instanceof DimensionRangePartitionsSpec) { - return getEffectiveRangePartitionsSpec((DimensionRangePartitionsSpec) partitionsSpecFromTuningConfig); - } else { - return partitionsSpecFromTuningConfig; - } - } - - @Nullable - private static List getNonPartitioningDimensions( - @Nullable final List dimensionSchemas, - @Nullable final PartitionsSpec partitionsSpec, - @Nullable final IndexSpec indexSpec - ) - { - final IndexSpec effectiveIndexSpec = (indexSpec == null ? IndexSpec.getDefault() : indexSpec).getEffectiveSpec(); - if (dimensionSchemas == null || !(partitionsSpec instanceof DimensionRangePartitionsSpec)) { - if (dimensionSchemas != null) { - return dimensionSchemas.stream() - .map(dim -> dim.getEffectiveSchema(effectiveIndexSpec)) - .collect(Collectors.toList()); - } - return null; - } - - final List partitionsDimensions = ((DimensionRangePartitionsSpec) partitionsSpec).getPartitionDimensions(); - return dimensionSchemas.stream() - .filter(dim -> !partitionsDimensions.contains(dim.getName())) - .map(dim -> dim.getEffectiveSchema(effectiveIndexSpec)) - .collect(Collectors.toList()); - } - - /** - * Converts to have only the effective maxRowsPerSegment to avoid false positives when targetRowsPerSegment is set but - * effectively translates to the same maxRowsPerSegment. - */ - static DimensionRangePartitionsSpec getEffectiveRangePartitionsSpec(DimensionRangePartitionsSpec partitionsSpec) - { - return new DimensionRangePartitionsSpec( - null, - partitionsSpec.getMaxRowsPerSegment(), - partitionsSpec.getPartitionDimensions(), - partitionsSpec.isAssumeGrouped() - ); - } -} diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionMode.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionMode.java new file mode 100644 index 000000000000..966b72d385a7 --- /dev/null +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionMode.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.server.compaction; + +import org.apache.druid.error.DruidException; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.timeline.DataSegment; + +import javax.annotation.Nullable; +import java.util.Objects; + +public enum CompactionMode +{ + FULL_COMPACTION { + @Override + public CompactionCandidate createCandidate( + CompactionCandidate.ProposedCompaction proposedCompaction, + CompactionStatus eligibility, + @Nullable String policyNote + ) + { + return new CompactionCandidate(proposedCompaction, eligibility, policyNote, this); + } + }, + INCREMENTAL_COMPACTION { + @Override + public CompactionCandidate createCandidate( + CompactionCandidate.ProposedCompaction proposedCompaction, + CompactionStatus eligibility, + @Nullable String policyNote + ) + { + CompactionCandidate.ProposedCompaction newProposed = new CompactionCandidate.ProposedCompaction( + Objects.requireNonNull(eligibility.getUncompactedSegments()), + proposedCompaction.getUmbrellaInterval(), + proposedCompaction.getCompactionInterval(), + Math.toIntExact(eligibility.getUncompactedSegments() + .stream() + .map(DataSegment::getInterval) + .distinct() + .count()) + ); + return new CompactionCandidate(newProposed, eligibility, policyNote, this); + } + }, + NOT_APPLICABLE; + + public CompactionCandidate createCandidate( + CompactionCandidate.ProposedCompaction proposedCompaction, + CompactionStatus eligibility + ) + { + return createCandidate(proposedCompaction, eligibility, null); + } + + public CompactionCandidate createCandidate( + CompactionCandidate.ProposedCompaction proposedCompaction, + CompactionStatus eligibility, + @Nullable String policyNote + ) + { + throw DruidException.defensive("Cannot create compaction candidate with mode[%s]", this); + } + + public static CompactionCandidate failWithPolicyCheck( + CompactionCandidate.ProposedCompaction proposedCompaction, + CompactionStatus eligibility, + String reasonFormat, + Object... args + ) + { + return new CompactionCandidate( + proposedCompaction, + eligibility, + StringUtils.format(reasonFormat, args), + CompactionMode.NOT_APPLICABLE + ); + } + + public static CompactionCandidate notEligible(CompactionCandidate.ProposedCompaction proposedCompaction, String reason) + { + // CompactionStatus returns an ineligible reason, have not even got to policy check yet + return new CompactionCandidate( + proposedCompaction, + CompactionStatus.notEligible(reason), + null, + CompactionMode.NOT_APPLICABLE + ); + } +} diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java index 919c4815bfab..1ffa48036ff9 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java @@ -25,6 +25,8 @@ import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig; import org.apache.druid.client.indexing.IndexingTotalWorkerCapacityInfo; import org.apache.druid.client.indexing.TaskPayloadResponse; +import org.apache.druid.common.guava.GuavaUtils; +import org.apache.druid.error.DruidException; import org.apache.druid.indexer.CompactionEngine; import org.apache.druid.indexer.TaskStatus; import org.apache.druid.indexer.TaskStatusPlus; @@ -87,33 +89,81 @@ public CompactionSimulateResult simulateRunWithConfig( // account for the active tasks final CompactionStatusTracker simulationStatusTracker = new CompactionStatusTracker() { + @Override - public CompactionStatus computeCompactionStatus(CompactionCandidate candidate) + public void onSkippedCandidate( + CompactionCandidate candidateSegments, + DataSourceCompactionConfig config + ) { - return statusTracker.computeCompactionStatus(candidate); + skippedIntervals.addRow(createRow( + candidateSegments, + null, + GuavaUtils.firstNonNull( + candidateSegments.getPolicyNote(), + candidateSegments.getEligibility().getReason() + ) + )); } @Override - public void onCompactionStatusComputed( + public void onCompactionCandidates( CompactionCandidate candidateSegments, DataSourceCompactionConfig config ) { - final CompactionStatus status = candidateSegments.getCurrentStatus(); - if (status == null) { - // do nothing - } else if (status.getState() == CompactionStatus.State.COMPLETE) { - compactedIntervals.addRow( - createRow(candidateSegments, null, null) - ); - } else if (status.getState() == CompactionStatus.State.RUNNING) { - runningIntervals.addRow( - createRow(candidateSegments, ClientCompactionTaskQueryTuningConfig.from(config), status.getReason()) - ); - } else if (status.getState() == CompactionStatus.State.SKIPPED) { - skippedIntervals.addRow( - createRow(candidateSegments, null, status.getReason()) - ); + switch (candidateSegments.getMode()) { + case NOT_APPLICABLE: + skippedIntervals.addRow(createRow( + candidateSegments, + null, + GuavaUtils.firstNonNull( + candidateSegments.getPolicyNote(), + candidateSegments.getEligibility().getReason() + ) + )); + break; + case INCREMENTAL_COMPACTION: + case FULL_COMPACTION: + queuedIntervals.addRow(createRow( + candidateSegments, + ClientCompactionTaskQueryTuningConfig.from(config), + GuavaUtils.firstNonNull( + candidateSegments.getPolicyNote(), + candidateSegments.getEligibility().getReason() + ) + )); + break; + default: + throw DruidException.defensive("unexpected compaction mode[%s]", candidateSegments.getMode()); + } + } + + @Override + public void onCompactionTaskStateComputed( + CompactionCandidate candidateSegments, + CompactionCandidate.TaskState taskState, + DataSourceCompactionConfig config + ) + { + switch (taskState) { + case RECENTLY_COMPLETED: + compactedIntervals.addRow(createRow(candidateSegments, null, null)); + break; + case TASK_IN_PROGRESS: + runningIntervals.addRow(createRow( + candidateSegments, + ClientCompactionTaskQueryTuningConfig.from(config), + GuavaUtils.firstNonNull( + candidateSegments.getPolicyNote(), + candidateSegments.getEligibility().getReason() + ) + )); + break; + case READY: + break; + default: + throw DruidException.defensive("unknown compaction task state[%s]", taskState); } } @@ -121,10 +171,11 @@ public void onCompactionStatusComputed( public void onTaskSubmitted(String taskId, CompactionCandidate candidateSegments) { // Add a row for each task in order of submission - final CompactionStatus status = candidateSegments.getCurrentStatus(); - queuedIntervals.addRow( - createRow(candidateSegments, null, status == null ? "" : status.getReason()) + final String reason = GuavaUtils.firstNonNull( + candidateSegments.getPolicyNote(), + candidateSegments.getEligibility().getReason() ); + queuedIntervals.addRow(createRow(candidateSegments, null, reason)); } }; @@ -147,21 +198,18 @@ public void onTaskSubmitted(String taskId, CompactionCandidate candidateSegments stats ); - final Map compactionStates = new HashMap<>(); + final Map compactionStates = new HashMap<>(); if (!compactedIntervals.isEmpty()) { - compactionStates.put(CompactionStatus.State.COMPLETE, compactedIntervals); + compactionStates.put(CompactionCandidate.TaskState.RECENTLY_COMPLETED, compactedIntervals); } if (!runningIntervals.isEmpty()) { - compactionStates.put(CompactionStatus.State.RUNNING, runningIntervals); + compactionStates.put(CompactionCandidate.TaskState.TASK_IN_PROGRESS, runningIntervals); } if (!queuedIntervals.isEmpty()) { - compactionStates.put(CompactionStatus.State.PENDING, queuedIntervals); - } - if (!skippedIntervals.isEmpty()) { - compactionStates.put(CompactionStatus.State.SKIPPED, skippedIntervals); + compactionStates.put(CompactionCandidate.TaskState.READY, queuedIntervals); } - return new CompactionSimulateResult(compactionStates); + return new CompactionSimulateResult(compactionStates, skippedIntervals); } private Object[] createRow( diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionSimulateResult.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionSimulateResult.java index 7a48ccf0e5ba..6b91c03ff40d 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionSimulateResult.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionSimulateResult.java @@ -26,19 +26,28 @@ public class CompactionSimulateResult { - private final Map compactionStates; + private final Map compactionStates; + private final Table skippedIntervals; @JsonCreator public CompactionSimulateResult( - @JsonProperty("compactionStates") Map compactionStates + @JsonProperty("compactionStates") Map compactionStates, + @JsonProperty("skippedIntervals") Table skippedIntervals ) { this.compactionStates = compactionStates; + this.skippedIntervals = skippedIntervals; } @JsonProperty - public Map getCompactionStates() + public Map getCompactionStates() { return compactionStates; } + + @JsonProperty + public Table getSkippedIntervals() + { + return skippedIntervals; + } } diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java index a163d4dede98..be6380445058 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java @@ -19,38 +19,131 @@ package org.apache.druid.server.compaction; +import com.google.common.base.Strings; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig; +import org.apache.druid.common.config.Configs; +import org.apache.druid.data.input.impl.DimensionSchema; +import org.apache.druid.error.DruidException; +import org.apache.druid.error.InvalidInput; +import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec; +import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; +import org.apache.druid.indexer.partitions.HashedPartitionsSpec; +import org.apache.druid.indexer.partitions.PartitionsSpec; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.granularity.Granularity; +import org.apache.druid.java.util.common.granularity.GranularityType; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.segment.IndexSpec; +import org.apache.druid.segment.metadata.IndexingStateFingerprintMapper; +import org.apache.druid.segment.transform.CompactionTransformSpec; +import org.apache.druid.server.coordinator.DataSourceCompactionConfig; +import org.apache.druid.server.coordinator.UserCompactionTaskGranularityConfig; +import org.apache.druid.timeline.CompactionState; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.utils.CollectionUtils; +import org.joda.time.Interval; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; /** - * Represents the status of compaction for a given {@link CompactionCandidate}. + * Describes the eligibility of an interval for compaction. */ public class CompactionStatus { - static final CompactionStatus COMPLETE = new CompactionStatus(State.COMPLETE, null); + public static final CompactionStatus COMPLETE = new CompactionStatus(State.COMPLETE, "", null, null, null); public enum State { - COMPLETE, PENDING, RUNNING, SKIPPED + COMPLETE, ELIGIBLE, NOT_ELIGIBLE + } + + /** + * List of checks performed to determine if compaction is already complete based on indexing state fingerprints. + */ + static final List> FINGERPRINT_CHECKS = List.of( + Evaluator::allFingerprintedCandidatesHaveExpectedFingerprint + ); + + /** + * List of checks performed to determine if compaction is already complete. + *

+ * The order of the checks must be honored while evaluating them. + */ + static final List> CHECKS = Arrays.asList( + Evaluator::partitionsSpecIsUpToDate, + Evaluator::indexSpecIsUpToDate, + Evaluator::segmentGranularityIsUpToDate, + Evaluator::queryGranularityIsUpToDate, + Evaluator::rollupIsUpToDate, + Evaluator::dimensionsSpecIsUpToDate, + Evaluator::metricsSpecIsUpToDate, + Evaluator::transformSpecFilterIsUpToDate, + Evaluator::projectionsAreUpToDate + ); + + public static CompactionStatus notEligible(String messageFormat, Object... args) + { + return new CompactionStatus(State.NOT_ELIGIBLE, StringUtils.format(messageFormat, args), null, null, null); } private final State state; private final String reason; - private CompactionStatus(State state, String reason) + @Nullable + private final CompactionStatistics compacted; + @Nullable + private final CompactionStatistics uncompacted; + @Nullable + private final List uncompactedSegments; + + private CompactionStatus( + State state, + String reason, + @Nullable CompactionStatistics compacted, + @Nullable CompactionStatistics uncompacted, + @Nullable List uncompactedSegments + ) { this.state = state; this.reason = reason; + switch (state) { + case COMPLETE: + break; + case NOT_ELIGIBLE: + InvalidInput.conditionalException(!Strings.isNullOrEmpty(reason), "must provide a reason"); + break; + case ELIGIBLE: + InvalidInput.conditionalException(compacted != null, "must provide compacted stats"); + InvalidInput.conditionalException(uncompacted != null, "must provide uncompacted stats"); + InvalidInput.conditionalException(uncompactedSegments != null, "must provide uncompactedSegments"); + break; + default: + throw DruidException.defensive("unexpected compaction status state[%s]", state); + } + this.compacted = compacted; + this.uncompacted = uncompacted; + this.uncompactedSegments = uncompactedSegments; } - public boolean isComplete() + static CompactionStatusBuilder builder(State state, String reason) { - return state == State.COMPLETE; + return new CompactionStatusBuilder(state, reason); } - public boolean isSkipped() + public State getState() { - return state == State.SKIPPED; + return state; } public String getReason() @@ -58,32 +151,737 @@ public String getReason() return reason; } - public State getState() + @Nullable + public CompactionStatistics getUncompactedStats() { - return state; + return uncompacted; + } + + @Nullable + public CompactionStatistics getCompactedStats() + { + return compacted; + } + + @Nullable + public List getUncompactedSegments() + { + return uncompactedSegments; + } + + /** + * Evaluates a compaction candidate to determine its eligibility and compaction status. + *

+ * This method performs a two-stage evaluation: + *

    + *
  1. First, uses {@link Evaluator} to check if the candidate needs compaction + * based on the compaction config (e.g., checking segment granularity, partitions spec, etc.)
    2. + *
  2. Then, applies the search policy to determine if this candidate should be compacted in the + * current run (e.g., checking minimum segment count, bytes, or other policy criteria)
    3. + *
+ * + * @param proposedCompaction the compaction candidate to evaluate + * @param config the compaction configuration for the datasource + * @param fingerprintMapper mapper for indexing state fingerprints + * @return a new {@link CompactionCandidate} with updated eligibility and status. For incremental + * compaction, returns a candidate containing only the uncompacted segments. + */ + public static CompactionStatus evaluate( + CompactionCandidate.ProposedCompaction proposedCompaction, + DataSourceCompactionConfig config, + IndexingStateFingerprintMapper fingerprintMapper + ) + { + return new Evaluator(proposedCompaction, config, fingerprintMapper).evaluate(); + } + + @Override + public boolean equals(Object object) + { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + CompactionStatus that = (CompactionStatus) object; + return state == that.state + && Objects.equals(reason, that.reason) + && Objects.equals(compacted, that.compacted) + && Objects.equals(uncompacted, that.uncompacted) + && Objects.equals(uncompactedSegments, that.uncompactedSegments); + } + + @Override + public int hashCode() + { + return Objects.hash(state, reason, compacted, uncompacted, uncompactedSegments); } @Override public String toString() { - return "CompactionStatus{" + - "state=" + state + - ", reason=" + reason + - '}'; + return "CompactionStatus{" + + "state=" + state + + ", reason='" + reason + '\'' + + ", compacted=" + compacted + + ", uncompacted=" + uncompacted + + ", uncompactedSegments=" + uncompactedSegments + + '}'; } - public static CompactionStatus pending(String reasonFormat, Object... args) + /** + * Returns a 'mismatch' reason to be eligible for compaction if config doesn't match, NULL if config matches. + */ + @Nullable + private static String getConfigMismatchReason( + String field, + T configured, + T current, + Function stringFunction + ) { - return new CompactionStatus(State.PENDING, StringUtils.format(reasonFormat, args)); + if (configured == null || configured.equals(current)) { + return null; + } else { + return configChanged(field, configured, current, stringFunction); + } } - public static CompactionStatus skipped(String reasonFormat, Object... args) + private static String configChanged( + String field, + T target, + T current, + Function stringFunction + ) { - return new CompactionStatus(State.SKIPPED, StringUtils.format(reasonFormat, args)); + return StringUtils.format( + "'%s' mismatch: required[%s], current[%s]", + field, + target == null ? null : stringFunction.apply(target), + current == null ? null : stringFunction.apply(current) + ); + } + + private static String asString(Granularity granularity) + { + if (granularity == null) { + return null; + } + for (GranularityType type : GranularityType.values()) { + if (type.getDefaultGranularity().equals(granularity)) { + return type.toString(); + } + } + return granularity.toString(); + } + + private static String asString(PartitionsSpec partitionsSpec) + { + if (partitionsSpec instanceof DimensionRangePartitionsSpec) { + DimensionRangePartitionsSpec rangeSpec = (DimensionRangePartitionsSpec) partitionsSpec; + return StringUtils.format( + "'range' on %s with %,d rows", + rangeSpec.getPartitionDimensions(), rangeSpec.getTargetRowsPerSegment() + ); + } else if (partitionsSpec instanceof HashedPartitionsSpec) { + HashedPartitionsSpec hashedSpec = (HashedPartitionsSpec) partitionsSpec; + return StringUtils.format( + "'hashed' on %s with %,d rows", + hashedSpec.getPartitionDimensions(), hashedSpec.getTargetRowsPerSegment() + ); + } else if (partitionsSpec instanceof DynamicPartitionsSpec) { + DynamicPartitionsSpec dynamicSpec = (DynamicPartitionsSpec) partitionsSpec; + return StringUtils.format( + "'dynamic' with %,d rows", + dynamicSpec.getMaxRowsPerSegment() + ); + } else { + return partitionsSpec.toString(); + } + } + + @Nullable + public static PartitionsSpec findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig tuningConfig) + { + final PartitionsSpec partitionsSpecFromTuningConfig = tuningConfig.getPartitionsSpec(); + if (partitionsSpecFromTuningConfig == null) { + final Long maxTotalRows = tuningConfig.getMaxTotalRows(); + final Integer maxRowsPerSegment = tuningConfig.getMaxRowsPerSegment(); + + if (maxTotalRows == null && maxRowsPerSegment == null) { + // If not specified, return null so that partitionsSpec is not compared + return null; + } else { + return new DynamicPartitionsSpec(maxRowsPerSegment, maxTotalRows); + } + } else if (partitionsSpecFromTuningConfig instanceof DynamicPartitionsSpec) { + return new DynamicPartitionsSpec( + partitionsSpecFromTuningConfig.getMaxRowsPerSegment(), + ((DynamicPartitionsSpec) partitionsSpecFromTuningConfig).getMaxTotalRowsOr(Long.MAX_VALUE) + ); + } else if (partitionsSpecFromTuningConfig instanceof DimensionRangePartitionsSpec) { + return getEffectiveRangePartitionsSpec((DimensionRangePartitionsSpec) partitionsSpecFromTuningConfig); + } else { + return partitionsSpecFromTuningConfig; + } + } + + @Nullable + private static List getNonPartitioningDimensions( + @Nullable final List dimensionSchemas, + @Nullable final PartitionsSpec partitionsSpec, + @Nullable final IndexSpec indexSpec + ) + { + final IndexSpec effectiveIndexSpec = (indexSpec == null ? IndexSpec.getDefault() : indexSpec).getEffectiveSpec(); + if (dimensionSchemas == null || !(partitionsSpec instanceof DimensionRangePartitionsSpec)) { + if (dimensionSchemas != null) { + return dimensionSchemas.stream() + .map(dim -> dim.getEffectiveSchema(effectiveIndexSpec)) + .collect(Collectors.toList()); + } + return null; + } + + final List partitionsDimensions = ((DimensionRangePartitionsSpec) partitionsSpec).getPartitionDimensions(); + return dimensionSchemas.stream() + .filter(dim -> !partitionsDimensions.contains(dim.getName())) + .map(dim -> dim.getEffectiveSchema(effectiveIndexSpec)) + .collect(Collectors.toList()); + } + + /** + * Converts to have only the effective maxRowsPerSegment to avoid false positives when targetRowsPerSegment is set but + * effectively translates to the same maxRowsPerSegment. + */ + static DimensionRangePartitionsSpec getEffectiveRangePartitionsSpec(DimensionRangePartitionsSpec partitionsSpec) + { + return new DimensionRangePartitionsSpec( + null, + partitionsSpec.getMaxRowsPerSegment(), + partitionsSpec.getPartitionDimensions(), + partitionsSpec.isAssumeGrouped() + ); + } + + /** + * Evaluates checks to determine the compaction status of a + * {@link CompactionCandidate}. + */ + private static class Evaluator + { + private static final Logger log = new Logger(Evaluator.class); + + private final DataSourceCompactionConfig compactionConfig; + private final CompactionCandidate.ProposedCompaction proposedCompaction; + private final ClientCompactionTaskQueryTuningConfig tuningConfig; + private final UserCompactionTaskGranularityConfig configuredGranularitySpec; + + private final List fingerprintedSegments = new ArrayList<>(); + private final List compactedSegments = new ArrayList<>(); + private final List uncompactedSegments = new ArrayList<>(); + private final Map> unknownStateToSegments = new HashMap<>(); + + @Nullable + private final IndexingStateFingerprintMapper fingerprintMapper; + @Nullable + private final String targetFingerprint; + + private Evaluator( + CompactionCandidate.ProposedCompaction proposedCompaction, + DataSourceCompactionConfig compactionConfig, + @Nullable IndexingStateFingerprintMapper fingerprintMapper + ) + { + this.proposedCompaction = proposedCompaction; + this.compactionConfig = compactionConfig; + this.tuningConfig = ClientCompactionTaskQueryTuningConfig.from(compactionConfig); + this.configuredGranularitySpec = compactionConfig.getGranularitySpec(); + this.fingerprintMapper = fingerprintMapper; + if (fingerprintMapper == null) { + targetFingerprint = null; + } else { + targetFingerprint = fingerprintMapper.generateFingerprint( + compactionConfig.getDataSource(), + compactionConfig.toCompactionState() + ); + } + } + + /** + * Evaluates the compaction status of candidate segments through a multi-step process: + *
    + *
  1. Validates input bytes are within limits
    2. + *
  2. Categorizes segments by compaction state (fingerprinted, uncompacted, or unknown)
    3. + *
  3. Performs fingerprint-based validation if available (fast path)
    4. + *
  4. Runs detailed checks against unknown states via {@link CompactionStatus#CHECKS}
    5. + *
+ * + * @return Pair of eligibility status and compaction status with reason for first failed check + */ + private CompactionStatus evaluate() + { + final String inputBytesCheck = inputBytesAreWithinLimit(); + if (inputBytesCheck != null) { + return CompactionStatus.notEligible(inputBytesCheck); + } + + List reasonsForCompaction = new ArrayList<>(); + String compactedOnceCheck = segmentsHaveBeenCompactedAtLeastOnce(); + if (compactedOnceCheck != null) { + reasonsForCompaction.add(compactedOnceCheck); + } + + if (fingerprintMapper != null && targetFingerprint != null) { + // First try fingerprint-based evaluation (fast path) + FINGERPRINT_CHECKS.stream() + .map(f -> f.apply(this)) + .filter(Objects::nonNull) + .findFirst() + .ifPresent(reasonsForCompaction::add); + + } + + if (!unknownStateToSegments.isEmpty()) { + // Run CHECKS against any states with uknown compaction status + reasonsForCompaction.addAll( + CHECKS.stream() + .map(f -> f.apply(this)) + .filter(Objects::nonNull) + .collect(Collectors.toList()) + ); + + // Any segments left in unknownStateToSegments passed all checks and are considered compacted + compactedSegments.addAll( + unknownStateToSegments + .values() + .stream() + .flatMap(List::stream) + .collect(Collectors.toList()) + ); + } + + if (reasonsForCompaction.isEmpty()) { + return CompactionStatus.COMPLETE; + } else { + return builder(State.ELIGIBLE, reasonsForCompaction.get(0)).compacted(createStats(compactedSegments)) + .uncompacted(createStats(uncompactedSegments)) + .uncompactedSegments(uncompactedSegments) + .build(); + } + } + + /** + * Evaluates the fingerprints of all fingerprinted candidate segments against the expected fingerprint. + *

+ * If all fingerprinted segments have the expected fingerprint, the check can quickly pass as COMPLETE. However, + * if any fingerprinted segment has a mismatched fingerprint, we need to investigate further by adding them to + * {@link #unknownStateToSegments} where their indexing states will be analyzed. + *

+ */ + private String allFingerprintedCandidatesHaveExpectedFingerprint() + { + Map> mismatchedFingerprintToSegmentMap = new HashMap<>(); + for (DataSegment segment : fingerprintedSegments) { + String fingerprint = segment.getIndexingStateFingerprint(); + if (fingerprint == null) { + // Should not happen since we are iterating over fingerprintedSegments + } else if (fingerprint.equals(targetFingerprint)) { + compactedSegments.add(segment); + } else { + mismatchedFingerprintToSegmentMap + .computeIfAbsent(fingerprint, k -> new ArrayList<>()) + .add(segment); + } + } + + if (mismatchedFingerprintToSegmentMap.isEmpty()) { + // All fingerprinted segments have the expected fingerprint - compaction is complete + return null; + } + + if (fingerprintMapper == null) { + // Cannot evaluate further without a fingerprint mapper + uncompactedSegments.addAll( + mismatchedFingerprintToSegmentMap.values() + .stream() + .flatMap(List::stream) + .collect(Collectors.toList()) + ); + return "Segments have a mismatched fingerprint and no fingerprint mapper is available"; + } + + boolean fingerprintedSegmentWithoutCachedStateFound = false; + + for (Map.Entry> e : mismatchedFingerprintToSegmentMap.entrySet()) { + String fingerprint = e.getKey(); + CompactionState stateToValidate = fingerprintMapper.getStateForFingerprint(fingerprint).orElse(null); + if (stateToValidate == null) { + log.warn("No indexing state found for fingerprint[%s]", fingerprint); + fingerprintedSegmentWithoutCachedStateFound = true; + uncompactedSegments.addAll(e.getValue()); + } else { + // Note that this does not mean we need compaction yet - we need to validate the state further to determine this + unknownStateToSegments.compute( + stateToValidate, + (state, segments) -> { + if (segments == null) { + segments = new ArrayList<>(); + } + segments.addAll(e.getValue()); + return segments; + } + ); + } + } + + if (fingerprintedSegmentWithoutCachedStateFound) { + return "One or more fingerprinted segments do not have a cached indexing state"; + } else { + return null; + } + } + + /** + * Checks if all the segments have been compacted at least once and groups them into uncompacted, fingerprinted, or + * non-fingerprinted. + */ + private String segmentsHaveBeenCompactedAtLeastOnce() + { + for (DataSegment segment : proposedCompaction.getSegments()) { + final String fingerprint = segment.getIndexingStateFingerprint(); + final CompactionState segmentState = segment.getLastCompactionState(); + if (fingerprint != null) { + fingerprintedSegments.add(segment); + } else if (segmentState == null) { + uncompactedSegments.add(segment); + } else { + unknownStateToSegments.computeIfAbsent(segmentState, s -> new ArrayList<>()).add(segment); + } + } + + if (uncompactedSegments.isEmpty()) { + return null; + } else { + return "not compacted yet"; + } + } + + private String partitionsSpecIsUpToDate() + { + return evaluateForAllCompactionStates(this::partitionsSpecIsUpToDate); + } + + private String indexSpecIsUpToDate() + { + return evaluateForAllCompactionStates(this::indexSpecIsUpToDate); + } + + private String projectionsAreUpToDate() + { + return evaluateForAllCompactionStates(this::projectionsAreUpToDate); + } + + private String segmentGranularityIsUpToDate() + { + return evaluateForAllCompactionStates(this::segmentGranularityIsUpToDate); + } + + private String rollupIsUpToDate() + { + return evaluateForAllCompactionStates(this::rollupIsUpToDate); + } + + private String queryGranularityIsUpToDate() + { + return evaluateForAllCompactionStates(this::queryGranularityIsUpToDate); + } + + private String dimensionsSpecIsUpToDate() + { + return evaluateForAllCompactionStates(this::dimensionsSpecIsUpToDate); + } + + private String metricsSpecIsUpToDate() + { + return evaluateForAllCompactionStates(this::metricsSpecIsUpToDate); + } + + private String transformSpecFilterIsUpToDate() + { + return evaluateForAllCompactionStates(this::transformSpecFilterIsUpToDate); + } + + private String partitionsSpecIsUpToDate(CompactionState lastCompactionState) + { + PartitionsSpec existingPartionsSpec = lastCompactionState.getPartitionsSpec(); + if (existingPartionsSpec instanceof DimensionRangePartitionsSpec) { + existingPartionsSpec = getEffectiveRangePartitionsSpec((DimensionRangePartitionsSpec) existingPartionsSpec); + } else if (existingPartionsSpec instanceof DynamicPartitionsSpec) { + existingPartionsSpec = new DynamicPartitionsSpec( + existingPartionsSpec.getMaxRowsPerSegment(), + ((DynamicPartitionsSpec) existingPartionsSpec).getMaxTotalRowsOr(Long.MAX_VALUE) + ); + } + return getConfigMismatchReason( + "partitionsSpec", + findPartitionsSpecFromConfig(tuningConfig), + existingPartionsSpec, + CompactionStatus::asString + ); + } + + private String indexSpecIsUpToDate(CompactionState lastCompactionState) + { + return getConfigMismatchReason( + "indexSpec", + Configs.valueOrDefault(tuningConfig.getIndexSpec(), IndexSpec.getDefault()).getEffectiveSpec(), + lastCompactionState.getIndexSpec().getEffectiveSpec(), + String::valueOf + ); + } + + private String projectionsAreUpToDate(CompactionState lastCompactionState) + { + return getConfigMismatchReason( + "projections", + compactionConfig.getProjections(), + lastCompactionState.getProjections(), + String::valueOf + ); + } + + @Nullable + private String inputBytesAreWithinLimit() + { + final long inputSegmentSize = compactionConfig.getInputSegmentSizeBytes(); + if (proposedCompaction.getTotalBytes() > inputSegmentSize) { + return StringUtils.format( + "'inputSegmentSize' exceeded: Total segment size[%d] is larger than allowed inputSegmentSize[%d]", + proposedCompaction.getTotalBytes(), inputSegmentSize + ); + } + return null; + } + + private String segmentGranularityIsUpToDate(CompactionState lastCompactionState) + { + if (configuredGranularitySpec == null + || configuredGranularitySpec.getSegmentGranularity() == null) { + return null; + } + + final Granularity configuredSegmentGranularity = configuredGranularitySpec.getSegmentGranularity(); + final UserCompactionTaskGranularityConfig existingGranularitySpec = getGranularitySpec(lastCompactionState); + final Granularity existingSegmentGranularity + = existingGranularitySpec == null ? null : existingGranularitySpec.getSegmentGranularity(); + + if (configuredSegmentGranularity.equals(existingSegmentGranularity)) { + return null; + } else if (existingSegmentGranularity == null) { + // Candidate segments were compacted without segment granularity specified + // Check if the segments already have the desired segment granularity + final List segmentsForState = unknownStateToSegments.get(lastCompactionState); + boolean needsCompaction = segmentsForState.stream().anyMatch( + segment -> !configuredSegmentGranularity.isAligned(segment.getInterval()) + ); + if (needsCompaction) { + return StringUtils.format( + "segmentGranularity: segments do not align with target[%s]", + CompactionStatus.asString(configuredSegmentGranularity) + ); + } + } else { + return configChanged( + "segmentGranularity", + configuredSegmentGranularity, + existingSegmentGranularity, + CompactionStatus::asString + ); + } + + return null; + } + + private String rollupIsUpToDate(CompactionState lastCompactionState) + { + if (configuredGranularitySpec == null) { + return null; + } else { + final UserCompactionTaskGranularityConfig existingGranularitySpec + = getGranularitySpec(lastCompactionState); + return getConfigMismatchReason( + "rollup", + configuredGranularitySpec.isRollup(), + existingGranularitySpec == null ? null : existingGranularitySpec.isRollup(), + String::valueOf + ); + } + } + + private String queryGranularityIsUpToDate(CompactionState lastCompactionState) + { + if (configuredGranularitySpec == null) { + return null; + } else { + final UserCompactionTaskGranularityConfig existingGranularitySpec + = getGranularitySpec(lastCompactionState); + return getConfigMismatchReason( + "queryGranularity", + configuredGranularitySpec.getQueryGranularity(), + existingGranularitySpec == null ? null : existingGranularitySpec.getQueryGranularity(), + CompactionStatus::asString + ); + } + } + + /** + * Removes partition dimensions before comparison, since they are placed in front of the sort order -- + * which can create a mismatch between expected and actual order of dimensions. Partition dimensions are separately + * covered in {@link Evaluator#partitionsSpecIsUpToDate()} check. + */ + private String dimensionsSpecIsUpToDate(CompactionState lastCompactionState) + { + if (compactionConfig.getDimensionsSpec() == null) { + return null; + } else { + List existingDimensions = getNonPartitioningDimensions( + lastCompactionState.getDimensionsSpec() == null + ? null + : lastCompactionState.getDimensionsSpec().getDimensions(), + lastCompactionState.getPartitionsSpec(), + lastCompactionState.getIndexSpec() + ); + List configuredDimensions = getNonPartitioningDimensions( + compactionConfig.getDimensionsSpec().getDimensions(), + compactionConfig.getTuningConfig() == null ? null : compactionConfig.getTuningConfig().getPartitionsSpec(), + compactionConfig.getTuningConfig() == null + ? IndexSpec.getDefault() + : compactionConfig.getTuningConfig().getIndexSpec() + ); + return getConfigMismatchReason( + "dimensionsSpec", + configuredDimensions, + existingDimensions, + String::valueOf + ); + } + } + + private String metricsSpecIsUpToDate(CompactionState lastCompactionState) + { + final AggregatorFactory[] configuredMetricsSpec = compactionConfig.getMetricsSpec(); + if (ArrayUtils.isEmpty(configuredMetricsSpec)) { + return null; + } + + final List metricSpecList = lastCompactionState.getMetricsSpec(); + final AggregatorFactory[] existingMetricsSpec + = CollectionUtils.isNullOrEmpty(metricSpecList) + ? null : metricSpecList.toArray(new AggregatorFactory[0]); + + if (existingMetricsSpec == null || !Arrays.deepEquals(configuredMetricsSpec, existingMetricsSpec)) { + return configChanged( + "metricsSpec", + configuredMetricsSpec, + existingMetricsSpec, + Arrays::toString + ); + } else { + return null; + } + } + + private String transformSpecFilterIsUpToDate(CompactionState lastCompactionState) + { + if (compactionConfig.getTransformSpec() == null) { + return null; + } + + CompactionTransformSpec existingTransformSpec = lastCompactionState.getTransformSpec(); + return getConfigMismatchReason( + "transformSpec filter", + compactionConfig.getTransformSpec().getFilter(), + existingTransformSpec == null ? null : existingTransformSpec.getFilter(), + String::valueOf + ); + } + + /** + * Evaluates the given check for each entry in the {@link #unknownStateToSegments}. + * If any entry fails the given check by returning a status which is not + * COMPLETE, all the segments with that state are moved to {@link #uncompactedSegments}. + * + * @return The first status which is not COMPLETE. + */ + private String evaluateForAllCompactionStates(Function check) + { + String firstIncomplete = null; + for (CompactionState state : List.copyOf(unknownStateToSegments.keySet())) { + final String eligibleReason = check.apply(state); + if (eligibleReason != null) { + uncompactedSegments.addAll(unknownStateToSegments.remove(state)); + if (firstIncomplete == null) { + firstIncomplete = eligibleReason; + } + } + } + + return firstIncomplete; + } + + private static UserCompactionTaskGranularityConfig getGranularitySpec( + CompactionState compactionState + ) + { + return UserCompactionTaskGranularityConfig.from(compactionState.getGranularitySpec()); + } + + private static CompactionStatistics createStats(List segments) + { + final Set segmentIntervals = + segments.stream().map(DataSegment::getInterval).collect(Collectors.toSet()); + final long totalBytes = segments.stream().mapToLong(DataSegment::getSize).sum(); + return CompactionStatistics.create(totalBytes, segments.size(), segmentIntervals.size()); + } } - public static CompactionStatus running(String message) + static class CompactionStatusBuilder { - return new CompactionStatus(State.RUNNING, message); + private State state; + private CompactionStatistics compacted; + private CompactionStatistics uncompacted; + private List uncompactedSegments; + private String reason; + + CompactionStatusBuilder(State state, String reason) + { + this.state = state; + this.reason = reason; + } + + CompactionStatusBuilder compacted(CompactionStatistics compacted) + { + this.compacted = compacted; + return this; + } + + CompactionStatusBuilder uncompacted(CompactionStatistics uncompacted) + { + this.uncompacted = uncompacted; + return this; + } + + CompactionStatusBuilder uncompactedSegments(List uncompactedSegments) + { + this.uncompactedSegments = uncompactedSegments; + return this; + } + + CompactionStatus build() + { + return new CompactionStatus(state, reason, compacted, uncompacted, uncompactedSegments); + } } } diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusTracker.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusTracker.java index 924d1300ea33..fb5dd40056e7 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusTracker.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusTracker.java @@ -80,12 +80,12 @@ public Set getSubmittedTaskIds() * This method assumes that the given candidate is eligible for compaction * based on the current compaction config/supervisor of the datasource. */ - public CompactionStatus computeCompactionStatus(CompactionCandidate candidate) + public CompactionCandidate.TaskState computeCompactionTaskState(CompactionCandidate candidate) { // Skip intervals that already have a running task final CompactionTaskStatus lastTaskStatus = getLatestTaskStatus(candidate); if (lastTaskStatus != null && lastTaskStatus.getState() == TaskState.RUNNING) { - return CompactionStatus.running("Task for interval is already running"); + return CompactionCandidate.TaskState.TASK_IN_PROGRESS; } // Skip intervals that have been recently compacted if segment timeline is not updated yet @@ -93,19 +93,17 @@ public CompactionStatus computeCompactionStatus(CompactionCandidate candidate) if (lastTaskStatus != null && lastTaskStatus.getState() == TaskState.SUCCESS && snapshotTime != null && snapshotTime.isBefore(lastTaskStatus.getUpdatedTime())) { - return CompactionStatus.skipped( - "Segment timeline not updated since last compaction task succeeded" - ); + return CompactionCandidate.TaskState.RECENTLY_COMPLETED; } - return CompactionStatus.pending("Not compacted yet"); + return CompactionCandidate.TaskState.READY; } /** * Tracks the latest compaction status of the given compaction candidates. * Used only by the {@link CompactionRunSimulator}. */ - public void onCompactionStatusComputed( + public void onSkippedCandidate( CompactionCandidate candidateSegments, DataSourceCompactionConfig config ) @@ -113,6 +111,23 @@ public void onCompactionStatusComputed( // Nothing to do, used by simulator } + public void onCompactionCandidates( + CompactionCandidate candidateSegments, + DataSourceCompactionConfig config + ) + { + // Nothing to do, used by simulator + } + + public void onCompactionTaskStateComputed( + CompactionCandidate candidateSegments, + CompactionCandidate.TaskState taskState, + DataSourceCompactionConfig config + ) + { + // Nothing to do, used by simulator + } + public void onSegmentTimelineUpdated(DateTime snapshotTime) { this.segmentSnapshotTime.set(snapshotTime); diff --git a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java index e753c7a8dce3..e316925f5d21 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java +++ b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java @@ -28,6 +28,7 @@ import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.JodaUtils; +import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.guava.Comparators; import org.apache.druid.java.util.common.logger.Logger; @@ -57,7 +58,6 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; -import java.util.Objects; import java.util.PriorityQueue; import java.util.Set; import java.util.stream.Collectors; @@ -126,11 +126,10 @@ private void populateQueue(SegmentTimeline timeline, List skipInterval // Do not use the target segment granularity in the CompactionCandidate // as Granularities.getIterable() will cause OOM due to the above issue CompactionCandidate candidatesWithStatus = - CompactionEligibility.fail("Segments have partial-eternity intervals") - .createCandidate(CompactionCandidate.ProposedCompaction.from( - partialEternitySegments, - null - )); + CompactionMode.notEligible( + CompactionCandidate.ProposedCompaction.from(partialEternitySegments, null), + "Segments have partial-eternity intervals" + ); skippedSegments.add(candidatesWithStatus); return; } @@ -336,29 +335,31 @@ private void findAndEnqueueSegmentsToCompact(CompactibleSegmentIterator compacti continue; } - CompactionCandidate.ProposedCompaction candidate = + CompactionCandidate.ProposedCompaction proposed = CompactionCandidate.ProposedCompaction.from(segments, config.getSegmentGranularity()); - final CompactionCandidate candidatesWithStatus = - CompactionEligibility.evaluate(candidate, config, searchPolicy, fingerprintMapper).createCandidate(candidate); - switch (Objects.requireNonNull(candidatesWithStatus.getPolicyEligibility()).getState()) { - case NOT_APPLICABLE: - compactedSegments.add(candidatesWithStatus); - break; - case NOT_ELIGIBLE: - skippedSegments.add(candidatesWithStatus); - break; - case FULL_COMPACTION: + final CompactionStatus eligibility = CompactionStatus.evaluate(proposed, config, fingerprintMapper); + final CompactionCandidate candidate = + CompactionStatus.State.ELIGIBLE.equals(eligibility.getState()) + ? searchPolicy.createCandidate(proposed, eligibility) + : CompactionMode.notEligible(proposed, eligibility.getReason()); + + switch (candidate.getMode()) { case INCREMENTAL_COMPACTION: - if (!queuedIntervals.contains(candidatesWithStatus.getProposedCompaction().getUmbrellaInterval())) { - queue.add(candidatesWithStatus); - queuedIntervals.add(candidatesWithStatus.getProposedCompaction().getUmbrellaInterval()); + case FULL_COMPACTION: + if (!queuedIntervals.contains(candidate.getProposedCompaction().getUmbrellaInterval())) { + queue.add(candidate); + queuedIntervals.add(candidate.getProposedCompaction().getUmbrellaInterval()); + } + break; + case NOT_APPLICABLE: + if (CompactionStatus.State.COMPLETE.equals(candidate.getEligibility().getState())) { + compactedSegments.add(candidate); + } else { + skippedSegments.add(candidate); } break; default: - throw DruidException.defensive( - "Unexpected eligibility state[%s]", - candidatesWithStatus.getPolicyEligibility().getState() - ); + throw DruidException.defensive("Unexpected compaction mode[%s]", candidate.getMode()); } } } @@ -394,14 +395,14 @@ private List findInitialSearchInterval( final CompactionCandidate.ProposedCompaction candidates = CompactionCandidate.ProposedCompaction.from(segments, config.getSegmentGranularity()); - final CompactionEligibility eligibility; + final String skipReason; if (candidates.getCompactionInterval().overlaps(latestSkipInterval)) { - eligibility = CompactionEligibility.fail("skip offset from latest[%s]", skipOffset); + skipReason = StringUtils.format("skip offset from latest[%s]", skipOffset); } else { - eligibility = CompactionEligibility.fail("interval locked by another task"); + skipReason = "interval locked by another task"; } - final CompactionCandidate candidatesWithStatus = eligibility.createCandidate(candidates); + final CompactionCandidate candidatesWithStatus = CompactionMode.notEligible(candidates, skipReason); skippedSegments.add(candidatesWithStatus); } } diff --git a/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java index 04040ee0ef5b..279beca7a654 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/FixedIntervalOrderPolicy.java @@ -56,14 +56,18 @@ public int compareCandidates(CompactionCandidate candidateA, CompactionCandidate } @Override - public CompactionEligibility checkEligibilityForCompaction( + public CompactionCandidate createCandidate( CompactionCandidate.ProposedCompaction candidate, - CompactionEligibility eligibility + CompactionStatus eligibility ) { return findIndex(candidate) < Integer.MAX_VALUE - ? eligibility - : CompactionEligibility.fail("Datasource/Interval is not in the list of 'eligibleCandidates'"); + ? CompactionMode.FULL_COMPACTION.createCandidate(candidate, eligibility) + : CompactionMode.failWithPolicyCheck( + candidate, + eligibility, + "Datasource/Interval is not in the list of 'eligibleCandidates'" + ); } private int findIndex(CompactionCandidate.ProposedCompaction candidate) diff --git a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java index 62257f6bf686..8fed350e2d6e 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java @@ -132,7 +132,7 @@ public Double getIncrementalCompactionUncompactedRatioThreshold() @Override protected Comparator getSegmentComparator() { - return Comparator.comparing(o -> Objects.requireNonNull(o.getPolicyEligibility()), this::compare); + return Comparator.comparing(o -> Objects.requireNonNull(o.getEligibility()), this::compare); } @Override @@ -180,7 +180,7 @@ public String toString() '}'; } - private int compare(CompactionEligibility candidateA, CompactionEligibility candidateB) + private int compare(CompactionStatus candidateA, CompactionStatus candidateB) { final double fragmentationDiff = computeFragmentationIndex(candidateB) - computeFragmentationIndex(candidateA); @@ -188,31 +188,41 @@ private int compare(CompactionEligibility candidateA, CompactionEligibility cand } @Override - public CompactionEligibility checkEligibilityForCompaction( + public CompactionCandidate createCandidate( CompactionCandidate.ProposedCompaction candidate, - CompactionEligibility eligibility + CompactionStatus eligibility ) { final CompactionStatistics uncompacted = Objects.requireNonNull(eligibility.getUncompactedStats()); + if (uncompacted.getNumSegments() < 1) { - return CompactionEligibility.fail("No uncompacted segments in interval"); + return CompactionMode.failWithPolicyCheck(candidate, eligibility, "No uncompacted segments in interval"); } else if (uncompacted.getNumSegments() < minUncompactedCount) { - return CompactionEligibility.fail( + return CompactionMode.failWithPolicyCheck( + candidate, + eligibility, "Uncompacted segments[%,d] in interval must be at least [%,d]", - uncompacted.getNumSegments(), minUncompactedCount + uncompacted.getNumSegments(), + minUncompactedCount ); } else if (uncompacted.getTotalBytes() < minUncompactedBytes.getBytes()) { - return CompactionEligibility.fail( + return CompactionMode.failWithPolicyCheck( + candidate, + eligibility, "Uncompacted bytes[%,d] in interval must be at least [%,d]", - uncompacted.getTotalBytes(), minUncompactedBytes.getBytes() + uncompacted.getTotalBytes(), + minUncompactedBytes.getBytes() ); } final long avgSegmentSize = (uncompacted.getTotalBytes() / uncompacted.getNumSegments()); if (avgSegmentSize > maxAverageUncompactedBytesPerSegment.getBytes()) { - return CompactionEligibility.fail( + return CompactionMode.failWithPolicyCheck( + candidate, + eligibility, "Average size[%,d] of uncompacted segments in interval must be at most [%,d]", - avgSegmentSize, maxAverageUncompactedBytesPerSegment.getBytes() + avgSegmentSize, + maxAverageUncompactedBytesPerSegment.getBytes() ); } @@ -220,18 +230,14 @@ public CompactionEligibility checkEligibilityForCompaction( (uncompacted.getTotalBytes() + eligibility.getCompactedStats() .getTotalBytes()); if (uncompactedBytesRatio < incrementalCompactionUncompactedBytesRatioThreshold) { - String reason = StringUtils.format( + String policyNote = StringUtils.format( "Uncompacted bytes ratio[%.2f] is below threshold[%.2f]", uncompactedBytesRatio, incrementalCompactionUncompactedBytesRatioThreshold ); - return CompactionEligibility.builder(CompactionEligibility.State.INCREMENTAL_COMPACTION, reason) - .compacted(eligibility.getCompactedStats()) - .uncompacted(eligibility.getUncompactedStats()) - .uncompactedSegments(eligibility.getUncompactedSegments()) - .build(); + return CompactionMode.INCREMENTAL_COMPACTION.createCandidate(candidate, eligibility, policyNote); } else { - return eligibility; + return CompactionMode.FULL_COMPACTION.createCandidate(candidate, eligibility); } } @@ -242,7 +248,7 @@ public CompactionEligibility checkEligibilityForCompaction( * A higher fragmentation index causes the candidate to be higher in priority * for compaction. */ - private double computeFragmentationIndex(CompactionEligibility eligibility) + private double computeFragmentationIndex(CompactionStatus eligibility) { final CompactionStatistics uncompacted = eligibility.getUncompactedStats(); if (uncompacted == null || uncompacted.getNumSegments() < 1 || uncompacted.getTotalBytes() < 1) { diff --git a/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java index 0c5a13a6682c..39a24d3eb5e8 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java @@ -32,7 +32,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.transform.CompactionTransformSpec; -import org.apache.druid.server.compaction.CompactionEligibility; +import org.apache.druid.server.compaction.CompactionStatus; import org.apache.druid.timeline.CompactionState; import org.joda.time.Period; @@ -112,7 +112,7 @@ default CompactionState toCompactionState() { ClientCompactionTaskQueryTuningConfig tuningConfig = ClientCompactionTaskQueryTuningConfig.from(this); - PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(tuningConfig); + PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(tuningConfig); IndexSpec indexSpec = tuningConfig.getIndexSpec() == null ? IndexSpec.getDefault().getEffectiveSpec() diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java index 66d31da79386..c44e34584335 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java @@ -47,7 +47,6 @@ import org.apache.druid.server.compaction.CompactionSegmentIterator; import org.apache.druid.server.compaction.CompactionSlotManager; import org.apache.druid.server.compaction.CompactionSnapshotBuilder; -import org.apache.druid.server.compaction.CompactionStatus; import org.apache.druid.server.compaction.CompactionStatusTracker; import org.apache.druid.server.compaction.PriorityBasedCompactionSegmentIterator; import org.apache.druid.server.coordinator.AutoCompactionSnapshot; @@ -243,19 +242,20 @@ private int submitCompactionTasks( final String dataSourceName = entry.getDataSource(); final DataSourceCompactionConfig config = compactionConfigs.get(dataSourceName); - final CompactionStatus compactionStatus = statusTracker.computeCompactionStatus(entry); - final CompactionCandidate candidatesWithStatus = entry.withCurrentStatus(compactionStatus); - statusTracker.onCompactionStatusComputed(candidatesWithStatus, config); - - if (compactionStatus.isComplete()) { - snapshotBuilder.addToComplete(candidatesWithStatus); - continue; - } else if (compactionStatus.isSkipped()) { - snapshotBuilder.addToSkipped(candidatesWithStatus); - continue; - } else { - // As these segments will be compacted, we will aggregate the statistic to the Compacted statistics - snapshotBuilder.addToComplete(entry); + final CompactionCandidate.TaskState compactionTaskState = statusTracker.computeCompactionTaskState(entry); + statusTracker.onCompactionTaskStateComputed(entry, compactionTaskState, config); + + switch (compactionTaskState) { + case READY: + case TASK_IN_PROGRESS: + // As these segments will be compacted, we will aggregate the statistic to the Compacted statistics + snapshotBuilder.addToComplete(entry); + break; + case RECENTLY_COMPLETED: + snapshotBuilder.addToComplete(entry); + break; + default: + throw DruidException.defensive("unexpected task state[%s]", compactionTaskState); } final ClientCompactionTaskQuery taskPayload = createCompactionTask( @@ -369,8 +369,8 @@ public static ClientCompactionTaskQuery createCompactionTask( } final Map autoCompactionContext = newAutoCompactionContext(config.getTaskContext()); - if (candidate.getCurrentStatus() != null) { - autoCompactionContext.put(COMPACTION_REASON_KEY, candidate.getCurrentStatus().getReason()); + if (candidate.getEligibility().getReason() != null) { + autoCompactionContext.put(COMPACTION_REASON_KEY, candidate.getEligibility().getReason()); } autoCompactionContext.put(STORE_COMPACTION_STATE_KEY, storeCompactionStatePerSegment); @@ -416,7 +416,7 @@ private void updateCompactionSnapshotStats( } iterator.getCompactedSegments().forEach(snapshotBuilder::addToComplete); iterator.getSkippedSegments().forEach(entry -> { - statusTracker.onCompactionStatusComputed(entry, datasourceToConfig.get(entry.getDataSource())); + statusTracker.onSkippedCandidate(entry, datasourceToConfig.get(entry.getDataSource())); snapshotBuilder.addToSkipped(entry); }); @@ -462,8 +462,7 @@ private static ClientCompactionTaskQuery compactSegments( final String taskId = IdUtils.newTaskId(TASK_ID_PREFIX, ClientCompactionTaskQuery.TYPE, dataSource, null); final ClientCompactionIntervalSpec clientCompactionIntervalSpec; - Preconditions.checkArgument(entry.getPolicyEligibility() != null, "Must have a policy eligibility"); - switch (entry.getPolicyEligibility().getState()) { + switch (entry.getMode()) { case FULL_COMPACTION: clientCompactionIntervalSpec = new ClientCompactionIntervalSpec(entry.getCompactionInterval(), null, null); break; @@ -475,7 +474,7 @@ private static ClientCompactionTaskQuery compactSegments( ); break; default: - throw DruidException.defensive("Unexpected policy eligibility[%s]", entry.getPolicyEligibility()); + throw DruidException.defensive("Unexpected compaction mode[%s]", entry.getMode()); } return new ClientCompactionTaskQuery( diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionCandidateTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionCandidateTest.java index 96a275681b0e..d17c7ec3a415 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionCandidateTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionCandidateTest.java @@ -38,14 +38,12 @@ public void testConstructorAndGetters() { List segments = createTestSegments(3); CompactionCandidate.ProposedCompaction proposed = CompactionCandidate.ProposedCompaction.from(segments, null); - CompactionEligibility eligibility = CompactionEligibility.fail("test reason"); - CompactionStatus status = CompactionStatus.COMPLETE; + CompactionStatus eligibility = CompactionStatus.notEligible("test reason"); - CompactionCandidate candidate = new CompactionCandidate(proposed, eligibility, status); + CompactionCandidate candidate = new CompactionCandidate(proposed, eligibility, null, CompactionMode.FULL_COMPACTION); Assert.assertEquals(proposed, candidate.getProposedCompaction()); - Assert.assertEquals(eligibility, candidate.getPolicyEligibility()); - Assert.assertEquals(status, candidate.getCurrentStatus()); + Assert.assertEquals(eligibility, candidate.getEligibility()); Assert.assertEquals(segments, candidate.getSegments()); Assert.assertEquals(DATASOURCE, candidate.getDataSource()); Assert.assertEquals(3, candidate.numSegments()); @@ -95,25 +93,6 @@ public void testProposedCompactionThrowsOnNullOrEmptySegments() ); } - @Test - public void testWithCurrentStatus() - { - List segments = createTestSegments(2); - CompactionCandidate.ProposedCompaction proposed = CompactionCandidate.ProposedCompaction.from(segments, null); - CompactionEligibility eligibility = CompactionEligibility.fail("test"); - CompactionStatus originalStatus = CompactionStatus.COMPLETE; - - CompactionCandidate candidate = new CompactionCandidate(proposed, eligibility, originalStatus); - CompactionStatus newStatus = CompactionStatus.pending("needs compaction"); - CompactionCandidate updated = candidate.withCurrentStatus(newStatus); - - Assert.assertNotSame(candidate, updated); - Assert.assertEquals(originalStatus, candidate.getCurrentStatus()); - Assert.assertEquals(newStatus, updated.getCurrentStatus()); - Assert.assertEquals(proposed, updated.getProposedCompaction()); - Assert.assertEquals(eligibility, updated.getPolicyEligibility()); - } - @Test public void testDelegationMethods() { @@ -121,8 +100,9 @@ public void testDelegationMethods() CompactionCandidate.ProposedCompaction proposed = CompactionCandidate.ProposedCompaction.from(segments, null); CompactionCandidate candidate = new CompactionCandidate( proposed, - CompactionEligibility.fail("test"), - CompactionStatus.COMPLETE + CompactionStatus.notEligible("test"), + null, + CompactionMode.FULL_COMPACTION ); Assert.assertEquals(proposed.getTotalBytes(), candidate.getTotalBytes()); diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityEvaluateTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityEvaluateTest.java deleted file mode 100644 index 5dc42f14afc5..000000000000 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityEvaluateTest.java +++ /dev/null @@ -1,895 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.server.compaction; - -import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig; -import org.apache.druid.data.input.impl.AggregateProjectionSpec; -import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.impl.LongDimensionSchema; -import org.apache.druid.data.input.impl.StringDimensionSchema; -import org.apache.druid.indexer.granularity.GranularitySpec; -import org.apache.druid.indexer.granularity.UniformGranularitySpec; -import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec; -import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; -import org.apache.druid.indexer.partitions.HashedPartitionsSpec; -import org.apache.druid.indexer.partitions.PartitionsSpec; -import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.query.aggregation.LongSumAggregatorFactory; -import org.apache.druid.segment.AutoTypeColumnSchema; -import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.TestDataSource; -import org.apache.druid.segment.data.CompressionStrategy; -import org.apache.druid.segment.metadata.DefaultIndexingStateFingerprintMapper; -import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; -import org.apache.druid.segment.metadata.IndexingStateCache; -import org.apache.druid.segment.metadata.IndexingStateFingerprintMapper; -import org.apache.druid.segment.nested.NestedCommonFormatColumnFormatSpec; -import org.apache.druid.server.coordinator.DataSourceCompactionConfig; -import org.apache.druid.server.coordinator.InlineSchemaDataSourceCompactionConfig; -import org.apache.druid.server.coordinator.UserCompactionTaskDimensionsConfig; -import org.apache.druid.server.coordinator.UserCompactionTaskGranularityConfig; -import org.apache.druid.server.coordinator.UserCompactionTaskQueryTuningConfig; -import org.apache.druid.timeline.CompactionState; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.SegmentId; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.util.Collections; -import java.util.List; - -public class CompactionEligibilityEvaluateTest -{ - private static final DataSegment WIKI_SEGMENT - = DataSegment.builder(SegmentId.of(TestDataSource.WIKI, Intervals.of("2013-01-01/PT1H"), "v1", 0)) - .size(100_000_000L) - .build(); - private static final DataSegment WIKI_SEGMENT_2 - = DataSegment.builder(SegmentId.of(TestDataSource.WIKI, Intervals.of("2013-01-01/PT1H"), "v1", 1)) - .size(100_000_000L) - .build(); - - private HeapMemoryIndexingStateStorage indexingStateStorage; - private IndexingStateCache indexingStateCache; - private IndexingStateFingerprintMapper fingerprintMapper; - - @Before - public void setUp() - { - indexingStateStorage = new HeapMemoryIndexingStateStorage(); - indexingStateCache = new IndexingStateCache(); - fingerprintMapper = new DefaultIndexingStateFingerprintMapper( - indexingStateCache, - new DefaultObjectMapper() - ); - } - - /** - * Helper to sync the cache with states stored in the manager (for tests that persist states). - */ - private void syncCacheFromManager() - { - indexingStateCache.resetIndexingStatesForPublishedSegments(indexingStateStorage.getAllStoredStates()); - } - - @Test - public void testFindPartitionsSpecWhenGivenIsNull() - { - final ClientCompactionTaskQueryTuningConfig tuningConfig - = ClientCompactionTaskQueryTuningConfig.from(null); - Assert.assertNull(CompactionEligibility.findPartitionsSpecFromConfig(tuningConfig)); - } - - @Test - public void testFindPartitionsSpecWhenGivenIsDynamicWithNullMaxTotalRows() - { - final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(null, null); - final ClientCompactionTaskQueryTuningConfig tuningConfig - = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); - Assert.assertEquals( - new DynamicPartitionsSpec(null, Long.MAX_VALUE), - CompactionEligibility.findPartitionsSpecFromConfig(tuningConfig) - ); - } - - @Test - public void testFindPartitionsSpecWhenGivenIsDynamicWithMaxTotalRows() - { - final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(null, 1000L); - final ClientCompactionTaskQueryTuningConfig tuningConfig - = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); - Assert.assertEquals( - partitionsSpec, - CompactionEligibility.findPartitionsSpecFromConfig(tuningConfig) - ); - } - - @Test - public void testFindPartitionsSpecWhenGivenIsDynamicWithMaxRowsPerSegment() - { - final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(100, 1000L); - final ClientCompactionTaskQueryTuningConfig tuningConfig - = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); - Assert.assertEquals( - partitionsSpec, - CompactionEligibility.findPartitionsSpecFromConfig(tuningConfig) - ); - } - - @Test - public void testFindPartitionsSpecFromConfigWithDeprecatedMaxRowsPerSegmentAndMaxTotalRowsReturnGivenValues() - { - final DataSourceCompactionConfig config = - InlineSchemaDataSourceCompactionConfig.builder() - .forDataSource("datasource") - .withMaxRowsPerSegment(100) - .withTuningConfig( - new UserCompactionTaskQueryTuningConfig( - null, - null, - null, - 1000L, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null - ) - ) - .build(); - Assert.assertEquals( - new DynamicPartitionsSpec(100, 1000L), - CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(config)) - ); - } - - @Test - public void testFindPartitionsSpecWhenGivenIsHashed() - { - final PartitionsSpec partitionsSpec = - new HashedPartitionsSpec(null, 100, Collections.singletonList("dim")); - final ClientCompactionTaskQueryTuningConfig tuningConfig - = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); - Assert.assertEquals( - partitionsSpec, - CompactionEligibility.findPartitionsSpecFromConfig(tuningConfig) - ); - } - - @Test - public void testFindPartitionsSpecWhenGivenIsRangeWithMaxRows() - { - final PartitionsSpec partitionsSpec = - new DimensionRangePartitionsSpec(null, 10000, Collections.singletonList("dim"), false); - final ClientCompactionTaskQueryTuningConfig tuningConfig - = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); - Assert.assertEquals( - partitionsSpec, - CompactionEligibility.findPartitionsSpecFromConfig(tuningConfig) - ); - } - - @Test - public void testFindPartitionsSpecWhenGivenIsRangeWithTargetRows() - { - final PartitionsSpec partitionsSpec = - new DimensionRangePartitionsSpec(10000, null, Collections.singletonList("dim"), false); - final ClientCompactionTaskQueryTuningConfig tuningConfig - = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); - Assert.assertEquals( - new DimensionRangePartitionsSpec(null, 15000, Collections.singletonList("dim"), false), - CompactionEligibility.findPartitionsSpecFromConfig(tuningConfig) - ); - } - - @Test - public void testStatusWhenLastCompactionStateIsNull() - { - verifyCompactionIsEligibleBecause( - null, - InlineSchemaDataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(), - "not compacted yet" - ); - } - - @Test - public void testStatusWhenLastCompactionStateIsEmpty() - { - final PartitionsSpec requiredPartitionsSpec = new DynamicPartitionsSpec(5_000_000, null); - verifyCompactionIsEligibleBecause( - new CompactionState(null, null, null, null, null, null, null), - InlineSchemaDataSourceCompactionConfig - .builder() - .withTuningConfig(createTuningConfig(requiredPartitionsSpec, null)) - .forDataSource(TestDataSource.WIKI) - .build(), - "'partitionsSpec' mismatch: required['dynamic' with 5,000,000 rows], current[null]" - ); - } - - @Test - public void testStatusOnPartitionsSpecMismatch() - { - final PartitionsSpec requiredPartitionsSpec = new DynamicPartitionsSpec(5_000_000, null); - final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); - - final CompactionState lastCompactionState - = new CompactionState(currentPartitionsSpec, null, null, null, null, null, null); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .withTuningConfig(createTuningConfig(requiredPartitionsSpec, null)) - .forDataSource(TestDataSource.WIKI) - .build(); - - verifyCompactionIsEligibleBecause( - lastCompactionState, - compactionConfig, - "'partitionsSpec' mismatch: required['dynamic' with 5,000,000 rows]," - + " current['dynamic' with 100 rows]" - ); - } - - @Test - public void testStatusOnIndexSpecMismatch() - { - final IndexSpec currentIndexSpec - = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); - - final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); - final CompactionState lastCompactionState = new CompactionState( - currentPartitionsSpec, - null, - null, - null, - currentIndexSpec, - null, - null - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withTuningConfig(createTuningConfig(currentPartitionsSpec, null)) - .build(); - - verifyCompactionIsEligibleBecause( - lastCompactionState, - compactionConfig, - "'indexSpec' mismatch: " - + "required[IndexSpec{bitmapSerdeFactory=RoaringBitmapSerdeFactory{}," - + " metadataCompression=none," - + " dimensionCompression=lz4, stringDictionaryEncoding=Utf8{}," - + " metricCompression=lz4, longEncoding=longs, complexMetricCompression=null," - + " autoColumnFormatSpec=null, jsonCompression=null, segmentLoader=null}], " - + "current[IndexSpec{bitmapSerdeFactory=RoaringBitmapSerdeFactory{}," - + " metadataCompression=none," - + " dimensionCompression=zstd, stringDictionaryEncoding=Utf8{}," - + " metricCompression=lz4, longEncoding=longs, complexMetricCompression=null," - + " autoColumnFormatSpec=null, jsonCompression=null, segmentLoader=null}]" - ); - } - - @Test - public void testStatusOnSegmentGranularityMismatch() - { - final GranularitySpec currentGranularitySpec - = new UniformGranularitySpec(Granularities.HOUR, null, null); - - final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); - final IndexSpec currentIndexSpec - = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); - final CompactionState lastCompactionState = new CompactionState( - currentPartitionsSpec, - null, - null, - null, - currentIndexSpec, - currentGranularitySpec, - null - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec)) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) - .build(); - - verifyCompactionIsEligibleBecause( - lastCompactionState, - compactionConfig, - "'segmentGranularity' mismatch: required[DAY], current[HOUR]" - ); - } - - @Test - public void testStatusWhenLastCompactionStateSameAsRequired() - { - final GranularitySpec currentGranularitySpec - = new UniformGranularitySpec(Granularities.HOUR, null, null); - final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); - final IndexSpec currentIndexSpec - = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); - final CompactionState lastCompactionState = new CompactionState( - currentPartitionsSpec, - null, - null, - null, - currentIndexSpec, - currentGranularitySpec, - null - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec)) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) - .build(); - - final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); - final CompactionEligibility status = CompactionEligibility.evaluate( - CompactionCandidate.ProposedCompaction.from(List.of(segment), Granularities.HOUR), - compactionConfig, - new NewestSegmentFirstPolicy(null), - fingerprintMapper - ); - Assert.assertEquals(CompactionEligibility.State.NOT_APPLICABLE, status.getState()); - } - - @Test - public void testStatusWhenProjectionsMatch() - { - final GranularitySpec currentGranularitySpec - = new UniformGranularitySpec(Granularities.HOUR, null, null); - final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); - final IndexSpec currentIndexSpec - = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); - final AggregateProjectionSpec projection1 = - AggregateProjectionSpec.builder("foo") - .virtualColumns( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ) - .groupingColumns( - new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - new StringDimensionSchema("a") - ) - .aggregators( - new LongSumAggregatorFactory("sum_long", "long") - ) - .build(); - final CompactionState lastCompactionState = new CompactionState( - currentPartitionsSpec, - null, - null, - null, - currentIndexSpec, - currentGranularitySpec, - List.of(projection1) - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec)) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) - .withProjections(List.of(projection1)) - .build(); - - final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); - final CompactionEligibility status = CompactionEligibility.evaluate( - CompactionCandidate.ProposedCompaction.from(List.of(segment), Granularities.HOUR), - compactionConfig, - new NewestSegmentFirstPolicy(null), - fingerprintMapper - ); - Assert.assertEquals(CompactionEligibility.NOT_APPLICABLE, status); - } - - @Test - public void testStatusWhenProjectionsMismatch() - { - final GranularitySpec currentGranularitySpec - = new UniformGranularitySpec(Granularities.HOUR, null, null); - final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); - final IndexSpec currentIndexSpec - = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); - final AggregateProjectionSpec projection1 = - AggregateProjectionSpec.builder("1") - .virtualColumns( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ) - .groupingColumns( - new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - new StringDimensionSchema("a") - ) - .aggregators( - new LongSumAggregatorFactory("sum_long", "long") - ) - .build(); - final AggregateProjectionSpec projection2 = - AggregateProjectionSpec.builder("2") - .aggregators(new LongSumAggregatorFactory("sum_long", "long")) - .build(); - - final CompactionState lastCompactionState = new CompactionState( - currentPartitionsSpec, - null, - null, - null, - currentIndexSpec, - currentGranularitySpec, - List.of(projection1) - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec)) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) - .withProjections(List.of(projection1, projection2)) - .build(); - - final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); - final CompactionEligibility status = CompactionEligibility.evaluate( - CompactionCandidate.ProposedCompaction.from(List.of(segment), Granularities.HOUR), - compactionConfig, - new NewestSegmentFirstPolicy(null), - fingerprintMapper - ); - Assert.assertEquals(CompactionEligibility.State.FULL_COMPACTION, status.getState()); - Assert.assertTrue(status.getReason().contains("'projections' mismatch")); - } - - @Test - public void testStatusWhenAutoSchemaMatch() - { - final GranularitySpec currentGranularitySpec - = new UniformGranularitySpec(Granularities.HOUR, null, null); - final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); - - final CompactionState lastCompactionState = new CompactionState( - currentPartitionsSpec, - DimensionsSpec.builder() - .setDimensions( - List.of( - AutoTypeColumnSchema.of("x").getEffectiveSchema(IndexSpec.getDefault().getEffectiveSpec()), - AutoTypeColumnSchema.of("y").getEffectiveSchema(IndexSpec.getDefault().getEffectiveSpec()) - ) - ) - .build(), - null, - null, - IndexSpec.getDefault().getEffectiveSpec(), - currentGranularitySpec, - Collections.emptyList() - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withDimensionsSpec( - new UserCompactionTaskDimensionsConfig( - List.of( - new AutoTypeColumnSchema( - "x", - null, - NestedCommonFormatColumnFormatSpec.builder() - .setDoubleColumnCompression(CompressionStrategy.LZ4) - .build() - ), - AutoTypeColumnSchema.of("y") - ) - ) - ) - .withTuningConfig(createTuningConfig(currentPartitionsSpec, IndexSpec.getDefault())) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) - .withProjections(Collections.emptyList()) - .build(); - - final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); - final CompactionEligibility status = CompactionEligibility.evaluate( - CompactionCandidate.ProposedCompaction.from(List.of(segment), null), - compactionConfig, - new NewestSegmentFirstPolicy(null), - fingerprintMapper - ); - Assert.assertEquals(CompactionEligibility.NOT_APPLICABLE, status); - } - - @Test - public void testStatusWhenAutoSchemaMismatch() - { - final GranularitySpec currentGranularitySpec - = new UniformGranularitySpec(Granularities.HOUR, null, null); - final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); - - final CompactionState lastCompactionState = new CompactionState( - currentPartitionsSpec, - DimensionsSpec.builder() - .setDimensions( - List.of( - AutoTypeColumnSchema.of("x").getEffectiveSchema(IndexSpec.getDefault()), - AutoTypeColumnSchema.of("y").getEffectiveSchema(IndexSpec.getDefault()) - ) - ) - .build(), - null, - null, - IndexSpec.getDefault(), - currentGranularitySpec, - Collections.emptyList() - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withDimensionsSpec( - new UserCompactionTaskDimensionsConfig( - List.of( - new AutoTypeColumnSchema( - "x", - null, - NestedCommonFormatColumnFormatSpec.builder() - .setDoubleColumnCompression(CompressionStrategy.ZSTD) - .build() - ), - AutoTypeColumnSchema.of("y") - ) - ) - ) - .withTuningConfig(createTuningConfig(currentPartitionsSpec, IndexSpec.getDefault())) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) - .withProjections(Collections.emptyList()) - .build(); - - final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); - final CompactionEligibility status = CompactionEligibility.evaluate( - CompactionCandidate.ProposedCompaction.from(List.of(segment), null), - compactionConfig, - new NewestSegmentFirstPolicy(null), - fingerprintMapper - ); - Assert.assertEquals(CompactionEligibility.State.FULL_COMPACTION, status.getState()); - Assert.assertTrue(status.getReason().contains("'dimensionsSpec' mismatch")); - } - - @Test - public void test_evaluate_needsCompactionWhenAllSegmentsHaveUnexpectedIndexingStateFingerprint() - { - List segments = List.of( - DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint("wrongFingerprint").build(), - DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint("wrongFingerprint").build() - ); - - final DataSourceCompactionConfig oldCompactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) - .build(); - CompactionState wrongState = oldCompactionConfig.toCompactionState(); - - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) - .build(); - - indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, "wrongFingerprint", wrongState, DateTimes.nowUtc()); - syncCacheFromManager(); - - verifyEvaluationNeedsCompactionBecauseWithCustomSegments( - CompactionCandidate.ProposedCompaction.from(segments, null), - compactionConfig, - "'segmentGranularity' mismatch: required[DAY], current[HOUR]" - ); - } - - @Test - public void test_evaluate_needsCompactionWhenSomeSegmentsHaveUnexpectedIndexingStateFingerprint() - { - final DataSourceCompactionConfig oldCompactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) - .build(); - CompactionState wrongState = oldCompactionConfig.toCompactionState(); - - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) - .build(); - - CompactionState expectedState = compactionConfig.toCompactionState(); - - String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); - - List segments = List.of( - DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), - DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint("wrongFingerprint").build() - ); - - indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, expectedFingerprint, expectedState, DateTimes.nowUtc()); - indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, "wrongFingerprint", wrongState, DateTimes.nowUtc()); - syncCacheFromManager(); - - verifyEvaluationNeedsCompactionBecauseWithCustomSegments( - CompactionCandidate.ProposedCompaction.from(segments, null), - compactionConfig, - "'segmentGranularity' mismatch: required[DAY], current[HOUR]" - ); - } - - @Test - public void test_evaluate_noCompacationIfUnexpectedFingerprintHasExpectedIndexingState() - { - List segments = List.of( - DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint("wrongFingerprint").build() - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) - .build(); - - CompactionState expectedState = compactionConfig.toCompactionState(); - indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, "wrongFingerprint", expectedState, DateTimes.nowUtc()); - syncCacheFromManager(); - - final CompactionEligibility status = CompactionEligibility.evaluate( - CompactionCandidate.ProposedCompaction.from(segments, null), - compactionConfig, - new NewestSegmentFirstPolicy(null), - fingerprintMapper - ); - Assert.assertEquals(CompactionEligibility.NOT_APPLICABLE, status); - } - - @Test - public void test_evaluate_needsCompactionWhenUnexpectedFingerprintAndNoFingerprintInMetadataStore() - { - List segments = List.of( - DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint("wrongFingerprint").build() - ); - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) - .build(); - - verifyEvaluationNeedsCompactionBecauseWithCustomSegments( - CompactionCandidate.ProposedCompaction.from(segments, null), - compactionConfig, - "One or more fingerprinted segments do not have a cached indexing state" - ); - } - - @Test - public void test_evaluate_noCompactionWhenAllSegmentsHaveExpectedIndexingStateFingerprint() - { - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) - .build(); - - CompactionState expectedState = compactionConfig.toCompactionState(); - - String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); - - List segments = List.of( - DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), - DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint(expectedFingerprint).build() - ); - - final CompactionEligibility status = CompactionEligibility.evaluate( - CompactionCandidate.ProposedCompaction.from(segments, null), - compactionConfig, - new NewestSegmentFirstPolicy(null), - fingerprintMapper - ); - Assert.assertEquals(CompactionEligibility.NOT_APPLICABLE, status); - } - - @Test - public void test_evaluate_needsCompactionWhenNonFingerprintedSegmentsFailChecksOnLastCompactionState() - { - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) - .build(); - - CompactionState expectedState = compactionConfig.toCompactionState(); - String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); - - indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, expectedFingerprint, expectedState, DateTimes.nowUtc()); - syncCacheFromManager(); - - List segments = List.of( - DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), - DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint(null).lastCompactionState(createCompactionStateWithGranularity(Granularities.HOUR)).build() - ); - - - verifyEvaluationNeedsCompactionBecauseWithCustomSegments( - CompactionCandidate.ProposedCompaction.from(segments, null), - compactionConfig, - "'segmentGranularity' mismatch: required[DAY], current[HOUR]" - ); - } - - @Test - public void test_evaluate_noCompactionWhenNonFingerprintedSegmentsPassChecksOnLastCompactionState() - { - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) - .build(); - - CompactionState expectedState = compactionConfig.toCompactionState(); - - String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); - - List segments = List.of( - DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), - DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint(null).lastCompactionState(createCompactionStateWithGranularity(Granularities.DAY)).build() - ); - - final CompactionEligibility status = CompactionEligibility.evaluate( - CompactionCandidate.ProposedCompaction.from(segments, null), - compactionConfig, - new NewestSegmentFirstPolicy(null), - fingerprintMapper - ); - Assert.assertEquals(CompactionEligibility.NOT_APPLICABLE, status); - } - - // ============================ - // SKIPPED status tests - // ============================ - - @Test - public void test_evaluate_isSkippedWhenInputBytesExceedLimit() - { - // Two segments with 100MB each = 200MB total - // inputSegmentSizeBytes is 150MB, so should be skipped - final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig - .builder() - .forDataSource(TestDataSource.WIKI) - .withInputSegmentSizeBytes(150_000_000L) - .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) - .build(); - - final CompactionState lastCompactionState = createCompactionStateWithGranularity(Granularities.HOUR); - List segments = List.of( - DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(), - DataSegment.builder(WIKI_SEGMENT_2).lastCompactionState(lastCompactionState).build() - ); - - final CompactionEligibility status = CompactionEligibility.evaluate( - CompactionCandidate.ProposedCompaction.from(segments, null), - compactionConfig, - new NewestSegmentFirstPolicy(null), - fingerprintMapper - ); - - Assert.assertFalse(status.getState().equals(CompactionEligibility.State.FULL_COMPACTION)); - Assert.assertTrue(status.getReason().contains("'inputSegmentSize' exceeded")); - Assert.assertTrue(status.getReason().contains("200000000")); - Assert.assertTrue(status.getReason().contains("150000000")); - } - - /** - * Verify that the evaluation indicates compaction is needed for the expected reason. - * Allows customization of the segments in the compaction candidate. - */ - private void verifyEvaluationNeedsCompactionBecauseWithCustomSegments( - CompactionCandidate.ProposedCompaction proposedCompaction, - DataSourceCompactionConfig compactionConfig, - String expectedReason - ) - { - final CompactionEligibility status = CompactionEligibility.evaluate( - proposedCompaction, - compactionConfig, - new NewestSegmentFirstPolicy(null), - fingerprintMapper - ); - - Assert.assertEquals(CompactionEligibility.State.FULL_COMPACTION, status.getState()); - Assert.assertEquals(expectedReason, status.getReason()); - } - - private void verifyCompactionIsEligibleBecause( - CompactionState lastCompactionState, - DataSourceCompactionConfig compactionConfig, - String expectedReason - ) - { - final DataSegment segment - = DataSegment.builder(WIKI_SEGMENT) - .lastCompactionState(lastCompactionState) - .build(); - final CompactionEligibility status = CompactionEligibility.evaluate( - CompactionCandidate.ProposedCompaction.from(List.of(segment), null), - compactionConfig, - new NewestSegmentFirstPolicy(null), - fingerprintMapper - ); - - Assert.assertEquals(CompactionEligibility.State.FULL_COMPACTION, status.getState()); - Assert.assertEquals(expectedReason, status.getReason()); - } - - private static DataSourceCompactionConfig createCompactionConfig( - PartitionsSpec partitionsSpec - ) - { - return InlineSchemaDataSourceCompactionConfig.builder() - .forDataSource(TestDataSource.WIKI) - .withTuningConfig(createTuningConfig(partitionsSpec, null)) - .build(); - } - - private static UserCompactionTaskQueryTuningConfig createTuningConfig( - PartitionsSpec partitionsSpec, - IndexSpec indexSpec - ) - { - return new UserCompactionTaskQueryTuningConfig( - null, - null, null, null, null, partitionsSpec, indexSpec, null, null, - null, null, null, null, null, null, null, null, null, null - ); - } - - /** - * Simple helper to create a CompactionState with only segmentGranularity set - */ - private static CompactionState createCompactionStateWithGranularity(Granularity segmentGranularity) - { - return new CompactionState( - null, - null, - null, - null, - IndexSpec.getDefault(), - new UniformGranularitySpec(segmentGranularity, null, null, null), - null - ); - } -} diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java index afed6f00abcc..c97827e6491a 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java @@ -80,13 +80,13 @@ public void testSimulateClusterCompactionConfigUpdate() Assert.assertNotNull(simulateResult); - final Map compactionStates = simulateResult.getCompactionStates(); + final Map compactionStates = simulateResult.getCompactionStates(); Assert.assertNotNull(compactionStates); - Assert.assertNull(compactionStates.get(CompactionStatus.State.COMPLETE)); - Assert.assertNull(compactionStates.get(CompactionStatus.State.RUNNING)); + Assert.assertNull(compactionStates.get(CompactionCandidate.TaskState.RECENTLY_COMPLETED)); + Assert.assertNull(compactionStates.get(CompactionCandidate.TaskState.TASK_IN_PROGRESS)); - final Table queuedTable = compactionStates.get(CompactionStatus.State.PENDING); + final Table queuedTable = compactionStates.get(CompactionCandidate.TaskState.READY); Assert.assertEquals( Arrays.asList("dataSource", "interval", "numSegments", "bytes", "maxTaskSlots", "reasonToCompact"), queuedTable.getColumnNames() @@ -106,7 +106,7 @@ public void testSimulateClusterCompactionConfigUpdate() queuedTable.getRows() ); - final Table skippedTable = compactionStates.get(CompactionStatus.State.SKIPPED); + final Table skippedTable = simulateResult.getSkippedIntervals(); Assert.assertEquals( Arrays.asList("dataSource", "interval", "numSegments", "bytes", "reasonToSkip"), skippedTable.getColumnNames() @@ -153,13 +153,13 @@ public void testSimulate_withFixedIntervalOrderPolicy() Assert.assertNotNull(simulateResult); - final Map compactionStates = simulateResult.getCompactionStates(); + final Map compactionStates = simulateResult.getCompactionStates(); Assert.assertNotNull(compactionStates); - Assert.assertNull(compactionStates.get(CompactionStatus.State.COMPLETE)); - Assert.assertNull(compactionStates.get(CompactionStatus.State.RUNNING)); + Assert.assertNull(compactionStates.get(CompactionCandidate.TaskState.RECENTLY_COMPLETED)); + Assert.assertNull(compactionStates.get(CompactionCandidate.TaskState.TASK_IN_PROGRESS)); - final Table pendingTable = compactionStates.get(CompactionStatus.State.PENDING); + final Table pendingTable = compactionStates.get(CompactionCandidate.TaskState.READY); Assert.assertEquals( List.of("dataSource", "interval", "numSegments", "bytes", "maxTaskSlots", "reasonToCompact"), pendingTable.getColumnNames() @@ -172,7 +172,7 @@ public void testSimulate_withFixedIntervalOrderPolicy() pendingTable.getRows() ); - final Table skippedTable = compactionStates.get(CompactionStatus.State.SKIPPED); + final Table skippedTable = simulateResult.getSkippedIntervals(); Assert.assertEquals( List.of("dataSource", "interval", "numSegments", "bytes", "reasonToSkip"), skippedTable.getColumnNames() diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusBuilderTest.java similarity index 71% rename from server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityTest.java rename to server/src/test/java/org/apache/druid/server/compaction/CompactionStatusBuilderTest.java index 4aed19cb4668..576489596297 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionEligibilityTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusBuilderTest.java @@ -29,28 +29,16 @@ import java.util.Collections; import java.util.List; -public class CompactionEligibilityTest +public class CompactionStatusBuilderTest { private static final String DATASOURCE = "test_datasource"; @Test - public void testNotApplicable() + public void testNotEligible() { - CompactionEligibility eligibility = CompactionEligibility.NOT_APPLICABLE; + CompactionStatus eligibility = CompactionStatus.notEligible("test reason: %s", "failure"); - Assert.assertEquals(CompactionEligibility.State.NOT_APPLICABLE, eligibility.getState()); - Assert.assertEquals("", eligibility.getReason()); - Assert.assertNull(eligibility.getCompactedStats()); - Assert.assertNull(eligibility.getUncompactedStats()); - Assert.assertNull(eligibility.getUncompactedSegments()); - } - - @Test - public void testFail() - { - CompactionEligibility eligibility = CompactionEligibility.fail("test reason: %s", "failure"); - - Assert.assertEquals(CompactionEligibility.State.NOT_ELIGIBLE, eligibility.getState()); + Assert.assertEquals(CompactionStatus.State.NOT_ELIGIBLE, eligibility.getState()); Assert.assertEquals("test reason: failure", eligibility.getReason()); Assert.assertNull(eligibility.getCompactedStats()); Assert.assertNull(eligibility.getUncompactedStats()); @@ -64,14 +52,14 @@ public void testBuilderWithCompactionStats() CompactionStatistics uncompactedStats = CompactionStatistics.create(500, 3, 1); List uncompactedSegments = createTestSegments(3); - CompactionEligibility eligibility = - CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "needs full compaction") + CompactionStatus eligibility = + CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "needs full compaction") .compacted(compactedStats) .uncompacted(uncompactedStats) .uncompactedSegments(uncompactedSegments) .build(); - Assert.assertEquals(CompactionEligibility.State.FULL_COMPACTION, eligibility.getState()); + Assert.assertEquals(CompactionStatus.State.ELIGIBLE, eligibility.getState()); Assert.assertEquals("needs full compaction", eligibility.getReason()); Assert.assertEquals(compactedStats, eligibility.getCompactedStats()); Assert.assertEquals(uncompactedStats, eligibility.getUncompactedStats()); @@ -82,17 +70,17 @@ public void testBuilderWithCompactionStats() public void testEqualsAndHashCode() { // Test with simple eligibility objects (same state and reason) - CompactionEligibility simple1 = CompactionEligibility.fail("reason"); - CompactionEligibility simple2 = CompactionEligibility.fail("reason"); + CompactionStatus simple1 = CompactionStatus.notEligible("reason"); + CompactionStatus simple2 = CompactionStatus.notEligible("reason"); Assert.assertEquals(simple1, simple2); Assert.assertEquals(simple1.hashCode(), simple2.hashCode()); // Test with different reasons - CompactionEligibility differentReason = CompactionEligibility.fail("different"); + CompactionStatus differentReason = CompactionStatus.notEligible("different"); Assert.assertNotEquals(simple1, differentReason); // Test with different states - CompactionEligibility differentState = CompactionEligibility.NOT_APPLICABLE; + CompactionStatus differentState = CompactionStatus.COMPLETE; Assert.assertNotEquals(simple1, differentState); // Test with full compaction eligibility (with stats and segments) @@ -100,15 +88,15 @@ public void testEqualsAndHashCode() CompactionStatistics stats2 = CompactionStatistics.create(500, 3, 1); List segments = createTestSegments(3); - CompactionEligibility withStats1 = - CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "reason") + CompactionStatus withStats1 = + CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "reason") .compacted(stats1) .uncompacted(stats2) .uncompactedSegments(segments) .build(); - CompactionEligibility withStats2 = - CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "reason") + CompactionStatus withStats2 = + CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "reason") .compacted(stats1) .uncompacted(stats2) .uncompactedSegments(segments) @@ -120,8 +108,8 @@ public void testEqualsAndHashCode() // Test with different compacted stats CompactionStatistics differentStats = CompactionStatistics.create(2000, 10, 5); - CompactionEligibility differentCompactedStats = - CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "reason") + CompactionStatus differentCompactedStats = + CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "reason") .compacted(differentStats) .uncompacted(stats2) .uncompactedSegments(segments) @@ -129,8 +117,8 @@ public void testEqualsAndHashCode() Assert.assertNotEquals(withStats1, differentCompactedStats); // Test with different uncompacted stats - CompactionEligibility differentUncompactedStats = - CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "reason") + CompactionStatus differentUncompactedStats = + CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "reason") .compacted(stats1) .uncompacted(differentStats) .uncompactedSegments(segments) @@ -139,8 +127,8 @@ public void testEqualsAndHashCode() // Test with different segment lists List differentSegments = createTestSegments(5); - CompactionEligibility differentSegmentList = - CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "reason") + CompactionStatus differentSegmentList = + CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "reason") .compacted(stats1) .uncompacted(stats2) .uncompactedSegments(differentSegments) @@ -153,7 +141,7 @@ public void testBuilderRequiresReasonForNotEligible() { Assert.assertThrows( DruidException.class, - () -> CompactionEligibility.builder(CompactionEligibility.State.NOT_ELIGIBLE, null).build() + () -> CompactionStatus.builder(CompactionStatus.State.NOT_ELIGIBLE, null).build() ); } @@ -162,19 +150,19 @@ public void testBuilderRequiresStatsForFullCompaction() { Assert.assertThrows( DruidException.class, - () -> CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "reason").build() + () -> CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "reason").build() ); Assert.assertThrows( DruidException.class, - () -> CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "reason") + () -> CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "reason") .compacted(CompactionStatistics.create(1000, 5, 2)) .build() ); Assert.assertThrows( DruidException.class, - () -> CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "reason") + () -> CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "reason") .compacted(CompactionStatistics.create(1000, 5, 2)) .uncompacted(CompactionStatistics.create(500, 3, 1)) .build() diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java index 5f44f9968d9a..44ea2781859c 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java @@ -19,52 +19,866 @@ package org.apache.druid.server.compaction; +import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig; +import org.apache.druid.data.input.impl.AggregateProjectionSpec; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.LongDimensionSchema; +import org.apache.druid.data.input.impl.StringDimensionSchema; +import org.apache.druid.indexer.granularity.GranularitySpec; +import org.apache.druid.indexer.granularity.UniformGranularitySpec; +import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec; +import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; +import org.apache.druid.indexer.partitions.HashedPartitionsSpec; +import org.apache.druid.indexer.partitions.PartitionsSpec; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.granularity.Granularity; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.segment.AutoTypeColumnSchema; +import org.apache.druid.segment.IndexSpec; +import org.apache.druid.segment.TestDataSource; +import org.apache.druid.segment.data.CompressionStrategy; +import org.apache.druid.segment.metadata.DefaultIndexingStateFingerprintMapper; +import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage; +import org.apache.druid.segment.metadata.IndexingStateCache; +import org.apache.druid.segment.metadata.IndexingStateFingerprintMapper; +import org.apache.druid.segment.nested.NestedCommonFormatColumnFormatSpec; +import org.apache.druid.server.coordinator.DataSourceCompactionConfig; +import org.apache.druid.server.coordinator.InlineSchemaDataSourceCompactionConfig; +import org.apache.druid.server.coordinator.UserCompactionTaskDimensionsConfig; +import org.apache.druid.server.coordinator.UserCompactionTaskGranularityConfig; +import org.apache.druid.server.coordinator.UserCompactionTaskQueryTuningConfig; +import org.apache.druid.timeline.CompactionState; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import java.util.Collections; +import java.util.List; + public class CompactionStatusTest { + private static final DataSegment WIKI_SEGMENT + = DataSegment.builder(SegmentId.of(TestDataSource.WIKI, Intervals.of("2013-01-01/PT1H"), "v1", 0)) + .size(100_000_000L) + .build(); + private static final DataSegment WIKI_SEGMENT_2 + = DataSegment.builder(SegmentId.of(TestDataSource.WIKI, Intervals.of("2013-01-01/PT1H"), "v1", 1)) + .size(100_000_000L) + .build(); + + private HeapMemoryIndexingStateStorage indexingStateStorage; + private IndexingStateCache indexingStateCache; + private IndexingStateFingerprintMapper fingerprintMapper; + + @Before + public void setUp() + { + indexingStateStorage = new HeapMemoryIndexingStateStorage(); + indexingStateCache = new IndexingStateCache(); + fingerprintMapper = new DefaultIndexingStateFingerprintMapper( + indexingStateCache, + new DefaultObjectMapper() + ); + } + + /** + * Helper to sync the cache with states stored in the manager (for tests that persist states). + */ + private void syncCacheFromManager() + { + indexingStateCache.resetIndexingStatesForPublishedSegments(indexingStateStorage.getAllStoredStates()); + } + + @Test + public void testFindPartitionsSpecWhenGivenIsNull() + { + final ClientCompactionTaskQueryTuningConfig tuningConfig + = ClientCompactionTaskQueryTuningConfig.from(null); + Assert.assertNull(CompactionStatus.findPartitionsSpecFromConfig(tuningConfig)); + } + + @Test + public void testFindPartitionsSpecWhenGivenIsDynamicWithNullMaxTotalRows() + { + final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(null, null); + final ClientCompactionTaskQueryTuningConfig tuningConfig + = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); + Assert.assertEquals( + new DynamicPartitionsSpec(null, Long.MAX_VALUE), + CompactionStatus.findPartitionsSpecFromConfig(tuningConfig) + ); + } + @Test - public void testCompleteConstant() + public void testFindPartitionsSpecWhenGivenIsDynamicWithMaxTotalRows() { - CompactionStatus status = CompactionStatus.COMPLETE; + final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(null, 1000L); + final ClientCompactionTaskQueryTuningConfig tuningConfig + = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); + Assert.assertEquals( + partitionsSpec, + CompactionStatus.findPartitionsSpecFromConfig(tuningConfig) + ); + } + @Test + public void testFindPartitionsSpecWhenGivenIsDynamicWithMaxRowsPerSegment() + { + final PartitionsSpec partitionsSpec = new DynamicPartitionsSpec(100, 1000L); + final ClientCompactionTaskQueryTuningConfig tuningConfig + = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); + Assert.assertEquals( + partitionsSpec, + CompactionStatus.findPartitionsSpecFromConfig(tuningConfig) + ); + } + + @Test + public void testFindPartitionsSpecFromConfigWithDeprecatedMaxRowsPerSegmentAndMaxTotalRowsReturnGivenValues() + { + final DataSourceCompactionConfig config = + InlineSchemaDataSourceCompactionConfig.builder() + .forDataSource("datasource") + .withMaxRowsPerSegment(100) + .withTuningConfig( + new UserCompactionTaskQueryTuningConfig( + null, + null, + null, + 1000L, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ) + .build(); + Assert.assertEquals( + new DynamicPartitionsSpec(100, 1000L), + CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(config)) + ); + } + + @Test + public void testFindPartitionsSpecWhenGivenIsHashed() + { + final PartitionsSpec partitionsSpec = + new HashedPartitionsSpec(null, 100, Collections.singletonList("dim")); + final ClientCompactionTaskQueryTuningConfig tuningConfig + = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); + Assert.assertEquals( + partitionsSpec, + CompactionStatus.findPartitionsSpecFromConfig(tuningConfig) + ); + } + + @Test + public void testFindPartitionsSpecWhenGivenIsRangeWithMaxRows() + { + final PartitionsSpec partitionsSpec = + new DimensionRangePartitionsSpec(null, 10000, Collections.singletonList("dim"), false); + final ClientCompactionTaskQueryTuningConfig tuningConfig + = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); + Assert.assertEquals( + partitionsSpec, + CompactionStatus.findPartitionsSpecFromConfig(tuningConfig) + ); + } + + @Test + public void testFindPartitionsSpecWhenGivenIsRangeWithTargetRows() + { + final PartitionsSpec partitionsSpec = + new DimensionRangePartitionsSpec(10000, null, Collections.singletonList("dim"), false); + final ClientCompactionTaskQueryTuningConfig tuningConfig + = ClientCompactionTaskQueryTuningConfig.from(createCompactionConfig(partitionsSpec)); + Assert.assertEquals( + new DimensionRangePartitionsSpec(null, 15000, Collections.singletonList("dim"), false), + CompactionStatus.findPartitionsSpecFromConfig(tuningConfig) + ); + } + + @Test + public void testStatusWhenLastCompactionStateIsNull() + { + verifyCompactionIsEligibleBecause( + null, + InlineSchemaDataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build(), + "not compacted yet" + ); + } + + @Test + public void testStatusWhenLastCompactionStateIsEmpty() + { + final PartitionsSpec requiredPartitionsSpec = new DynamicPartitionsSpec(5_000_000, null); + verifyCompactionIsEligibleBecause( + new CompactionState(null, null, null, null, null, null, null), + InlineSchemaDataSourceCompactionConfig + .builder() + .withTuningConfig(createTuningConfig(requiredPartitionsSpec, null)) + .forDataSource(TestDataSource.WIKI) + .build(), + "'partitionsSpec' mismatch: required['dynamic' with 5,000,000 rows], current[null]" + ); + } + + @Test + public void testStatusOnPartitionsSpecMismatch() + { + final PartitionsSpec requiredPartitionsSpec = new DynamicPartitionsSpec(5_000_000, null); + final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); + + final CompactionState lastCompactionState + = new CompactionState(currentPartitionsSpec, null, null, null, null, null, null); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .withTuningConfig(createTuningConfig(requiredPartitionsSpec, null)) + .forDataSource(TestDataSource.WIKI) + .build(); + + verifyCompactionIsEligibleBecause( + lastCompactionState, + compactionConfig, + "'partitionsSpec' mismatch: required['dynamic' with 5,000,000 rows]," + + " current['dynamic' with 100 rows]" + ); + } + + @Test + public void testStatusOnIndexSpecMismatch() + { + final IndexSpec currentIndexSpec + = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); + + final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); + final CompactionState lastCompactionState = new CompactionState( + currentPartitionsSpec, + null, + null, + null, + currentIndexSpec, + null, + null + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withTuningConfig(createTuningConfig(currentPartitionsSpec, null)) + .build(); + + verifyCompactionIsEligibleBecause( + lastCompactionState, + compactionConfig, + "'indexSpec' mismatch: " + + "required[IndexSpec{bitmapSerdeFactory=RoaringBitmapSerdeFactory{}," + + " metadataCompression=none," + + " dimensionCompression=lz4, stringDictionaryEncoding=Utf8{}," + + " metricCompression=lz4, longEncoding=longs, complexMetricCompression=null," + + " autoColumnFormatSpec=null, jsonCompression=null, segmentLoader=null}], " + + "current[IndexSpec{bitmapSerdeFactory=RoaringBitmapSerdeFactory{}," + + " metadataCompression=none," + + " dimensionCompression=zstd, stringDictionaryEncoding=Utf8{}," + + " metricCompression=lz4, longEncoding=longs, complexMetricCompression=null," + + " autoColumnFormatSpec=null, jsonCompression=null, segmentLoader=null}]" + ); + } + + @Test + public void testStatusOnSegmentGranularityMismatch() + { + final GranularitySpec currentGranularitySpec + = new UniformGranularitySpec(Granularities.HOUR, null, null); + + final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); + final IndexSpec currentIndexSpec + = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); + final CompactionState lastCompactionState = new CompactionState( + currentPartitionsSpec, + null, + null, + null, + currentIndexSpec, + currentGranularitySpec, + null + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec)) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + verifyCompactionIsEligibleBecause( + lastCompactionState, + compactionConfig, + "'segmentGranularity' mismatch: required[DAY], current[HOUR]" + ); + } + + @Test + public void testStatusWhenLastCompactionStateSameAsRequired() + { + final GranularitySpec currentGranularitySpec + = new UniformGranularitySpec(Granularities.HOUR, null, null); + final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); + final IndexSpec currentIndexSpec + = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); + final CompactionState lastCompactionState = new CompactionState( + currentPartitionsSpec, + null, + null, + null, + currentIndexSpec, + currentGranularitySpec, + null + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec)) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .build(); + + final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); + final CompactionStatus status = CompactionStatus.evaluate( + CompactionCandidate.ProposedCompaction.from(List.of(segment), Granularities.HOUR), + compactionConfig, + fingerprintMapper + ); Assert.assertEquals(CompactionStatus.State.COMPLETE, status.getState()); - Assert.assertNull(status.getReason()); - Assert.assertTrue(status.isComplete()); - Assert.assertFalse(status.isSkipped()); } @Test - public void testPendingFactoryMethod() + public void testStatusWhenProjectionsMatch() + { + final GranularitySpec currentGranularitySpec + = new UniformGranularitySpec(Granularities.HOUR, null, null); + final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); + final IndexSpec currentIndexSpec + = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); + final AggregateProjectionSpec projection1 = + AggregateProjectionSpec.builder("foo") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns( + new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + new StringDimensionSchema("a") + ) + .aggregators( + new LongSumAggregatorFactory("sum_long", "long") + ) + .build(); + final CompactionState lastCompactionState = new CompactionState( + currentPartitionsSpec, + null, + null, + null, + currentIndexSpec, + currentGranularitySpec, + List.of(projection1) + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec)) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .withProjections(List.of(projection1)) + .build(); + + final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); + final CompactionStatus status = CompactionStatus.evaluate( + CompactionCandidate.ProposedCompaction.from(List.of(segment), Granularities.HOUR), + compactionConfig, + fingerprintMapper + ); + Assert.assertEquals(CompactionStatus.COMPLETE, status); + } + + @Test + public void testStatusWhenProjectionsMismatch() + { + final GranularitySpec currentGranularitySpec + = new UniformGranularitySpec(Granularities.HOUR, null, null); + final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); + final IndexSpec currentIndexSpec + = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); + final AggregateProjectionSpec projection1 = + AggregateProjectionSpec.builder("1") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns( + new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + new StringDimensionSchema("a") + ) + .aggregators( + new LongSumAggregatorFactory("sum_long", "long") + ) + .build(); + final AggregateProjectionSpec projection2 = + AggregateProjectionSpec.builder("2") + .aggregators(new LongSumAggregatorFactory("sum_long", "long")) + .build(); + + final CompactionState lastCompactionState = new CompactionState( + currentPartitionsSpec, + null, + null, + null, + currentIndexSpec, + currentGranularitySpec, + List.of(projection1) + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withTuningConfig(createTuningConfig(currentPartitionsSpec, currentIndexSpec)) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .withProjections(List.of(projection1, projection2)) + .build(); + + final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); + final CompactionStatus status = CompactionStatus.evaluate( + CompactionCandidate.ProposedCompaction.from(List.of(segment), Granularities.HOUR), + compactionConfig, + fingerprintMapper + ); + Assert.assertEquals(CompactionStatus.State.ELIGIBLE, status.getState()); + Assert.assertTrue(status.getReason().contains("'projections' mismatch")); + } + + @Test + public void testStatusWhenAutoSchemaMatch() + { + final GranularitySpec currentGranularitySpec + = new UniformGranularitySpec(Granularities.HOUR, null, null); + final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); + + final CompactionState lastCompactionState = new CompactionState( + currentPartitionsSpec, + DimensionsSpec.builder() + .setDimensions( + List.of( + AutoTypeColumnSchema.of("x").getEffectiveSchema(IndexSpec.getDefault().getEffectiveSpec()), + AutoTypeColumnSchema.of("y").getEffectiveSchema(IndexSpec.getDefault().getEffectiveSpec()) + ) + ) + .build(), + null, + null, + IndexSpec.getDefault().getEffectiveSpec(), + currentGranularitySpec, + Collections.emptyList() + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withDimensionsSpec( + new UserCompactionTaskDimensionsConfig( + List.of( + new AutoTypeColumnSchema( + "x", + null, + NestedCommonFormatColumnFormatSpec.builder() + .setDoubleColumnCompression(CompressionStrategy.LZ4) + .build() + ), + AutoTypeColumnSchema.of("y") + ) + ) + ) + .withTuningConfig(createTuningConfig(currentPartitionsSpec, IndexSpec.getDefault())) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .withProjections(Collections.emptyList()) + .build(); + + final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); + final CompactionStatus status = CompactionStatus.evaluate( + CompactionCandidate.ProposedCompaction.from(List.of(segment), null), + compactionConfig, + fingerprintMapper + ); + Assert.assertEquals(CompactionStatus.COMPLETE, status); + } + + @Test + public void testStatusWhenAutoSchemaMismatch() { - CompactionStatus status = CompactionStatus.pending("needs compaction: %d segments", 5); + final GranularitySpec currentGranularitySpec + = new UniformGranularitySpec(Granularities.HOUR, null, null); + final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, null); + + final CompactionState lastCompactionState = new CompactionState( + currentPartitionsSpec, + DimensionsSpec.builder() + .setDimensions( + List.of( + AutoTypeColumnSchema.of("x").getEffectiveSchema(IndexSpec.getDefault()), + AutoTypeColumnSchema.of("y").getEffectiveSchema(IndexSpec.getDefault()) + ) + ) + .build(), + null, + null, + IndexSpec.getDefault(), + currentGranularitySpec, + Collections.emptyList() + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withDimensionsSpec( + new UserCompactionTaskDimensionsConfig( + List.of( + new AutoTypeColumnSchema( + "x", + null, + NestedCommonFormatColumnFormatSpec.builder() + .setDoubleColumnCompression(CompressionStrategy.ZSTD) + .build() + ), + AutoTypeColumnSchema.of("y") + ) + ) + ) + .withTuningConfig(createTuningConfig(currentPartitionsSpec, IndexSpec.getDefault())) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .withProjections(Collections.emptyList()) + .build(); - Assert.assertEquals(CompactionStatus.State.PENDING, status.getState()); - Assert.assertEquals("needs compaction: 5 segments", status.getReason()); - Assert.assertFalse(status.isComplete()); - Assert.assertFalse(status.isSkipped()); + final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); + final CompactionStatus status = CompactionStatus.evaluate( + CompactionCandidate.ProposedCompaction.from(List.of(segment), null), + compactionConfig, + fingerprintMapper + ); + Assert.assertEquals(CompactionStatus.State.ELIGIBLE, status.getState()); + Assert.assertTrue(status.getReason().contains("'dimensionsSpec' mismatch")); } @Test - public void testSkippedFactoryMethod() + public void test_evaluate_needsCompactionWhenAllSegmentsHaveUnexpectedIndexingStateFingerprint() { - CompactionStatus status = CompactionStatus.skipped("already compacted"); + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint("wrongFingerprint").build(), + DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint("wrongFingerprint").build() + ); + + final DataSourceCompactionConfig oldCompactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .build(); + CompactionState wrongState = oldCompactionConfig.toCompactionState(); + + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); - Assert.assertEquals(CompactionStatus.State.SKIPPED, status.getState()); - Assert.assertEquals("already compacted", status.getReason()); - Assert.assertFalse(status.isComplete()); - Assert.assertTrue(status.isSkipped()); + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, "wrongFingerprint", wrongState, DateTimes.nowUtc()); + syncCacheFromManager(); + + verifyEvaluationNeedsCompactionBecauseWithCustomSegments( + CompactionCandidate.ProposedCompaction.from(segments, null), + compactionConfig, + "'segmentGranularity' mismatch: required[DAY], current[HOUR]" + ); } @Test - public void testRunningFactoryMethod() + public void test_evaluate_needsCompactionWhenSomeSegmentsHaveUnexpectedIndexingStateFingerprint() { - CompactionStatus status = CompactionStatus.running("task-123"); + final DataSourceCompactionConfig oldCompactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .build(); + CompactionState wrongState = oldCompactionConfig.toCompactionState(); + + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + CompactionState expectedState = compactionConfig.toCompactionState(); + + String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); + + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), + DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint("wrongFingerprint").build() + ); - Assert.assertEquals(CompactionStatus.State.RUNNING, status.getState()); - Assert.assertEquals("task-123", status.getReason()); - Assert.assertFalse(status.isComplete()); - Assert.assertFalse(status.isSkipped()); + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, expectedFingerprint, expectedState, DateTimes.nowUtc()); + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, "wrongFingerprint", wrongState, DateTimes.nowUtc()); + syncCacheFromManager(); + + verifyEvaluationNeedsCompactionBecauseWithCustomSegments( + CompactionCandidate.ProposedCompaction.from(segments, null), + compactionConfig, + "'segmentGranularity' mismatch: required[DAY], current[HOUR]" + ); + } + + @Test + public void test_evaluate_noCompacationIfUnexpectedFingerprintHasExpectedIndexingState() + { + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint("wrongFingerprint").build() + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null)) + .build(); + + CompactionState expectedState = compactionConfig.toCompactionState(); + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, "wrongFingerprint", expectedState, DateTimes.nowUtc()); + syncCacheFromManager(); + + final CompactionStatus status = CompactionStatus.evaluate( + CompactionCandidate.ProposedCompaction.from(segments, null), + compactionConfig, + fingerprintMapper + ); + Assert.assertEquals(CompactionStatus.COMPLETE, status); + } + + @Test + public void test_evaluate_needsCompactionWhenUnexpectedFingerprintAndNoFingerprintInMetadataStore() + { + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint("wrongFingerprint").build() + ); + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + verifyEvaluationNeedsCompactionBecauseWithCustomSegments( + CompactionCandidate.ProposedCompaction.from(segments, null), + compactionConfig, + "One or more fingerprinted segments do not have a cached indexing state" + ); + } + + @Test + public void test_evaluate_noCompactionWhenAllSegmentsHaveExpectedIndexingStateFingerprint() + { + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + CompactionState expectedState = compactionConfig.toCompactionState(); + + String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); + + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), + DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint(expectedFingerprint).build() + ); + + final CompactionStatus status = CompactionStatus.evaluate( + CompactionCandidate.ProposedCompaction.from(segments, null), + compactionConfig, + fingerprintMapper + ); + Assert.assertEquals(CompactionStatus.COMPLETE, status); + } + + @Test + public void test_evaluate_needsCompactionWhenNonFingerprintedSegmentsFailChecksOnLastCompactionState() + { + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + CompactionState expectedState = compactionConfig.toCompactionState(); + String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); + + indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, expectedFingerprint, expectedState, DateTimes.nowUtc()); + syncCacheFromManager(); + + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), + DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint(null).lastCompactionState(createCompactionStateWithGranularity(Granularities.HOUR)).build() + ); + + + verifyEvaluationNeedsCompactionBecauseWithCustomSegments( + CompactionCandidate.ProposedCompaction.from(segments, null), + compactionConfig, + "'segmentGranularity' mismatch: required[DAY], current[HOUR]" + ); + } + + @Test + public void test_evaluate_noCompactionWhenNonFingerprintedSegmentsPassChecksOnLastCompactionState() + { + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + CompactionState expectedState = compactionConfig.toCompactionState(); + + String expectedFingerprint = fingerprintMapper.generateFingerprint(TestDataSource.WIKI, expectedState); + + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).indexingStateFingerprint(expectedFingerprint).build(), + DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint(null).lastCompactionState(createCompactionStateWithGranularity(Granularities.DAY)).build() + ); + + final CompactionStatus status = CompactionStatus.evaluate( + CompactionCandidate.ProposedCompaction.from(segments, null), + compactionConfig, + fingerprintMapper + ); + Assert.assertEquals(CompactionStatus.COMPLETE, status); + } + + // ============================ + // SKIPPED status tests + // ============================ + + @Test + public void test_evaluate_isSkippedWhenInputBytesExceedLimit() + { + // Two segments with 100MB each = 200MB total + // inputSegmentSizeBytes is 150MB, so should be skipped + final DataSourceCompactionConfig compactionConfig = InlineSchemaDataSourceCompactionConfig + .builder() + .forDataSource(TestDataSource.WIKI) + .withInputSegmentSizeBytes(150_000_000L) + .withGranularitySpec(new UserCompactionTaskGranularityConfig(Granularities.DAY, null, null)) + .build(); + + final CompactionState lastCompactionState = createCompactionStateWithGranularity(Granularities.HOUR); + List segments = List.of( + DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(), + DataSegment.builder(WIKI_SEGMENT_2).lastCompactionState(lastCompactionState).build() + ); + + final CompactionStatus status = CompactionStatus.evaluate( + CompactionCandidate.ProposedCompaction.from(segments, null), + compactionConfig, + fingerprintMapper + ); + + Assert.assertFalse(status.getState().equals(CompactionStatus.State.ELIGIBLE)); + Assert.assertTrue(status.getReason().contains("'inputSegmentSize' exceeded")); + Assert.assertTrue(status.getReason().contains("200000000")); + Assert.assertTrue(status.getReason().contains("150000000")); + } + + /** + * Verify that the evaluation indicates compaction is needed for the expected reason. + * Allows customization of the segments in the compaction candidate. + */ + private void verifyEvaluationNeedsCompactionBecauseWithCustomSegments( + CompactionCandidate.ProposedCompaction proposedCompaction, + DataSourceCompactionConfig compactionConfig, + String expectedReason + ) + { + final CompactionStatus status = CompactionStatus.evaluate( + proposedCompaction, + compactionConfig, + fingerprintMapper + ); + + Assert.assertEquals(CompactionStatus.State.ELIGIBLE, status.getState()); + Assert.assertEquals(expectedReason, status.getReason()); + } + + private void verifyCompactionIsEligibleBecause( + CompactionState lastCompactionState, + DataSourceCompactionConfig compactionConfig, + String expectedReason + ) + { + final DataSegment segment + = DataSegment.builder(WIKI_SEGMENT) + .lastCompactionState(lastCompactionState) + .build(); + final CompactionStatus status = CompactionStatus.evaluate( + CompactionCandidate.ProposedCompaction.from(List.of(segment), null), + compactionConfig, + fingerprintMapper + ); + + Assert.assertEquals(CompactionStatus.State.ELIGIBLE, status.getState()); + Assert.assertEquals(expectedReason, status.getReason()); + } + + private static DataSourceCompactionConfig createCompactionConfig( + PartitionsSpec partitionsSpec + ) + { + return InlineSchemaDataSourceCompactionConfig.builder() + .forDataSource(TestDataSource.WIKI) + .withTuningConfig(createTuningConfig(partitionsSpec, null)) + .build(); + } + + private static UserCompactionTaskQueryTuningConfig createTuningConfig( + PartitionsSpec partitionsSpec, + IndexSpec indexSpec + ) + { + return new UserCompactionTaskQueryTuningConfig( + null, + null, null, null, null, partitionsSpec, indexSpec, null, null, + null, null, null, null, null, null, null, null, null, null + ); + } + + /** + * Simple helper to create a CompactionState with only segmentGranularity set + */ + private static CompactionState createCompactionStateWithGranularity(Granularity segmentGranularity) + { + return new CompactionState( + null, + null, + null, + null, + IndexSpec.getDefault(), + new UniformGranularitySpec(segmentGranularity, null, null, null), + null + ); } } diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java index e08fb2c9184f..855ea57d2753 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java @@ -100,36 +100,30 @@ public void testGetLatestTaskStatusForRepeatedlyFailingTask() } @Test - public void testComputeCompactionStatusForSuccessfulTask() + public void testComputeCompactionTaskStateForSuccessfulTask() { final NewestSegmentFirstPolicy policy = new NewestSegmentFirstPolicy(null); final CompactionCandidate candidateSegments = createCandidate(List.of(WIKI_SEGMENT), null); // Verify that interval is originally eligible for compaction - CompactionStatus status - = statusTracker.computeCompactionStatus(candidateSegments); - Assert.assertEquals(CompactionStatus.State.PENDING, status.getState()); - Assert.assertEquals("Not compacted yet", status.getReason()); + CompactionCandidate.TaskState status = statusTracker.computeCompactionTaskState(candidateSegments); + Assert.assertEquals(CompactionCandidate.TaskState.READY, status); // Verify that interval is skipped for compaction after task has finished statusTracker.onSegmentTimelineUpdated(DateTimes.nowUtc().minusMinutes(1)); statusTracker.onTaskSubmitted("task1", candidateSegments); statusTracker.onTaskFinished("task1", TaskStatus.success("task1")); - status = statusTracker.computeCompactionStatus(candidateSegments); - Assert.assertEquals(CompactionStatus.State.SKIPPED, status.getState()); - Assert.assertEquals( - "Segment timeline not updated since last compaction task succeeded", - status.getReason() - ); + status = statusTracker.computeCompactionTaskState(candidateSegments); + Assert.assertEquals(CompactionCandidate.TaskState.RECENTLY_COMPLETED, status); // Verify that interval becomes eligible again after timeline has been updated statusTracker.onSegmentTimelineUpdated(DateTimes.nowUtc()); - status = statusTracker.computeCompactionStatus(candidateSegments); - Assert.assertEquals(CompactionStatus.State.PENDING, status.getState()); + status = statusTracker.computeCompactionTaskState(candidateSegments); + Assert.assertEquals(CompactionCandidate.TaskState.READY, status); } - public static CompactionCandidate createCandidate( + private static CompactionCandidate createCandidate( List segments, @Nullable Granularity targetSegmentGranularity ) @@ -138,10 +132,11 @@ public static CompactionCandidate createCandidate( segments, targetSegmentGranularity ); - return CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "approve without check") - .compacted(CompactionStatistics.create(1, 1, 1)) - .uncompacted(CompactionStatistics.create(1, 1, 1)) - .uncompactedSegments(List.of()).build() - .createCandidate(proposedCompaction); + CompactionStatus status = CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "approve without check") + .compacted(CompactionStatistics.create(1, 1, 1)) + .uncompacted(CompactionStatistics.create(1, 1, 1)) + .uncompactedSegments(List.of()) + .build(); + return CompactionMode.FULL_COMPACTION.createCandidate(proposedCompaction, status); } } diff --git a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java index 869eabb1662b..5fb2359252c1 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java @@ -22,8 +22,8 @@ import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.HumanReadableBytes; -import org.apache.druid.segment.TestDataSource; -import org.apache.druid.server.coordinator.CreateDataSegments; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.segment.TestSegmentUtils; import org.apache.druid.timeline.DataSegment; import org.junit.Test; import org.junit.jupiter.api.Assertions; @@ -34,9 +34,11 @@ public class MostFragmentedIntervalFirstPolicyTest { private static final DataSegment SEGMENT = - CreateDataSegments.ofDatasource(TestDataSource.WIKI).eachOfSizeInMb(100).get(0); + TestSegmentUtils.makeSegment("foo", "1", Intervals.ETERNITY); + private static final DataSegment SEGMENT2 = + TestSegmentUtils.makeSegment("foo", "2", Intervals.ETERNITY); private static final CompactionCandidate.ProposedCompaction PROPOSED_COMPACTION = - CompactionCandidate.ProposedCompaction.from(List.of(SEGMENT), null); + CompactionCandidate.ProposedCompaction.from(List.of(SEGMENT, SEGMENT2), null); private static final CompactionStatistics DUMMY_COMPACTION_STATS = CompactionStatistics.create(1L, 1L, 1L); @@ -52,7 +54,7 @@ public void test_thresholdValues_ofDefaultPolicy() } @Test - public void test_checkEligibilityForCompaction_fails_ifUncompactedCountLessThanCutoff() + public void test_createCandidate_fails_ifUncompactedCountLessThanCutoff() { final int minUncompactedCount = 10_000; final MostFragmentedIntervalFirstPolicy policy = new MostFragmentedIntervalFirstPolicy( @@ -63,20 +65,23 @@ public void test_checkEligibilityForCompaction_fails_ifUncompactedCountLessThanC null ); - final CompactionEligibility eligibility1 = + final CompactionStatus eligibility1 = eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(1, 100L)).build(); + final CompactionCandidate candidate1 = policy.createCandidate(PROPOSED_COMPACTION, eligibility1); Assertions.assertEquals( - CompactionEligibility.fail("Uncompacted segments[1] in interval must be at least [10,000]"), - policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility1) + "Uncompacted segments[1] in interval must be at least [10,000]", + candidate1.getPolicyNote() ); + Assertions.assertEquals(CompactionMode.NOT_APPLICABLE, candidate1.getMode()); - final CompactionEligibility eligibility2 = + final CompactionStatus eligibility2 = eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(10_001, 100L)).build(); - Assertions.assertEquals(eligibility2, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility2)); + final CompactionCandidate candidate2 = policy.createCandidate(PROPOSED_COMPACTION, eligibility2); + Assertions.assertEquals(CompactionMode.FULL_COMPACTION, candidate2.getMode()); } @Test - public void test_checkEligibilityForCompaction_fails_ifUncompactedBytesLessThanCutoff() + public void test_createCandidate_fails_ifUncompactedBytesLessThanCutoff() { final HumanReadableBytes minUncompactedBytes = HumanReadableBytes.valueOf(10_000); final MostFragmentedIntervalFirstPolicy policy = new MostFragmentedIntervalFirstPolicy( @@ -87,20 +92,20 @@ public void test_checkEligibilityForCompaction_fails_ifUncompactedBytesLessThanC null ); - final CompactionEligibility eligibility1 = + final CompactionStatus eligibility1 = eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(1, 100L)).build(); - Assertions.assertEquals( - CompactionEligibility.fail("Uncompacted bytes[100] in interval must be at least [10,000]"), - policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility1) - ); + final CompactionCandidate candidate1 = policy.createCandidate(PROPOSED_COMPACTION, eligibility1); + Assertions.assertEquals("Uncompacted bytes[100] in interval must be at least [10,000]", candidate1.getPolicyNote()); + Assertions.assertEquals(CompactionMode.NOT_APPLICABLE, candidate1.getMode()); - final CompactionEligibility eligibility2 = + final CompactionStatus eligibility2 = eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(100, 10_000L)).build(); - Assertions.assertEquals(eligibility2, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility2)); + final CompactionCandidate candidate2 = policy.createCandidate(PROPOSED_COMPACTION, eligibility2); + Assertions.assertEquals(CompactionMode.FULL_COMPACTION, candidate2.getMode()); } @Test - public void test_checkEligibilityForCompaction_fails_ifAvgSegmentSizeGreaterThanCutoff() + public void test_createCandidate_fails_ifAvgSegmentSizeGreaterThanCutoff() { final HumanReadableBytes maxAvgSegmentSize = HumanReadableBytes.valueOf(100); final MostFragmentedIntervalFirstPolicy policy = new MostFragmentedIntervalFirstPolicy( @@ -111,15 +116,18 @@ public void test_checkEligibilityForCompaction_fails_ifAvgSegmentSizeGreaterThan null ); - final CompactionEligibility eligibility1 = + final CompactionStatus eligibility1 = eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(1, 10_000L)).build(); + final CompactionCandidate candidate1 = policy.createCandidate(PROPOSED_COMPACTION, eligibility1); Assertions.assertEquals( - CompactionEligibility.fail("Average size[10,000] of uncompacted segments in interval must be at most [100]"), - policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility1) + "Average size[10,000] of uncompacted segments in interval must be at most [100]", + candidate1.getPolicyNote() ); - final CompactionEligibility eligibility2 = + Assertions.assertEquals(CompactionMode.NOT_APPLICABLE, candidate1.getMode()); + final CompactionStatus eligibility2 = eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(1, 100L)).build(); - Assertions.assertEquals(eligibility2, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility2)); + final CompactionCandidate candidate2 = policy.createCandidate(PROPOSED_COMPACTION, eligibility2); + Assertions.assertEquals(CompactionMode.FULL_COMPACTION, candidate2.getMode()); } @Test @@ -133,16 +141,13 @@ public void test_policy_favorsIntervalWithMoreUncompactedSegments_ifTotalBytesIs null ); - final CompactionEligibility eligibility1 = + final CompactionStatus eligibility1 = eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(1, 1_000L)).build(); - final CompactionEligibility eligibility2 = + final CompactionStatus eligibility2 = eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(2, 500L)).build(); - Assertions.assertEquals(eligibility1, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility1)); - Assertions.assertEquals(eligibility2, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility2)); - - final CompactionCandidate candidateA = eligibility1.createCandidate(PROPOSED_COMPACTION); - final CompactionCandidate candidateB = eligibility2.createCandidate(PROPOSED_COMPACTION); + final CompactionCandidate candidateA = policy.createCandidate(PROPOSED_COMPACTION, eligibility1); + final CompactionCandidate candidateB = policy.createCandidate(PROPOSED_COMPACTION, eligibility2); Assertions.assertTrue(policy.compareCandidates(candidateA, candidateB) > 0); Assertions.assertTrue(policy.compareCandidates(candidateB, candidateA) < 0); } @@ -158,17 +163,13 @@ public void test_policy_favorsIntervalWithMoreUncompactedSegments_ifAverageSizeI null ); - final CompactionEligibility eligibility1 = + final CompactionStatus eligibility1 = eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(1, 1000L)).build(); - final CompactionEligibility eligibility2 = + final CompactionStatus eligibility2 = eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(2, 1000L)).build(); - - Assertions.assertEquals(eligibility1, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility1)); - Assertions.assertEquals(eligibility2, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility2)); - - final CompactionCandidate candidateA = eligibility1.createCandidate(PROPOSED_COMPACTION); - final CompactionCandidate candidateB = eligibility2.createCandidate(PROPOSED_COMPACTION); + final CompactionCandidate candidateA = policy.createCandidate(PROPOSED_COMPACTION, eligibility1); + final CompactionCandidate candidateB = policy.createCandidate(PROPOSED_COMPACTION, eligibility2); Assertions.assertTrue(policy.compareCandidates(candidateA, candidateB) > 0); Assertions.assertTrue(policy.compareCandidates(candidateB, candidateA) < 0); } @@ -184,16 +185,13 @@ public void test_policy_favorsIntervalWithSmallerSegments_ifCountIsEqual() null ); - final CompactionEligibility eligibility1 = + final CompactionStatus eligibility1 = eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(10, 500L)).build(); - final CompactionEligibility eligibility2 = + final CompactionStatus eligibility2 = eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(10, 1000L)).build(); - Assertions.assertEquals(eligibility1, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility1)); - Assertions.assertEquals(eligibility2, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility2)); - - final CompactionCandidate candidateA = eligibility1.createCandidate(PROPOSED_COMPACTION); - final CompactionCandidate candidateB = eligibility2.createCandidate(PROPOSED_COMPACTION); + final CompactionCandidate candidateA = policy.createCandidate(PROPOSED_COMPACTION, eligibility1); + final CompactionCandidate candidateB = policy.createCandidate(PROPOSED_COMPACTION, eligibility2); Assertions.assertTrue(policy.compareCandidates(candidateA, candidateB) < 0); Assertions.assertTrue(policy.compareCandidates(candidateB, candidateA) > 0); } @@ -209,16 +207,13 @@ public void test_compareCandidates_returnsZeroIfSegmentCountAndAvgSizeScaleEquiv null ); - final CompactionEligibility eligibility1 = + final CompactionStatus eligibility1 = eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(100, 25)).build(); - final CompactionEligibility eligibility2 = + final CompactionStatus eligibility2 = eligibilityBuilder().compacted(DUMMY_COMPACTION_STATS).uncompacted(createStats(400, 100)).build(); - Assertions.assertEquals(eligibility1, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility1)); - Assertions.assertEquals(eligibility2, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility2)); - - final CompactionCandidate candidateA = eligibility1.createCandidate(PROPOSED_COMPACTION); - final CompactionCandidate candidateB = eligibility2.createCandidate(PROPOSED_COMPACTION); + final CompactionCandidate candidateA = policy.createCandidate(PROPOSED_COMPACTION, eligibility1); + final CompactionCandidate candidateB = policy.createCandidate(PROPOSED_COMPACTION, eligibility2); Assertions.assertEquals(0, policy.compareCandidates(candidateA, candidateB)); Assertions.assertEquals(0, policy.compareCandidates(candidateB, candidateA)); } @@ -260,7 +255,7 @@ public void test_serde_noFieldsSet() throws IOException } @Test - public void test_checkEligibilityForCompaction_returnsIncrementalCompaction_whenRatioBelowThreshold() + public void test_createCandidate_returnsIncrementalCompaction_whenRatioBelowThreshold() { // Set threshold to 0.5 (50%) final MostFragmentedIntervalFirstPolicy policy = new MostFragmentedIntervalFirstPolicy( @@ -273,27 +268,23 @@ public void test_checkEligibilityForCompaction_returnsIncrementalCompaction_when final CompactionStatistics compacted = CompactionStatistics.create(1200L, 10, 1L); final CompactionStatistics uncompacted = CompactionStatistics.create(400L, 100, 1L); - final CompactionEligibility candidate = eligibilityBuilder() + final CompactionStatus eligibility = eligibilityBuilder() .compacted(compacted) .uncompacted(uncompacted) .uncompactedSegments(List.of(SEGMENT)) .build(); + final CompactionCandidate candidate = policy.createCandidate(PROPOSED_COMPACTION, eligibility); + Assertions.assertEquals("Uncompacted bytes ratio[0.25] is below threshold[0.50]", candidate.getPolicyNote()); + Assertions.assertEquals(CompactionMode.INCREMENTAL_COMPACTION, candidate.getMode()); Assertions.assertEquals( - CompactionEligibility.builder( - CompactionEligibility.State.INCREMENTAL_COMPACTION, - "Uncompacted bytes ratio[0.25] is below threshold[0.50]" - ) - .compacted(compacted) - .uncompacted(uncompacted) - .uncompactedSegments(List.of(SEGMENT)) - .build(), - policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, candidate) + CompactionCandidate.ProposedCompaction.from(List.of(SEGMENT), null), + candidate.getProposedCompaction() ); } @Test - public void test_checkEligibilityForCompaction_returnsFullCompaction_whenRatioAboveThreshold() + public void test_createCandidate_returnsFullCompaction_whenRatioAboveThreshold() { // Set threshold to 0.5 (50%) final MostFragmentedIntervalFirstPolicy policy = new MostFragmentedIntervalFirstPolicy( @@ -304,20 +295,18 @@ public void test_checkEligibilityForCompaction_returnsFullCompaction_whenRatioAb null ); - final CompactionEligibility eligibility = + final CompactionStatus eligibility = eligibilityBuilder() .compacted(CompactionStatistics.create(500L, 5, 1)) .uncompacted(CompactionStatistics.create(600L, 100, 1)) .build(); + final CompactionCandidate candidate = policy.createCandidate(PROPOSED_COMPACTION, eligibility); - Assertions.assertEquals( - eligibility, - policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility) - ); + Assertions.assertEquals(CompactionMode.FULL_COMPACTION, candidate.getMode()); } @Test - public void test_checkEligibilityForCompaction_returnsFullCompaction_whenThresholdIsDefault() + public void test_createCandidate_returnsFullCompaction_whenThresholdIsDefault() { // Default threshold is 0.0 final MostFragmentedIntervalFirstPolicy policy = new MostFragmentedIntervalFirstPolicy( @@ -329,13 +318,14 @@ public void test_checkEligibilityForCompaction_returnsFullCompaction_whenThresho ); // With default threshold 0.0, any positive ratio >= 0.0, so always FULL_COMPACTION_ELIGIBLE - final CompactionEligibility eligibility = + final CompactionStatus eligibility = eligibilityBuilder() .compacted(CompactionStatistics.create(1_000L, 10, 1)) .uncompacted(CompactionStatistics.create(100L, 100, 1)) .build(); + final CompactionCandidate candidate = policy.createCandidate(PROPOSED_COMPACTION, eligibility); - Assertions.assertEquals(eligibility, policy.checkEligibilityForCompaction(PROPOSED_COMPACTION, eligibility)); + Assertions.assertEquals(CompactionMode.FULL_COMPACTION, candidate.getMode()); } private CompactionStatistics createStats(int numSegments, long avgSizeBytes) @@ -343,9 +333,9 @@ private CompactionStatistics createStats(int numSegments, long avgSizeBytes) return CompactionStatistics.create(avgSizeBytes * numSegments, numSegments, 1L); } - private static CompactionEligibility.CompactionEligibilityBuilder eligibilityBuilder() + private static CompactionStatus.CompactionStatusBuilder eligibilityBuilder() { - return CompactionEligibility.builder(CompactionEligibility.State.FULL_COMPACTION, "approve") - .uncompactedSegments(List.of()); + return CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "approve") + .uncompactedSegments(List.of()); } } diff --git a/server/src/test/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicyTest.java b/server/src/test/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicyTest.java index 91309f56a409..4e86abcd2468 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicyTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/NewestSegmentFirstPolicyTest.java @@ -740,7 +740,7 @@ public void testIteratorReturnsNothingAsSegmentsWasCompactedAndHaveSameSegmentGr // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY @@ -769,7 +769,7 @@ public void testIteratorReturnsNothingAsSegmentsWasCompactedAndHaveSameSegmentGr // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY @@ -805,7 +805,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentSeg // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY @@ -853,7 +853,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentSeg // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY @@ -902,7 +902,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentTim // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY @@ -952,7 +952,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentOri // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY @@ -1001,7 +1001,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentRol // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have @@ -1090,7 +1090,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentQue // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have @@ -1185,7 +1185,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentDim // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have @@ -1414,7 +1414,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentFil // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have @@ -1536,7 +1536,7 @@ public void testIteratorReturnsSegmentsAsSegmentsWasCompactedAndHaveDifferentMet // Same indexSpec as what is set in the auto compaction config IndexSpec indexSpec = IndexSpec.getDefault(); // Same partitionsSpec as what is set in the auto compaction config - PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have @@ -1688,7 +1688,7 @@ public void testIteratorReturnsSegmentsAsCompactionStateChangedWithCompactedStat { // Different indexSpec as what is set in the auto compaction config IndexSpec newIndexSpec = IndexSpec.builder().withBitmapSerdeFactory(new ConciseBitmapSerdeFactory()).build(); - PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY @@ -1734,7 +1734,7 @@ public void testIteratorReturnsSegmentsAsCompactionStateChangedWithCompactedStat @Test public void testIteratorDoesNotReturnSegmentWithChangingAppendableIndexSpec() { - PartitionsSpec partitionsSpec = CompactionEligibility.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( + PartitionsSpec partitionsSpec = CompactionStatus.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from( null)); final SegmentTimeline timeline = createTimeline( createSegments() @@ -2066,13 +2066,13 @@ TestDataSource.KOALA, configBuilder().forDataSource(TestDataSource.KOALA).build( // Verify that the segments of WIKI are preferred even though they are older Assert.assertTrue(iterator.hasNext()); CompactionCandidate next = iterator.next(); - Assert.assertEquals(TestDataSource.WIKI, next.getProposedCompaction().getDataSource()); - Assert.assertEquals(Intervals.of("2012-01-01/P1D"), next.getProposedCompaction().getUmbrellaInterval()); + Assert.assertEquals(TestDataSource.WIKI, next.getDataSource()); + Assert.assertEquals(Intervals.of("2012-01-01/P1D"), next.getUmbrellaInterval()); Assert.assertTrue(iterator.hasNext()); next = iterator.next(); - Assert.assertEquals(TestDataSource.KOALA, next.getProposedCompaction().getDataSource()); - Assert.assertEquals(Intervals.of("2013-01-01/P1D"), next.getProposedCompaction().getUmbrellaInterval()); + Assert.assertEquals(TestDataSource.KOALA, next.getDataSource()); + Assert.assertEquals(Intervals.of("2013-01-01/P1D"), next.getUmbrellaInterval()); } private CompactionSegmentIterator createIterator(DataSourceCompactionConfig config, SegmentTimeline timeline) From aa6a5edf3befa14a3fd8c996ec46b12b49243245 Mon Sep 17 00:00:00 2001 From: cecemei Date: Sat, 7 Feb 2026 03:19:17 -0800 Subject: [PATCH 14/19] format --- .../server/compaction/CompactionMode.java | 18 ++++++++++----- .../server/compaction/CompactionStatus.java | 18 ++++++++++----- .../DataSourceCompactibleSegmentIterator.java | 20 ++++++++++++----- .../compaction/CompactionStatusTest.java | 22 +++++++++---------- 4 files changed, 51 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionMode.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionMode.java index 966b72d385a7..e89a5b2a7329 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionMode.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionMode.java @@ -52,10 +52,10 @@ public CompactionCandidate createCandidate( proposedCompaction.getUmbrellaInterval(), proposedCompaction.getCompactionInterval(), Math.toIntExact(eligibility.getUncompactedSegments() - .stream() - .map(DataSegment::getInterval) - .distinct() - .count()) + .stream() + .map(DataSegment::getInterval) + .distinct() + .count()) ); return new CompactionCandidate(newProposed, eligibility, policyNote, this); } @@ -94,7 +94,10 @@ public static CompactionCandidate failWithPolicyCheck( ); } - public static CompactionCandidate notEligible(CompactionCandidate.ProposedCompaction proposedCompaction, String reason) + public static CompactionCandidate notEligible( + CompactionCandidate.ProposedCompaction proposedCompaction, + String reason + ) { // CompactionStatus returns an ineligible reason, have not even got to policy check yet return new CompactionCandidate( @@ -104,4 +107,9 @@ public static CompactionCandidate notEligible(CompactionCandidate.ProposedCompac CompactionMode.NOT_APPLICABLE ); } + + public static CompactionCandidate complete(CompactionCandidate.ProposedCompaction proposedCompaction) + { + return new CompactionCandidate(proposedCompaction, CompactionStatus.COMPLETE, null, CompactionMode.NOT_APPLICABLE); + } } diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java index be6380445058..4b9f6a9efdf3 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java @@ -121,12 +121,18 @@ private CompactionStatus( case COMPLETE: break; case NOT_ELIGIBLE: - InvalidInput.conditionalException(!Strings.isNullOrEmpty(reason), "must provide a reason"); + InvalidInput.conditionalException( + !Strings.isNullOrEmpty(reason), + "must provide a reason why compaction not eligible" + ); break; case ELIGIBLE: - InvalidInput.conditionalException(compacted != null, "must provide compacted stats"); - InvalidInput.conditionalException(uncompacted != null, "must provide uncompacted stats"); - InvalidInput.conditionalException(uncompactedSegments != null, "must provide uncompactedSegments"); + InvalidInput.conditionalException(compacted != null, "must provide compacted stats for compaction"); + InvalidInput.conditionalException(uncompacted != null, "must provide uncompacted stats for compaction"); + InvalidInput.conditionalException( + uncompactedSegments != null, + "must provide uncompactedSegments for compaction" + ); break; default: throw DruidException.defensive("unexpected compaction status state[%s]", state); @@ -186,7 +192,7 @@ public List getUncompactedSegments() * @return a new {@link CompactionCandidate} with updated eligibility and status. For incremental * compaction, returns a candidate containing only the uncompacted segments. */ - public static CompactionStatus evaluate( + public static CompactionStatus compute( CompactionCandidate.ProposedCompaction proposedCompaction, DataSourceCompactionConfig config, IndexingStateFingerprintMapper fingerprintMapper @@ -366,7 +372,7 @@ static DimensionRangePartitionsSpec getEffectiveRangePartitionsSpec(DimensionRan } /** - * Evaluates checks to determine the compaction status of a + * Evaluates {@link #CHECKS} to determine the compaction status of a * {@link CompactionCandidate}. */ private static class Evaluator diff --git a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java index e316925f5d21..26ae52b23eda 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java +++ b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java @@ -337,11 +337,21 @@ private void findAndEnqueueSegmentsToCompact(CompactibleSegmentIterator compacti CompactionCandidate.ProposedCompaction proposed = CompactionCandidate.ProposedCompaction.from(segments, config.getSegmentGranularity()); - final CompactionStatus eligibility = CompactionStatus.evaluate(proposed, config, fingerprintMapper); - final CompactionCandidate candidate = - CompactionStatus.State.ELIGIBLE.equals(eligibility.getState()) - ? searchPolicy.createCandidate(proposed, eligibility) - : CompactionMode.notEligible(proposed, eligibility.getReason()); + final CompactionStatus eligibility = CompactionStatus.compute(proposed, config, fingerprintMapper); + final CompactionCandidate candidate; + switch (eligibility.getState()) { + case COMPLETE: + candidate = CompactionMode.complete(proposed); + break; + case NOT_ELIGIBLE: + candidate = CompactionMode.notEligible(proposed, eligibility.getReason()); + break; + case ELIGIBLE: + candidate = searchPolicy.createCandidate(proposed, eligibility); + break; + default: + throw DruidException.defensive("unknown compaction state[%s]", eligibility.getState()); + } switch (candidate.getMode()) { case INCREMENTAL_COMPACTION: diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java index 44ea2781859c..52ddfa44d526 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java @@ -357,7 +357,7 @@ public void testStatusWhenLastCompactionStateSameAsRequired() .build(); final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); - final CompactionStatus status = CompactionStatus.evaluate( + final CompactionStatus status = CompactionStatus.compute( CompactionCandidate.ProposedCompaction.from(List.of(segment), Granularities.HOUR), compactionConfig, fingerprintMapper @@ -407,7 +407,7 @@ public void testStatusWhenProjectionsMatch() .build(); final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); - final CompactionStatus status = CompactionStatus.evaluate( + final CompactionStatus status = CompactionStatus.compute( CompactionCandidate.ProposedCompaction.from(List.of(segment), Granularities.HOUR), compactionConfig, fingerprintMapper @@ -462,7 +462,7 @@ public void testStatusWhenProjectionsMismatch() .build(); final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); - final CompactionStatus status = CompactionStatus.evaluate( + final CompactionStatus status = CompactionStatus.compute( CompactionCandidate.ProposedCompaction.from(List.of(segment), Granularities.HOUR), compactionConfig, fingerprintMapper @@ -517,7 +517,7 @@ public void testStatusWhenAutoSchemaMatch() .build(); final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); - final CompactionStatus status = CompactionStatus.evaluate( + final CompactionStatus status = CompactionStatus.compute( CompactionCandidate.ProposedCompaction.from(List.of(segment), null), compactionConfig, fingerprintMapper @@ -571,7 +571,7 @@ public void testStatusWhenAutoSchemaMismatch() .build(); final DataSegment segment = DataSegment.builder(WIKI_SEGMENT).lastCompactionState(lastCompactionState).build(); - final CompactionStatus status = CompactionStatus.evaluate( + final CompactionStatus status = CompactionStatus.compute( CompactionCandidate.ProposedCompaction.from(List.of(segment), null), compactionConfig, fingerprintMapper @@ -663,7 +663,7 @@ public void test_evaluate_noCompacationIfUnexpectedFingerprintHasExpectedIndexin indexingStateStorage.upsertIndexingState(TestDataSource.WIKI, "wrongFingerprint", expectedState, DateTimes.nowUtc()); syncCacheFromManager(); - final CompactionStatus status = CompactionStatus.evaluate( + final CompactionStatus status = CompactionStatus.compute( CompactionCandidate.ProposedCompaction.from(segments, null), compactionConfig, fingerprintMapper @@ -708,7 +708,7 @@ public void test_evaluate_noCompactionWhenAllSegmentsHaveExpectedIndexingStateFi DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint(expectedFingerprint).build() ); - final CompactionStatus status = CompactionStatus.evaluate( + final CompactionStatus status = CompactionStatus.compute( CompactionCandidate.ProposedCompaction.from(segments, null), compactionConfig, fingerprintMapper @@ -762,7 +762,7 @@ public void test_evaluate_noCompactionWhenNonFingerprintedSegmentsPassChecksOnLa DataSegment.builder(WIKI_SEGMENT_2).indexingStateFingerprint(null).lastCompactionState(createCompactionStateWithGranularity(Granularities.DAY)).build() ); - final CompactionStatus status = CompactionStatus.evaluate( + final CompactionStatus status = CompactionStatus.compute( CompactionCandidate.ProposedCompaction.from(segments, null), compactionConfig, fingerprintMapper @@ -792,7 +792,7 @@ public void test_evaluate_isSkippedWhenInputBytesExceedLimit() DataSegment.builder(WIKI_SEGMENT_2).lastCompactionState(lastCompactionState).build() ); - final CompactionStatus status = CompactionStatus.evaluate( + final CompactionStatus status = CompactionStatus.compute( CompactionCandidate.ProposedCompaction.from(segments, null), compactionConfig, fingerprintMapper @@ -814,7 +814,7 @@ private void verifyEvaluationNeedsCompactionBecauseWithCustomSegments( String expectedReason ) { - final CompactionStatus status = CompactionStatus.evaluate( + final CompactionStatus status = CompactionStatus.compute( proposedCompaction, compactionConfig, fingerprintMapper @@ -834,7 +834,7 @@ private void verifyCompactionIsEligibleBecause( = DataSegment.builder(WIKI_SEGMENT) .lastCompactionState(lastCompactionState) .build(); - final CompactionStatus status = CompactionStatus.evaluate( + final CompactionStatus status = CompactionStatus.compute( CompactionCandidate.ProposedCompaction.from(List.of(segment), null), compactionConfig, fingerprintMapper From eb8b118d8dc7e2ce8a1c4db24a6af794914b24bb Mon Sep 17 00:00:00 2001 From: cecemei Date: Sat, 7 Feb 2026 15:22:27 -0800 Subject: [PATCH 15/19] revert incremental compaction change --- .../server/compaction/CompactionMode.java | 23 ----- .../compaction/CompactionRunSimulator.java | 1 - .../DataSourceCompactibleSegmentIterator.java | 1 - .../MostFragmentedIntervalFirstPolicy.java | 49 +---------- .../coordinator/duty/CompactSegments.java | 7 -- ...MostFragmentedIntervalFirstPolicyTest.java | 86 +------------------ 6 files changed, 5 insertions(+), 162 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionMode.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionMode.java index e89a5b2a7329..16782a3c28e0 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionMode.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionMode.java @@ -21,10 +21,8 @@ import org.apache.druid.error.DruidException; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.timeline.DataSegment; import javax.annotation.Nullable; -import java.util.Objects; public enum CompactionMode { @@ -39,27 +37,6 @@ public CompactionCandidate createCandidate( return new CompactionCandidate(proposedCompaction, eligibility, policyNote, this); } }, - INCREMENTAL_COMPACTION { - @Override - public CompactionCandidate createCandidate( - CompactionCandidate.ProposedCompaction proposedCompaction, - CompactionStatus eligibility, - @Nullable String policyNote - ) - { - CompactionCandidate.ProposedCompaction newProposed = new CompactionCandidate.ProposedCompaction( - Objects.requireNonNull(eligibility.getUncompactedSegments()), - proposedCompaction.getUmbrellaInterval(), - proposedCompaction.getCompactionInterval(), - Math.toIntExact(eligibility.getUncompactedSegments() - .stream() - .map(DataSegment::getInterval) - .distinct() - .count()) - ); - return new CompactionCandidate(newProposed, eligibility, policyNote, this); - } - }, NOT_APPLICABLE; public CompactionCandidate createCandidate( diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java index 1ffa48036ff9..1f12b0c2f327 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java @@ -123,7 +123,6 @@ public void onCompactionCandidates( ) )); break; - case INCREMENTAL_COMPACTION: case FULL_COMPACTION: queuedIntervals.addRow(createRow( candidateSegments, diff --git a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java index 26ae52b23eda..12b53a363545 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java +++ b/server/src/main/java/org/apache/druid/server/compaction/DataSourceCompactibleSegmentIterator.java @@ -354,7 +354,6 @@ private void findAndEnqueueSegmentsToCompact(CompactibleSegmentIterator compacti } switch (candidate.getMode()) { - case INCREMENTAL_COMPACTION: case FULL_COMPACTION: if (!queuedIntervals.contains(candidate.getProposedCompaction().getUmbrellaInterval())) { queue.add(candidate); diff --git a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java index 8fed350e2d6e..e07b21238b89 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java +++ b/server/src/main/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicy.java @@ -25,7 +25,6 @@ import org.apache.druid.error.InvalidInput; import org.apache.druid.guice.annotations.UnstableApi; import org.apache.druid.java.util.common.HumanReadableBytes; -import org.apache.druid.java.util.common.StringUtils; import javax.annotation.Nullable; import java.util.Comparator; @@ -48,7 +47,6 @@ public class MostFragmentedIntervalFirstPolicy extends BaseCandidateSearchPolicy private final int minUncompactedCount; private final HumanReadableBytes minUncompactedBytes; private final HumanReadableBytes maxAverageUncompactedBytesPerSegment; - private final double incrementalCompactionUncompactedBytesRatioThreshold; @JsonCreator public MostFragmentedIntervalFirstPolicy( @@ -56,8 +54,6 @@ public MostFragmentedIntervalFirstPolicy( @JsonProperty("minUncompactedBytes") @Nullable HumanReadableBytes minUncompactedBytes, @JsonProperty("maxAverageUncompactedBytesPerSegment") @Nullable HumanReadableBytes maxAverageUncompactedBytesPerSegment, - @JsonProperty("incrementalCompactionUncompactedBytesRatioThreshold") @Nullable - Double incrementalCompactionUncompactedBytesRatioThreshold, @JsonProperty("priorityDatasource") @Nullable String priorityDatasource ) { @@ -73,19 +69,11 @@ public MostFragmentedIntervalFirstPolicy( "'minUncompactedCount'[%s] must be greater than 0", maxAverageUncompactedBytesPerSegment ); - InvalidInput.conditionalException( - incrementalCompactionUncompactedBytesRatioThreshold == null - || incrementalCompactionUncompactedBytesRatioThreshold > 0, - "'incrementalCompactionUncompactedBytesRatioThreshold'[%s] must be greater than 0", - incrementalCompactionUncompactedBytesRatioThreshold - ); this.minUncompactedCount = Configs.valueOrDefault(minUncompactedCount, 100); this.minUncompactedBytes = Configs.valueOrDefault(minUncompactedBytes, SIZE_10_MB); this.maxAverageUncompactedBytesPerSegment = Configs.valueOrDefault(maxAverageUncompactedBytesPerSegment, SIZE_2_GB); - this.incrementalCompactionUncompactedBytesRatioThreshold = - Configs.valueOrDefault(incrementalCompactionUncompactedBytesRatioThreshold, 0.0d); } /** @@ -118,17 +106,6 @@ public HumanReadableBytes getMaxAverageUncompactedBytesPerSegment() return maxAverageUncompactedBytesPerSegment; } - /** - * Threshold ratio of uncompacted bytes to compacted bytes below which - * incremental compaction is eligible instead of full compaction. - * Default value is 0.0. - */ - @JsonProperty - public Double getIncrementalCompactionUncompactedRatioThreshold() - { - return incrementalCompactionUncompactedBytesRatioThreshold; - } - @Override protected Comparator getSegmentComparator() { @@ -147,12 +124,7 @@ public boolean equals(Object o) MostFragmentedIntervalFirstPolicy policy = (MostFragmentedIntervalFirstPolicy) o; return minUncompactedCount == policy.minUncompactedCount && Objects.equals(minUncompactedBytes, policy.minUncompactedBytes) - && Objects.equals(maxAverageUncompactedBytesPerSegment, policy.maxAverageUncompactedBytesPerSegment) - // Use Double.compare instead of == to handle NaN correctly and keep equals() consistent with hashCode() (especially for +0.0 vs -0.0). - && Double.compare( - incrementalCompactionUncompactedBytesRatioThreshold, - policy.incrementalCompactionUncompactedBytesRatioThreshold - ) == 0; + && Objects.equals(maxAverageUncompactedBytesPerSegment, policy.maxAverageUncompactedBytesPerSegment); } @Override @@ -162,8 +134,7 @@ public int hashCode() super.hashCode(), minUncompactedCount, minUncompactedBytes, - maxAverageUncompactedBytesPerSegment, - incrementalCompactionUncompactedBytesRatioThreshold + maxAverageUncompactedBytesPerSegment ); } @@ -175,7 +146,6 @@ public String toString() "minUncompactedCount=" + minUncompactedCount + ", minUncompactedBytes=" + minUncompactedBytes + ", maxAverageUncompactedBytesPerSegment=" + maxAverageUncompactedBytesPerSegment + - ", incrementalCompactionUncompactedBytesRatioThreshold=" + incrementalCompactionUncompactedBytesRatioThreshold + ", priorityDataSource='" + getPriorityDatasource() + '\'' + '}'; } @@ -225,20 +195,7 @@ public CompactionCandidate createCandidate( maxAverageUncompactedBytesPerSegment.getBytes() ); } - - final double uncompactedBytesRatio = (double) uncompacted.getTotalBytes() / - (uncompacted.getTotalBytes() + eligibility.getCompactedStats() - .getTotalBytes()); - if (uncompactedBytesRatio < incrementalCompactionUncompactedBytesRatioThreshold) { - String policyNote = StringUtils.format( - "Uncompacted bytes ratio[%.2f] is below threshold[%.2f]", - uncompactedBytesRatio, - incrementalCompactionUncompactedBytesRatioThreshold - ); - return CompactionMode.INCREMENTAL_COMPACTION.createCandidate(candidate, eligibility, policyNote); - } else { - return CompactionMode.FULL_COMPACTION.createCandidate(candidate, eligibility); - } + return CompactionMode.FULL_COMPACTION.createCandidate(candidate, eligibility); } /** diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java index c44e34584335..068644a2f16f 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java @@ -466,13 +466,6 @@ private static ClientCompactionTaskQuery compactSegments( case FULL_COMPACTION: clientCompactionIntervalSpec = new ClientCompactionIntervalSpec(entry.getCompactionInterval(), null, null); break; - case INCREMENTAL_COMPACTION: - clientCompactionIntervalSpec = new ClientCompactionIntervalSpec( - entry.getCompactionInterval(), - entry.getSegments().stream().map(DataSegment::toDescriptor).collect(Collectors.toList()), - null - ); - break; default: throw DruidException.defensive("Unexpected compaction mode[%s]", entry.getMode()); } diff --git a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java index 5fb2359252c1..2d5d9343ec7c 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java @@ -46,7 +46,7 @@ public class MostFragmentedIntervalFirstPolicyTest public void test_thresholdValues_ofDefaultPolicy() { final MostFragmentedIntervalFirstPolicy policy = - new MostFragmentedIntervalFirstPolicy(null, null, null, null, null); + new MostFragmentedIntervalFirstPolicy(null, null, null, null); Assertions.assertEquals(100, policy.getMinUncompactedCount()); Assertions.assertEquals(new HumanReadableBytes("10MiB"), policy.getMinUncompactedBytes()); Assertions.assertEquals(new HumanReadableBytes("2GiB"), policy.getMaxAverageUncompactedBytesPerSegment()); @@ -61,7 +61,6 @@ public void test_createCandidate_fails_ifUncompactedCountLessThanCutoff() minUncompactedCount, HumanReadableBytes.valueOf(1), HumanReadableBytes.valueOf(10_000), - null, null ); @@ -88,7 +87,6 @@ public void test_createCandidate_fails_ifUncompactedBytesLessThanCutoff() 1, minUncompactedBytes, HumanReadableBytes.valueOf(10_000), - null, null ); @@ -112,7 +110,6 @@ public void test_createCandidate_fails_ifAvgSegmentSizeGreaterThanCutoff() 1, HumanReadableBytes.valueOf(100), maxAvgSegmentSize, - null, null ); @@ -137,7 +134,6 @@ public void test_policy_favorsIntervalWithMoreUncompactedSegments_ifTotalBytesIs 1, HumanReadableBytes.valueOf(1), HumanReadableBytes.valueOf(10_000), - null, null ); @@ -159,7 +155,6 @@ public void test_policy_favorsIntervalWithMoreUncompactedSegments_ifAverageSizeI 1, HumanReadableBytes.valueOf(1), HumanReadableBytes.valueOf(10_000), - null, null ); @@ -181,7 +176,6 @@ public void test_policy_favorsIntervalWithSmallerSegments_ifCountIsEqual() 1, HumanReadableBytes.valueOf(1), HumanReadableBytes.valueOf(10_000), - null, null ); @@ -203,7 +197,6 @@ public void test_compareCandidates_returnsZeroIfSegmentCountAndAvgSizeScaleEquiv 100, HumanReadableBytes.valueOf(1), HumanReadableBytes.valueOf(100), - null, null ); @@ -234,7 +227,6 @@ public void test_serde_allFieldsSet() throws IOException 1, HumanReadableBytes.valueOf(2), HumanReadableBytes.valueOf(3), - null, "foo" ); final DefaultObjectMapper mapper = new DefaultObjectMapper(); @@ -247,87 +239,13 @@ public void test_serde_allFieldsSet() throws IOException public void test_serde_noFieldsSet() throws IOException { final MostFragmentedIntervalFirstPolicy policy = - new MostFragmentedIntervalFirstPolicy(null, null, null, null, null); + new MostFragmentedIntervalFirstPolicy(null, null, null, null); final DefaultObjectMapper mapper = new DefaultObjectMapper(); final CompactionCandidateSearchPolicy policy2 = mapper.readValue(mapper.writeValueAsString(policy), CompactionCandidateSearchPolicy.class); Assertions.assertEquals(policy, policy2); } - @Test - public void test_createCandidate_returnsIncrementalCompaction_whenRatioBelowThreshold() - { - // Set threshold to 0.5 (50%) - final MostFragmentedIntervalFirstPolicy policy = new MostFragmentedIntervalFirstPolicy( - 1, - HumanReadableBytes.valueOf(1), - HumanReadableBytes.valueOf(10_000), - 0.5, - null - ); - - final CompactionStatistics compacted = CompactionStatistics.create(1200L, 10, 1L); - final CompactionStatistics uncompacted = CompactionStatistics.create(400L, 100, 1L); - final CompactionStatus eligibility = eligibilityBuilder() - .compacted(compacted) - .uncompacted(uncompacted) - .uncompactedSegments(List.of(SEGMENT)) - .build(); - - final CompactionCandidate candidate = policy.createCandidate(PROPOSED_COMPACTION, eligibility); - Assertions.assertEquals("Uncompacted bytes ratio[0.25] is below threshold[0.50]", candidate.getPolicyNote()); - Assertions.assertEquals(CompactionMode.INCREMENTAL_COMPACTION, candidate.getMode()); - Assertions.assertEquals( - CompactionCandidate.ProposedCompaction.from(List.of(SEGMENT), null), - candidate.getProposedCompaction() - ); - } - - @Test - public void test_createCandidate_returnsFullCompaction_whenRatioAboveThreshold() - { - // Set threshold to 0.5 (50%) - final MostFragmentedIntervalFirstPolicy policy = new MostFragmentedIntervalFirstPolicy( - 1, - HumanReadableBytes.valueOf(1), - HumanReadableBytes.valueOf(10_000), - 0.5, - null - ); - - final CompactionStatus eligibility = - eligibilityBuilder() - .compacted(CompactionStatistics.create(500L, 5, 1)) - .uncompacted(CompactionStatistics.create(600L, 100, 1)) - .build(); - final CompactionCandidate candidate = policy.createCandidate(PROPOSED_COMPACTION, eligibility); - - Assertions.assertEquals(CompactionMode.FULL_COMPACTION, candidate.getMode()); - } - - @Test - public void test_createCandidate_returnsFullCompaction_whenThresholdIsDefault() - { - // Default threshold is 0.0 - final MostFragmentedIntervalFirstPolicy policy = new MostFragmentedIntervalFirstPolicy( - 1, - HumanReadableBytes.valueOf(1), - HumanReadableBytes.valueOf(10_000), - null, - null - ); - - // With default threshold 0.0, any positive ratio >= 0.0, so always FULL_COMPACTION_ELIGIBLE - final CompactionStatus eligibility = - eligibilityBuilder() - .compacted(CompactionStatistics.create(1_000L, 10, 1)) - .uncompacted(CompactionStatistics.create(100L, 100, 1)) - .build(); - final CompactionCandidate candidate = policy.createCandidate(PROPOSED_COMPACTION, eligibility); - - Assertions.assertEquals(CompactionMode.FULL_COMPACTION, candidate.getMode()); - } - private CompactionStatistics createStats(int numSegments, long avgSizeBytes) { return CompactionStatistics.create(avgSizeBytes * numSegments, numSegments, 1L); From 599090c1be201bc4a24dc4565ba2669b5304defe Mon Sep 17 00:00:00 2001 From: cecemei Date: Sat, 7 Feb 2026 15:27:43 -0800 Subject: [PATCH 16/19] revert incremental compaction change 2 --- .../ClientCompactionTaskQuerySerdeTest.java | 2 +- .../ClientCompactionIntervalSpec.java | 35 +--------------- .../coordinator/duty/CompactSegments.java | 2 +- .../ClientCompactionIntervalSpecTest.java | 41 ------------------- .../coordinator/duty/CompactSegmentsTest.java | 1 - 5 files changed, 3 insertions(+), 78 deletions(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java index 842227efb01e..1c04c7b5bd2b 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java @@ -301,7 +301,7 @@ private ClientCompactionTaskQuery createCompactionTaskQuery(String id, Compactio id, "datasource", new ClientCompactionIOConfig( - new ClientCompactionIntervalSpec(Intervals.of("2019/2020"), null, "testSha256OfSortedSegmentIds"), true + new ClientCompactionIntervalSpec(Intervals.of("2019/2020"), "testSha256OfSortedSegmentIds"), true ), new ClientCompactionTaskQueryTuningConfig( 100, diff --git a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpec.java b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpec.java index 46707e1ea55a..7a7f65572319 100644 --- a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpec.java +++ b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpec.java @@ -20,16 +20,12 @@ package org.apache.druid.client.indexing; import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.java.util.common.IAE; -import org.apache.druid.query.SegmentDescriptor; import org.joda.time.Interval; import javax.annotation.Nullable; -import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; /** * InputSpec for {@link ClientCompactionIOConfig}. @@ -42,14 +38,11 @@ public class ClientCompactionIntervalSpec private final Interval interval; @Nullable - private final List uncompactedSegments; - @Nullable private final String sha256OfSortedSegmentIds; @JsonCreator public ClientCompactionIntervalSpec( @JsonProperty("interval") Interval interval, - @JsonProperty("uncompactedSegments") @Nullable List uncompactedSegments, @JsonProperty("sha256OfSortedSegmentIds") @Nullable String sha256OfSortedSegmentIds ) { @@ -57,22 +50,6 @@ public ClientCompactionIntervalSpec( throw new IAE("Interval[%s] is empty, must specify a nonempty interval", interval); } this.interval = interval; - if (uncompactedSegments == null) { - // perform a full compaction - } else if (uncompactedSegments.isEmpty()) { - throw new IAE("Can not supply empty segments as input, please use either null or non-empty segments."); - } else if (interval != null) { - List segmentsNotInInterval = - uncompactedSegments.stream().filter(s -> !interval.contains(s.getInterval())).collect(Collectors.toList()); - if (!segmentsNotInInterval.isEmpty()) { - throw new IAE( - "Can not supply segments outside interval[%s], got segments[%s].", - interval, - segmentsNotInInterval - ); - } - } - this.uncompactedSegments = uncompactedSegments; this.sha256OfSortedSegmentIds = sha256OfSortedSegmentIds; } @@ -88,14 +65,6 @@ public Interval getInterval() return interval; } - @Nullable - @JsonProperty - @JsonInclude(JsonInclude.Include.NON_NULL) - public List getUncompactedSegments() - { - return uncompactedSegments; - } - @Nullable @JsonProperty public String getSha256OfSortedSegmentIds() @@ -114,14 +83,13 @@ public boolean equals(Object o) } ClientCompactionIntervalSpec that = (ClientCompactionIntervalSpec) o; return Objects.equals(interval, that.interval) && - Objects.equals(uncompactedSegments, that.uncompactedSegments) && Objects.equals(sha256OfSortedSegmentIds, that.sha256OfSortedSegmentIds); } @Override public int hashCode() { - return Objects.hash(interval, uncompactedSegments, sha256OfSortedSegmentIds); + return Objects.hash(interval, sha256OfSortedSegmentIds); } @Override @@ -129,7 +97,6 @@ public String toString() { return "ClientCompactionIntervalSpec{" + "interval=" + interval + - ", uncompactedSegments=" + uncompactedSegments + ", sha256OfSortedSegmentIds='" + sha256OfSortedSegmentIds + '\'' + '}'; } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java index 068644a2f16f..25286e1becda 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java @@ -464,7 +464,7 @@ private static ClientCompactionTaskQuery compactSegments( final ClientCompactionIntervalSpec clientCompactionIntervalSpec; switch (entry.getMode()) { case FULL_COMPACTION: - clientCompactionIntervalSpec = new ClientCompactionIntervalSpec(entry.getCompactionInterval(), null, null); + clientCompactionIntervalSpec = new ClientCompactionIntervalSpec(entry.getCompactionInterval(), null); break; default: throw DruidException.defensive("Unexpected compaction mode[%s]", entry.getMode()); diff --git a/server/src/test/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpecTest.java b/server/src/test/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpecTest.java index 0c4d8d5d7b45..6aa1f976bad4 100644 --- a/server/src/test/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpecTest.java +++ b/server/src/test/java/org/apache/druid/client/indexing/ClientCompactionIntervalSpecTest.java @@ -19,18 +19,13 @@ package org.apache.druid.client.indexing; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.segment.IndexIO; import org.apache.druid.server.compaction.CompactionCandidate.ProposedCompaction; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.NoneShardSpec; -import org.joda.time.Interval; import org.junit.Assert; import org.junit.Test; @@ -126,40 +121,4 @@ public void testFromSegmentWithFinerSegmentGranularityAndUmbrellaIntervalNotAlig // Hence the compaction interval is modified to aling with the segmentGranularity Assert.assertEquals(Intervals.of("2015-02-09/2015-04-20"), actual.getCompactionInterval()); } - - @Test - public void testClientCompactionIntervalSpec_throwsException_whenEmptySegmentsList() - { - Interval interval = Intervals.of("2015-04-11/2015-04-12"); - List emptySegments = List.of(); - - Assert.assertThrows( - IAE.class, - () -> new ClientCompactionIntervalSpec(interval, emptySegments, null) - ); - } - - @Test - public void testClientCompactionIntervalSpec_serde() throws Exception - { - ObjectMapper mapper = new DefaultObjectMapper(); - Interval interval = Intervals.of("2015-04-11/2015-04-12"); - List segments = List.of( - new SegmentDescriptor(Intervals.of("2015-04-11/2015-04-12"), "v1", 0) - ); - - // Test with uncompactedSegments (incremental compaction) - ClientCompactionIntervalSpec withSegments = new ClientCompactionIntervalSpec(interval, segments, "sha256hash"); - String json1 = mapper.writeValueAsString(withSegments); - ClientCompactionIntervalSpec deserialized1 = mapper.readValue(json1, ClientCompactionIntervalSpec.class); - Assert.assertEquals(withSegments, deserialized1); - Assert.assertEquals(segments, deserialized1.getUncompactedSegments()); - - // Test without uncompactedSegments (full compaction) - ClientCompactionIntervalSpec withoutSegments = new ClientCompactionIntervalSpec(interval, null, null); - String json2 = mapper.writeValueAsString(withoutSegments); - ClientCompactionIntervalSpec deserialized2 = mapper.readValue(json2, ClientCompactionIntervalSpec.class); - Assert.assertEquals(withoutSegments, deserialized2); - Assert.assertNull(deserialized2.getUncompactedSegments()); - } } diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java index 5fec43a2fc7e..122f7996c327 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java @@ -1107,7 +1107,6 @@ public void testCompactWithGranularitySpecConflictWithActiveCompactionTask() new ClientCompactionIOConfig( new ClientCompactionIntervalSpec( Intervals.of("2000/2099"), - null, "testSha256OfSortedSegmentIds" ), null From ac6d1475e2b5e5b1dab4d430dd06b1a5ff687674 Mon Sep 17 00:00:00 2001 From: cecemei Date: Sat, 7 Feb 2026 15:42:01 -0800 Subject: [PATCH 17/19] fix bug --- .../apache/druid/server/coordinator/duty/CompactSegments.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java index 25286e1becda..af44f3b7923f 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java @@ -248,10 +248,8 @@ private int submitCompactionTasks( switch (compactionTaskState) { case READY: case TASK_IN_PROGRESS: - // As these segments will be compacted, we will aggregate the statistic to the Compacted statistics - snapshotBuilder.addToComplete(entry); - break; case RECENTLY_COMPLETED: + // As these segments will be compacted, we will aggregate the statistic to the Compacted statistics snapshotBuilder.addToComplete(entry); break; default: From e44fb348a280afa07bb0bd2d2152a4a90f098e30 Mon Sep 17 00:00:00 2001 From: cecemei Date: Sat, 7 Feb 2026 15:49:43 -0800 Subject: [PATCH 18/19] format --- .../druid/server/coordinator/duty/CompactSegments.java | 9 +++++++-- .../druid/server/compaction/CompactionStatusTest.java | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java index af44f3b7923f..b18b69f51b91 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java @@ -32,6 +32,7 @@ import org.apache.druid.client.indexing.ClientCompactionTaskQuery; import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig; import org.apache.druid.common.guava.FutureUtils; +import org.apache.druid.common.guava.GuavaUtils; import org.apache.druid.common.utils.IdUtils; import org.apache.druid.data.input.impl.AggregateProjectionSpec; import org.apache.druid.error.DruidException; @@ -44,6 +45,7 @@ import org.apache.druid.segment.transform.CompactionTransformSpec; import org.apache.druid.server.compaction.CompactionCandidate; import org.apache.druid.server.compaction.CompactionCandidateSearchPolicy; +import org.apache.druid.server.compaction.CompactionMode; import org.apache.druid.server.compaction.CompactionSegmentIterator; import org.apache.druid.server.compaction.CompactionSlotManager; import org.apache.druid.server.compaction.CompactionSnapshotBuilder; @@ -367,8 +369,11 @@ public static ClientCompactionTaskQuery createCompactionTask( } final Map autoCompactionContext = newAutoCompactionContext(config.getTaskContext()); - if (candidate.getEligibility().getReason() != null) { - autoCompactionContext.put(COMPACTION_REASON_KEY, candidate.getEligibility().getReason()); + if (CompactionMode.NOT_APPLICABLE.equals(candidate.getMode())) { + autoCompactionContext.put( + COMPACTION_REASON_KEY, + GuavaUtils.firstNonNull(candidate.getPolicyNote(), candidate.getEligibility().getReason()) + ); } autoCompactionContext.put(STORE_COMPACTION_STATE_KEY, storeCompactionStatePerSegment); diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java index 52ddfa44d526..cb82f64dd6d5 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java @@ -798,7 +798,7 @@ public void test_evaluate_isSkippedWhenInputBytesExceedLimit() fingerprintMapper ); - Assert.assertFalse(status.getState().equals(CompactionStatus.State.ELIGIBLE)); + Assert.assertEquals(CompactionStatus.State.NOT_ELIGIBLE, status.getState()); Assert.assertTrue(status.getReason().contains("'inputSegmentSize' exceeded")); Assert.assertTrue(status.getReason().contains("200000000")); Assert.assertTrue(status.getReason().contains("150000000")); From 55405ef16702f477a92775b98594760a51539093 Mon Sep 17 00:00:00 2001 From: cecemei Date: Sat, 7 Feb 2026 16:06:56 -0800 Subject: [PATCH 19/19] revert incremental compaction change 3 --- .../server/compaction/CompactionStatus.java | 36 ++++--------------- .../CompactionStatusBuilderTest.java | 32 ++++------------- .../CompactionStatusTrackerTest.java | 1 - ...MostFragmentedIntervalFirstPolicyTest.java | 3 +- 4 files changed, 14 insertions(+), 58 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java index 4b9f6a9efdf3..917ec0c85994 100644 --- a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java +++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatus.java @@ -61,7 +61,7 @@ */ public class CompactionStatus { - public static final CompactionStatus COMPLETE = new CompactionStatus(State.COMPLETE, "", null, null, null); + public static final CompactionStatus COMPLETE = new CompactionStatus(State.COMPLETE, "", null, null); public enum State { @@ -94,7 +94,7 @@ public enum State public static CompactionStatus notEligible(String messageFormat, Object... args) { - return new CompactionStatus(State.NOT_ELIGIBLE, StringUtils.format(messageFormat, args), null, null, null); + return new CompactionStatus(State.NOT_ELIGIBLE, StringUtils.format(messageFormat, args), null, null); } private final State state; @@ -104,15 +104,12 @@ public static CompactionStatus notEligible(String messageFormat, Object... args) private final CompactionStatistics compacted; @Nullable private final CompactionStatistics uncompacted; - @Nullable - private final List uncompactedSegments; private CompactionStatus( State state, String reason, @Nullable CompactionStatistics compacted, - @Nullable CompactionStatistics uncompacted, - @Nullable List uncompactedSegments + @Nullable CompactionStatistics uncompacted ) { this.state = state; @@ -129,17 +126,12 @@ private CompactionStatus( case ELIGIBLE: InvalidInput.conditionalException(compacted != null, "must provide compacted stats for compaction"); InvalidInput.conditionalException(uncompacted != null, "must provide uncompacted stats for compaction"); - InvalidInput.conditionalException( - uncompactedSegments != null, - "must provide uncompactedSegments for compaction" - ); break; default: throw DruidException.defensive("unexpected compaction status state[%s]", state); } this.compacted = compacted; this.uncompacted = uncompacted; - this.uncompactedSegments = uncompactedSegments; } static CompactionStatusBuilder builder(State state, String reason) @@ -169,12 +161,6 @@ public CompactionStatistics getCompactedStats() return compacted; } - @Nullable - public List getUncompactedSegments() - { - return uncompactedSegments; - } - /** * Evaluates a compaction candidate to determine its eligibility and compaction status. *

@@ -214,14 +200,13 @@ public boolean equals(Object object) return state == that.state && Objects.equals(reason, that.reason) && Objects.equals(compacted, that.compacted) - && Objects.equals(uncompacted, that.uncompacted) - && Objects.equals(uncompactedSegments, that.uncompactedSegments); + && Objects.equals(uncompacted, that.uncompacted); } @Override public int hashCode() { - return Objects.hash(state, reason, compacted, uncompacted, uncompactedSegments); + return Objects.hash(state, reason, compacted, uncompacted); } @Override @@ -232,7 +217,6 @@ public String toString() + ", reason='" + reason + '\'' + ", compacted=" + compacted + ", uncompacted=" + uncompacted - + ", uncompactedSegments=" + uncompactedSegments + '}'; } @@ -473,7 +457,6 @@ private CompactionStatus evaluate() } else { return builder(State.ELIGIBLE, reasonsForCompaction.get(0)).compacted(createStats(compactedSegments)) .uncompacted(createStats(uncompactedSegments)) - .uncompactedSegments(uncompactedSegments) .build(); } } @@ -858,7 +841,6 @@ static class CompactionStatusBuilder private State state; private CompactionStatistics compacted; private CompactionStatistics uncompacted; - private List uncompactedSegments; private String reason; CompactionStatusBuilder(State state, String reason) @@ -879,15 +861,9 @@ CompactionStatusBuilder uncompacted(CompactionStatistics uncompacted) return this; } - CompactionStatusBuilder uncompactedSegments(List uncompactedSegments) - { - this.uncompactedSegments = uncompactedSegments; - return this; - } - CompactionStatus build() { - return new CompactionStatus(state, reason, compacted, uncompacted, uncompactedSegments); + return new CompactionStatus(state, reason, compacted, uncompacted); } } } diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusBuilderTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusBuilderTest.java index 576489596297..d0c381b02706 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusBuilderTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusBuilderTest.java @@ -42,7 +42,6 @@ public void testNotEligible() Assert.assertEquals("test reason: failure", eligibility.getReason()); Assert.assertNull(eligibility.getCompactedStats()); Assert.assertNull(eligibility.getUncompactedStats()); - Assert.assertNull(eligibility.getUncompactedSegments()); } @Test @@ -50,20 +49,17 @@ public void testBuilderWithCompactionStats() { CompactionStatistics compactedStats = CompactionStatistics.create(1000, 5, 2); CompactionStatistics uncompactedStats = CompactionStatistics.create(500, 3, 1); - List uncompactedSegments = createTestSegments(3); CompactionStatus eligibility = CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "needs full compaction") .compacted(compactedStats) .uncompacted(uncompactedStats) - .uncompactedSegments(uncompactedSegments) .build(); Assert.assertEquals(CompactionStatus.State.ELIGIBLE, eligibility.getState()); Assert.assertEquals("needs full compaction", eligibility.getReason()); Assert.assertEquals(compactedStats, eligibility.getCompactedStats()); Assert.assertEquals(uncompactedStats, eligibility.getUncompactedStats()); - Assert.assertEquals(uncompactedSegments, eligibility.getUncompactedSegments()); } @Test @@ -83,23 +79,20 @@ public void testEqualsAndHashCode() CompactionStatus differentState = CompactionStatus.COMPLETE; Assert.assertNotEquals(simple1, differentState); - // Test with full compaction eligibility (with stats and segments) + // Test with full compaction eligibility (with stats) CompactionStatistics stats1 = CompactionStatistics.create(1000, 5, 2); CompactionStatistics stats2 = CompactionStatistics.create(500, 3, 1); - List segments = createTestSegments(3); CompactionStatus withStats1 = CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "reason") .compacted(stats1) .uncompacted(stats2) - .uncompactedSegments(segments) .build(); CompactionStatus withStats2 = CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "reason") .compacted(stats1) .uncompacted(stats2) - .uncompactedSegments(segments) .build(); // Same values - should be equal @@ -112,7 +105,6 @@ public void testEqualsAndHashCode() CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "reason") .compacted(differentStats) .uncompacted(stats2) - .uncompactedSegments(segments) .build(); Assert.assertNotEquals(withStats1, differentCompactedStats); @@ -121,19 +113,8 @@ public void testEqualsAndHashCode() CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "reason") .compacted(stats1) .uncompacted(differentStats) - .uncompactedSegments(segments) .build(); Assert.assertNotEquals(withStats1, differentUncompactedStats); - - // Test with different segment lists - List differentSegments = createTestSegments(5); - CompactionStatus differentSegmentList = - CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "reason") - .compacted(stats1) - .uncompacted(stats2) - .uncompactedSegments(differentSegments) - .build(); - Assert.assertNotEquals(withStats1, differentSegmentList); } @Test @@ -148,11 +129,13 @@ public void testBuilderRequiresReasonForNotEligible() @Test public void testBuilderRequiresStatsForFullCompaction() { + // Should throw when neither stat is provided Assert.assertThrows( DruidException.class, () -> CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "reason").build() ); + // Should throw when only compacted stat is provided Assert.assertThrows( DruidException.class, () -> CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "reason") @@ -160,13 +143,12 @@ public void testBuilderRequiresStatsForFullCompaction() .build() ); - Assert.assertThrows( - DruidException.class, - () -> CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "reason") + // Should succeed when both stats are provided + CompactionStatus status = CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "reason") .compacted(CompactionStatistics.create(1000, 5, 2)) .uncompacted(CompactionStatistics.create(500, 3, 1)) - .build() - ); + .build(); + Assert.assertNotNull(status); } private static List createTestSegments(int count) diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java index 855ea57d2753..c5dc7644781e 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java @@ -135,7 +135,6 @@ private static CompactionCandidate createCandidate( CompactionStatus status = CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "approve without check") .compacted(CompactionStatistics.create(1, 1, 1)) .uncompacted(CompactionStatistics.create(1, 1, 1)) - .uncompactedSegments(List.of()) .build(); return CompactionMode.FULL_COMPACTION.createCandidate(proposedCompaction, status); } diff --git a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java index 2d5d9343ec7c..dc88c61d5aaf 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/MostFragmentedIntervalFirstPolicyTest.java @@ -253,7 +253,6 @@ private CompactionStatistics createStats(int numSegments, long avgSizeBytes) private static CompactionStatus.CompactionStatusBuilder eligibilityBuilder() { - return CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "approve") - .uncompactedSegments(List.of()); + return CompactionStatus.builder(CompactionStatus.State.ELIGIBLE, "approve"); } }