Skip to content

introduce clear separation between proposed compaction, eligibility status, and compaction mode, along with improved task state tracking #18968

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cecemei wants to merge 20 commits into
base: master
Choose a base branch
from
Open

introduce clear separation between proposed compaction, eligibility status, and compaction mode, along with improved task state tracking #18968

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
35dad44
wip
cecemei Jan 30, 2026
0a4a0df
test
cecemei Feb 1, 2026
ca0bc0c
eligibility
cecemei Feb 2, 2026
ab79904
Double.compare
cecemei Feb 2, 2026
08d70b8
CompactionEligibility
cecemei Feb 4, 2026
5aba7d9
candidate
cecemei Feb 4, 2026
5001d31
format
cecemei Feb 4, 2026
e38257b
policy-status
cecemei Feb 5, 2026
502854d
eligible
cecemei Feb 6, 2026
3ec2d34
revert drop existing change
cecemei Feb 6, 2026
f9f1bc9
javadoc
cecemei Feb 6, 2026
5bfcf0c
test
cecemei Feb 6, 2026
5487f8e
Merge branch 'master' into compact3
cecemei Feb 7, 2026
a39a35e
compaction mode
cecemei Feb 7, 2026
aa6a5ed
format
cecemei Feb 7, 2026
eb8b118
revert incremental compaction change
cecemei Feb 7, 2026
599090c
revert incremental compaction change 2
cecemei Feb 7, 2026
ac6d147
fix bug
cecemei Feb 7, 2026
e44fb34
format
cecemei Feb 7, 2026
55405ef
revert incremental compaction change 3
cecemei Feb 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions ...ice/src/main/java/org/apache/druid/indexing/compact/CompactionConfigBasedJobTemplate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
import org.apache.druid.server.compaction.CompactionCandidate;
import org.apache.druid.server.compaction.CompactionSlotManager;
import org.apache.druid.server.compaction.DataSourceCompactibleSegmentIterator;
import org.apache.druid.server.compaction.NewestSegmentFirstPolicy;
import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
import org.apache.druid.server.coordinator.duty.CompactSegments;
import org.apache.druid.timeline.CompactionState;
Expand Down Expand Up @@ -133,9 +132,7 @@ DataSourceCompactibleSegmentIterator getCompactibleCandidates(
config,
timeline,
Intervals.complementOf(searchInterval),
// This policy is used only while creating jobs
// The actual order of jobs is determined by the policy used in CompactionJobQueue
new NewestSegmentFirstPolicy(null),
params.getClusterCompactionConfig().getCompactionPolicy(),
params.getFingerprintMapper()
);

Expand Down
28 changes: 12 additions & 16 deletions indexing-service/src/main/java/org/apache/druid/indexing/compact/CompactionJobQueue.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.druid.client.indexing.ClientCompactionTaskQuery;
import org.apache.druid.client.indexing.ClientTaskQuery;
import org.apache.druid.common.guava.FutureUtils;
import org.apache.druid.error.DruidException;
import org.apache.druid.indexer.TaskState;
import org.apache.druid.indexer.TaskStatus;
import org.apache.druid.indexing.common.actions.TaskActionClientFactory;
Expand All @@ -43,7 +44,6 @@
import org.apache.druid.server.compaction.CompactionCandidateSearchPolicy;
import org.apache.druid.server.compaction.CompactionSlotManager;
import org.apache.druid.server.compaction.CompactionSnapshotBuilder;
import org.apache.druid.server.compaction.CompactionStatus;
import org.apache.druid.server.compaction.CompactionStatusTracker;
import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
import org.apache.druid.server.coordinator.ClusterCompactionConfig;
Expand Down Expand Up @@ -217,7 +217,7 @@ public void runReadyJobs()
final List<CompactionJob> pendingJobs = new ArrayList<>();
while (!queue.isEmpty()) {
final CompactionJob job = queue.poll();
if (startJobIfPendingAndReady(job, searchPolicy, pendingJobs, slotManager)) {
if (startJobIfPendingAndReady(job, pendingJobs, slotManager)) {
runStats.add(Stats.Compaction.SUBMITTED_TASKS, RowKey.of(Dimension.DATASOURCE, job.getDataSource()), 1);
}
}
Expand Down Expand Up @@ -267,7 +267,6 @@ public Map<String, AutoCompactionSnapshot> getSnapshots()
*/
private boolean startJobIfPendingAndReady(
CompactionJob job,
CompactionCandidateSearchPolicy policy,
List<CompactionJob> pendingJobs,
CompactionSlotManager slotManager
)
Expand All @@ -282,18 +281,17 @@ private boolean startJobIfPendingAndReady(
}

// Check if the job is already running, completed or skipped
final CompactionStatus compactionStatus = getCurrentStatusForJob(job, policy);
switch (compactionStatus.getState()) {
case RUNNING:
final CompactionCandidate.TaskState candidateState = getCurrentTaskStateForJob(job);
Copy link
Contributor

@kfaraz kfaraz Feb 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please do not add a new task state. Use the existing CompactionTaskStatus class.

switch (candidateState) {
case TASK_IN_PROGRESS:
return false;
case COMPLETE:
case RECENTLY_COMPLETED:
snapshotBuilder.moveFromPendingToCompleted(candidate);
return false;
case SKIPPED:
snapshotBuilder.moveFromPendingToSkipped(candidate);
return false;
default:
case READY:
break;
default:
throw DruidException.defensive("unknown compaction candidate state[%s]", candidateState);
}

// Check if enough compaction task slots are available
Expand Down Expand Up @@ -378,12 +376,10 @@ private void persistPendingIndexingState(CompactionJob job)
}
}

public CompactionStatus getCurrentStatusForJob(CompactionJob job, CompactionCandidateSearchPolicy policy)
public CompactionCandidate.TaskState getCurrentTaskStateForJob(CompactionJob job)
{
final CompactionStatus compactionStatus = statusTracker.computeCompactionStatus(job.getCandidate(), policy);
final CompactionCandidate candidatesWithStatus = job.getCandidate().withCurrentStatus(null);
statusTracker.onCompactionStatusComputed(candidatesWithStatus, null);
return compactionStatus;
statusTracker.onCompactionCandidates(job.getCandidate(), null);
return statusTracker.computeCompactionTaskState(job.getCandidate());
}

public static CompactionConfigValidationResult validateCompactionJob(BatchIndexingJob job)
Expand Down
2 changes: 1 addition & 1 deletion ...-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -499,7 +499,7 @@ public CompactionSimulateResult simulateRunWithConfigUpdate(ClusterCompactionCon
updateRequest.getEngine()
);
} else {
return new CompactionSimulateResult(Collections.emptyMap());
return new CompactionSimulateResult(Collections.emptyMap(), null);
}
}

Expand Down
4 changes: 2 additions & 2 deletions ...vice/src/test/java/org/apache/druid/indexing/compact/OverlordCompactionSchedulerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,9 @@
import org.apache.druid.segment.TestIndex;
import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage;
import org.apache.druid.segment.metadata.IndexingStateCache;
import org.apache.druid.server.compaction.CompactionCandidate;
import org.apache.druid.server.compaction.CompactionSimulateResult;
import org.apache.druid.server.compaction.CompactionStatistics;
import org.apache.druid.server.compaction.CompactionStatus;
import org.apache.druid.server.compaction.CompactionStatusTracker;
import org.apache.druid.server.compaction.Table;
import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
Expand Down Expand Up @@ -451,7 +451,7 @@ public void test_simulateRunWithConfigUpdate()
new ClusterCompactionConfig(null, null, null, null, null, null)
);
Assert.assertEquals(1, simulateResult.getCompactionStates().size());
final Table pendingCompactionTable = simulateResult.getCompactionStates().get(CompactionStatus.State.PENDING);
final Table pendingCompactionTable = simulateResult.getCompactionStates().get(CompactionCandidate.TaskState.READY);
Assert.assertEquals(
Arrays.asList("dataSource", "interval", "numSegments", "bytes", "maxTaskSlots", "reasonToCompact"),
pendingCompactionTable.getColumnNames()
Expand Down
8 changes: 4 additions & 4 deletions server/src/main/java/org/apache/druid/server/compaction/BaseCandidateSearchPolicy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,12 @@ public final int compareCandidates(CompactionCandidate o1, CompactionCandidate o
}

@Override
public Eligibility checkEligibilityForCompaction(
CompactionCandidate candidate,
CompactionTaskStatus latestTaskStatus
public CompactionCandidate createCandidate(
CompactionCandidate.ProposedCompaction proposedCompaction,
CompactionStatus eligibility
)
{
return Eligibility.OK;
return CompactionMode.FULL_COMPACTION.createCandidate(proposedCompaction, eligibility);
}

/**
Expand Down
Loading
Loading