Skip to content

Commit

Permalink
IGNITE-24157 CDC WAL archive metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
@maksaska
maksaska committed Jan 16, 2025
1 parent 326cc47 commit 0647624
Show file tree
Hide file tree
Showing 4 changed files with 10 additions and 85 deletions.
3 changes: 1 addition & 2 deletions docs/_docs/monitoring-metrics/new-metrics.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -419,8 +419,6 @@ Register name: `io.datastorage`
[cols="2,1,3",opts="header"]
|===
|Name | Type | Description
|CdcWalArchiveSegments | integer | Current number of WAL segments to be processed by CDC clients in the CDC WAL archive.
|CdcWalTotalSize | long | Total size in bytes for storage wal files in the CDC WAL archive.
|CheckpointBeforeLockHistogram| histogram | Histogram of checkpoint action before taken write lock duration in milliseconds.
|CheckpointFsyncHistogram| histogram | Histogram of checkpoint fsync duration in milliseconds.
|CheckpointHistogram| histogram | Histogram of checkpoint duration in milliseconds.
Expand All @@ -433,6 +431,7 @@ Register name: `io.datastorage`
|CheckpointTotalTime| long | Total duration of checkpoint
|CheckpointWalRecordFsyncHistogram| histogram | Histogram of the WAL fsync after logging ChTotalNodeseckpointRecord on begin of checkpoint duration in milliseconds.
|CheckpointWriteEntryHistogram| histogram | Histogram of entry buffer writing to file duration in milliseconds.
|LastArchivedSegment | long | Last archived file absolute index, 0-based.
|LastCheckpointBeforeLockDuration| long | Duration of the checkpoint action before taken write lock in milliseconds.
|LastCheckpointCopiedOnWritePagesNumber| long | Number of pages copied to a temporary checkpoint buffer during the last checkpoint.
|LastCheckpointDataPagesNumber| long | Total number of data pages written during the last checkpoint.
Expand Down
48 changes: 7 additions & 41 deletions .../java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,6 @@ public class DataStorageMetricsImpl {
/** */
private volatile IgniteOutClosure<Long> walSizeProvider;

/** */
private volatile IgniteOutClosure<Integer> cdcWalArchiveSegmentsProvider;

/** */
private volatile IgniteOutClosure<Long> cdcWalTotalSizeProvider;

/** */
private final AtomicLongMetric lastWalSegmentRollOverTime;

Expand Down Expand Up @@ -277,17 +271,13 @@ public DataStorageMetricsImpl(
this::walArchiveSegments,
"Current number of WAL segments in the WAL archive.");

mreg.register("CdcWalArchiveSegments",
this::cdcWalArchiveSegments,
"Current number of WAL segments to be processed by CDC clients in the CDC WAL archive.");

mreg.register("WalTotalSize",
this::walTotalSize,
"Total size in bytes for storage wal files.");

mreg.register("CdcWalTotalSize",
this::cdcWalTotalSize,
"Total size in bytes for storage wal files in the CDC WAL archive.");
mreg.register("LastArchivedSegment",
this::lastArchivedSegment,
"Last archived file absolute index, 0-based.");

long[] cpBounds = new long[] {100, 500, 1000, 5000, 30000};

Expand Down Expand Up @@ -376,14 +366,14 @@ private int walArchiveSegments() {
return walMgr == null ? 0 : walMgr.walArchiveSegments();
}

/** @return Current number of WAL segments in the WAL archive. */
private int cdcWalArchiveSegments() {
/** @return Last archived file absolute index, 0-based. */
private long lastArchivedSegment() {
if (!metricsEnabled)
return 0;

IgniteOutClosure<Integer> cdcWalArchiveSegments = this.cdcWalArchiveSegmentsProvider;
IgniteWriteAheadLogManager walMgr = this.wal;

return cdcWalArchiveSegments != null ? cdcWalArchiveSegments.apply() : 0;
return walMgr == null ? 0 : walMgr.lastArchivedSegment();
}

/**
Expand Down Expand Up @@ -411,16 +401,6 @@ private long walTotalSize() {
return walSize != null ? walSize.apply() : 0;
}

/** @return Total size in bytes for storage wal files in the CDC WAL archive. */
private long cdcWalTotalSize() {
if (!metricsEnabled)
return 0;

IgniteOutClosure<Long> cdcWalTotalSize = this.cdcWalTotalSizeProvider;

return cdcWalTotalSize != null ? cdcWalTotalSize.apply() : 0;
}

/**
* Total dirty pages for the next checkpoint.
*
Expand Down Expand Up @@ -655,20 +635,6 @@ public void setWalSizeProvider(IgniteOutClosure<Long> walSizeProvider) {
this.walSizeProvider = walSizeProvider;
}

/**
* @param cdcWalArchiveSegmentsProvider Current number of WAL segments provider for CDC WAL archive.
*/
public void setCdcWalArchiveSegmentsProvider(IgniteOutClosure<Integer> cdcWalArchiveSegmentsProvider) {
this.cdcWalArchiveSegmentsProvider = cdcWalArchiveSegmentsProvider;
}

/**
* @param cdcWalTotalSizeProvider Wal size provider for CDC WAL archive.
*/
public void setCdcWalTotalSizeProvider(IgniteOutClosure<Long> cdcWalTotalSizeProvider) {
this.cdcWalTotalSizeProvider = cdcWalTotalSizeProvider;
}

/**
*
*/
Expand Down
36 changes: 0 additions & 36 deletions ...org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -552,42 +552,6 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
return size;
}
});

final File walCdcDir0 = walCdcDir;

if (metrics != null) {
metrics.setCdcWalTotalSizeProvider(new CO<Long>() {
/** {@inheritDoc} */
@Override public Long apply() {
long size = 0;

if (isArchiverEnabled()) {
File[] cdcFiles = walCdcDir0.listFiles();

if (cdcFiles == null)
return size;

for (File f : cdcFiles)
size += f.length();
}

return size;
}
});

metrics.setCdcWalArchiveSegmentsProvider(new CO<Integer>() {
/** {@inheritDoc} */
@Override public Integer apply() {
if (isArchiverEnabled()) {
File[] cdcFiles = walCdcDir0.listFiles();

return cdcFiles == null ? 0 : cdcFiles.length;
}

return 0;
}
});
}
}

segmentAware = new SegmentAware(
Expand Down
8 changes: 2 additions & 6 deletions ...apache/ignite/internal/processors/cache/persistence/IgniteDataStorageMetricsSelfTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@
import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentRouter;
import org.apache.ignite.internal.processors.metric.impl.AtomicLongMetric;
import org.apache.ignite.internal.processors.metric.impl.IntGauge;
import org.apache.ignite.internal.processors.metric.impl.LongAdderMetric;
import org.apache.ignite.internal.processors.metric.impl.LongGauge;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
Expand Down Expand Up @@ -542,13 +541,10 @@ private void checkWalArchiveAndTotalSize(IgniteEx igniteEx, boolean hasWalArchiv
assertEquals(totalSize, dsMetricRegistry(igniteEx).<LongGauge>findMetric("WalTotalSize").value());

if (router.hasArchive()) {
long totalCdcArchiveSize = walMgr.totalSize(walFiles(walMgr.walCdcDirectory()));

assertEquals(totalCdcArchiveSize, dsMetricRegistry(igniteEx).<LongGauge>findMetric("CdcWalTotalSize").value());

long cdcWalArchiveSegments = walFiles(walMgr.walCdcDirectory()).length;

assertEquals(cdcWalArchiveSegments, dsMetricRegistry(igniteEx).<IntGauge>findMetric("CdcWalArchiveSegments").value());
// Count of segments = LastArchivedSegmentIndex + 1
assertEquals(cdcWalArchiveSegments, dsMetricRegistry(igniteEx).<LongGauge>findMetric("LastArchivedSegment").value() + 1);
}
}

Expand Down

0 comments on commit 0647624

Please sign in to comment.