Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HDFS-17694.RBF:Make get datanodes reports configurable. #7235

Open
wants to merge 1 commit into
base: trunk
Choose a base branch
from
+34 −13
Open

HDFS-17694.RBF:Make get datanodes reports configurable. #7235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 22 additions & 13 deletions ...f/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,8 @@ public class NamenodeBeanMetrics
/** Timeout to get the DN report. */
private final long dnReportTimeOut;
/** DN type -> full DN report in JSON. */
private final LoadingCache<DatanodeReportType, String> dnCache;
private LoadingCache<DatanodeReportType, String> dnCache = null;
private final boolean dnReportEnable;


public NamenodeBeanMetrics(Router router) {
Expand Down Expand Up @@ -139,18 +140,23 @@ public NamenodeBeanMetrics(Router router) {
this.dnReportTimeOut = conf.getTimeDuration(
RBFConfigKeys.DN_REPORT_TIME_OUT,
RBFConfigKeys.DN_REPORT_TIME_OUT_MS_DEFAULT, TimeUnit.MILLISECONDS);
long dnCacheExpire = conf.getTimeDuration(
RBFConfigKeys.DN_REPORT_CACHE_EXPIRE,
RBFConfigKeys.DN_REPORT_CACHE_EXPIRE_MS_DEFAULT, TimeUnit.MILLISECONDS);
this.dnCache = CacheBuilder.newBuilder()
.expireAfterWrite(dnCacheExpire, TimeUnit.MILLISECONDS)
.build(
new CacheLoader<DatanodeReportType, String>() {
@Override
public String load(DatanodeReportType type) throws Exception {
return getNodesImpl(type);
}
});
this.dnReportEnable = conf.getBoolean(RBFConfigKeys.DFS_ROUTER_DN_REPORT_ENABLE_KEY,
RBFConfigKeys.DFS_ROUTER_DN_REPORT_ENABLE_DEFAULT);
if (dnReportEnable) {
long dnCacheExpire = conf.getTimeDuration(
RBFConfigKeys.DN_REPORT_CACHE_EXPIRE,
RBFConfigKeys.DN_REPORT_CACHE_EXPIRE_MS_DEFAULT, TimeUnit.MILLISECONDS);
this.dnCache = CacheBuilder.newBuilder()
.expireAfterWrite(dnCacheExpire, TimeUnit.MILLISECONDS)
.build(
new CacheLoader<DatanodeReportType, String>() {
@Override
public String load(DatanodeReportType type) throws Exception {
return getNodesImpl(type);
}
});
}

}

/**
Expand Down Expand Up @@ -458,6 +464,9 @@ public String getDecomNodes() {
* @return JSON with the nodes.
*/
private String getNodes(final DatanodeReportType type) {
if (!dnReportEnable) {
return "N/A";
}
try {
return this.dnCache.get(type);
} catch (ExecutionException e) {
Expand Down
3 changes: 3 additions & 0 deletions ...hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,9 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic {
public static final String DFS_ROUTER_ENABLE_GET_DN_USAGE_KEY =
FEDERATION_ROUTER_PREFIX + "enable.get.dn.usage";
public static final boolean DFS_ROUTER_ENABLE_GET_DN_USAGE_DEFAULT = true;
public static final String DFS_ROUTER_DN_REPORT_ENABLE_KEY =
FEDERATION_ROUTER_PREFIX + "dn.report.enable";
public static final boolean DFS_ROUTER_DN_REPORT_ENABLE_DEFAULT = true;

// HDFS Router-based federation quota
public static final String DFS_ROUTER_QUOTA_ENABLE =
Expand Down
9 changes: 9 additions & 0 deletions hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,15 @@
</description>
</property>

<property>
<name>dfs.federation.router.dn.report.enable</name>
<value>true</value>
<description>
If false, the getLiveNodes|getDeadNodes|getDecomNodes methods in NamenodeBeanMetrics will
return "N/A", else will return the cached result collecting from downstream nameservices.
</description>
</property>

<property>
<name>dfs.federation.router.metrics.class</name>
<value>org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCPerformanceMonitor</value>
Expand Down
Loading