Skip to content

Commit 4e886eb

Browse files
jian-hejian-he
committed
YARN-3379. Fixed missing data in localityTable and ResourceRequests table in RM WebUI. Contributed by Xuan Gong
1 parent e37ca22 commit 4e886eb

File tree

7 files changed

+290
-148
lines changed

7 files changed

+290
-148
lines changed

‎hadoop-yarn-project/CHANGES.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -796,6 +796,9 @@ Release 2.7.0 - UNRELEASED
796796
YARN-3349. Treat all exceptions as failure in
797797
TestFSRMStateStore#testFSRMStateStoreClientRetry. (Zhihai Xu via ozawa)
798798

799+
YARN-3379. Fixed missing data in localityTable and ResourceRequests table
800+
in RM WebUI. (Xuan Gong via jianhe)
801+
799802
Release 2.6.0 - 2014-11-18
800803

801804
INCOMPATIBLE CHANGES

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java‎

Lines changed: 5 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,6 @@
1919

2020
import static org.apache.hadoop.yarn.util.StringHelper.join;
2121
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ATTEMPT_ID;
22-
import static org.apache.hadoop.yarn.webapp.YarnWebParams.WEB_UI_TYPE;
23-
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._EVEN;
24-
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
25-
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._ODD;
26-
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
27-
2822
import java.security.PrivilegedExceptionAction;
2923
import java.util.Collection;
3024

@@ -43,20 +37,18 @@
4337
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
4438
import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
4539
import org.apache.hadoop.yarn.util.ConverterUtils;
46-
import org.apache.hadoop.yarn.webapp.YarnWebParams;
4740
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
48-
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
4941
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
5042
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
5143
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
5244
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
53-
5445
import com.google.inject.Inject;
5546

5647
public class AppAttemptBlock extends HtmlBlock {
5748

5849
private static final Log LOG = LogFactory.getLog(AppAttemptBlock.class);
5950
protected ApplicationBaseProtocol appBaseProt;
51+
protected ApplicationAttemptId appAttemptId = null;
6052

6153
@Inject
6254
public AppAttemptBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) {
@@ -66,14 +58,12 @@ public AppAttemptBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) {
6658

6759
@Override
6860
protected void render(Block html) {
69-
String webUiType = $(WEB_UI_TYPE);
7061
String attemptid = $(APPLICATION_ATTEMPT_ID);
7162
if (attemptid.isEmpty()) {
7263
puts("Bad request: requires application attempt ID");
7364
return;
7465
}
7566

76-
ApplicationAttemptId appAttemptId = null;
7767
try {
7868
appAttemptId = ConverterUtils.toApplicationAttemptId(attemptid);
7969
} catch (IllegalArgumentException e) {
@@ -183,17 +173,7 @@ public Collection<ContainerReport> run() throws Exception {
183173
return;
184174
}
185175

186-
// TODO need to render applicationHeadRoom value from
187-
// ApplicationAttemptMetrics after YARN-3284
188-
if (webUiType.equals(YarnWebParams.RM_WEB_UI)) {
189-
if (!isApplicationInFinalState(appAttempt.getAppAttemptState())) {
190-
DIV<Hamlet> pdiv = html._(InfoBlock.class).div(_INFO_WRAP);
191-
info("Application Attempt Overview").clear();
192-
info("Application Attempt Metrics")._(
193-
"Application Attempt Headroom : ", 0);
194-
pdiv._();
195-
}
196-
}
176+
createAttemptHeadRoomTable(html);
197177
html._(InfoBlock.class);
198178

199179
// Container Table
@@ -236,45 +216,6 @@ public Collection<ContainerReport> run() throws Exception {
236216
._("var containersTableData=" + containersTableData)._();
237217

238218
tbody._()._();
239-
240-
if (webUiType.equals(YarnWebParams.RM_WEB_UI)) {
241-
createContainerLocalityTable(html); // TODO:YARN-3284
242-
}
243-
}
244-
245-
//TODO: YARN-3284
246-
//The containerLocality metrics will be exposed from AttemptReport
247-
private void createContainerLocalityTable(Block html) {
248-
int totalAllocatedContainers = 0; //TODO: YARN-3284
249-
int[][] localityStatistics = new int[0][0];//TODO:YARN-3284
250-
DIV<Hamlet> div = html.div(_INFO_WRAP);
251-
TABLE<DIV<Hamlet>> table =
252-
div.h3(
253-
"Total Allocated Containers: "
254-
+ totalAllocatedContainers).h3("Each table cell"
255-
+ " represents the number of NodeLocal/RackLocal/OffSwitch containers"
256-
+ " satisfied by NodeLocal/RackLocal/OffSwitch resource requests.").table(
257-
"#containerLocality");
258-
table.
259-
tr().
260-
th(_TH, "").
261-
th(_TH, "Node Local Request").
262-
th(_TH, "Rack Local Request").
263-
th(_TH, "Off Switch Request").
264-
_();
265-
266-
String[] containersType =
267-
{ "Num Node Local Containers (satisfied by)", "Num Rack Local Containers (satisfied by)",
268-
"Num Off Switch Containers (satisfied by)" };
269-
boolean odd = false;
270-
for (int i = 0; i < localityStatistics.length; i++) {
271-
table.tr((odd = !odd) ? _ODD : _EVEN).td(containersType[i])
272-
.td(String.valueOf(localityStatistics[i][0]))
273-
.td(i == 0 ? "" : String.valueOf(localityStatistics[i][1]))
274-
.td(i <= 1 ? "" : String.valueOf(localityStatistics[i][2]))._();
275-
}
276-
table._();
277-
div._();
278219
}
279220

280221
private boolean hasAMContainer(ContainerId containerId,
@@ -286,10 +227,8 @@ private boolean hasAMContainer(ContainerId containerId,
286227
}
287228
return false;
288229
}
289-
290-
private boolean isApplicationInFinalState(YarnApplicationAttemptState state) {
291-
return state == YarnApplicationAttemptState.FINISHED
292-
|| state == YarnApplicationAttemptState.FAILED
293-
|| state == YarnApplicationAttemptState.KILLED;
230+
231+
protected void createAttemptHeadRoomTable(Block html) {
232+
294233
}
295234
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java‎

Lines changed: 9 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,8 @@
2121
import static org.apache.hadoop.yarn.util.StringHelper.join;
2222
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID;
2323
import static org.apache.hadoop.yarn.webapp.YarnWebParams.WEB_UI_TYPE;
24-
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
2524
import java.security.PrivilegedExceptionAction;
2625
import java.util.Collection;
27-
import java.util.List;
28-
2926
import org.apache.commons.lang.StringEscapeUtils;
3027
import org.apache.commons.logging.Log;
3128
import org.apache.commons.logging.LogFactory;
@@ -39,12 +36,9 @@
3936
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
4037
import org.apache.hadoop.yarn.api.records.ApplicationId;
4138
import org.apache.hadoop.yarn.api.records.ApplicationReport;
42-
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
4339
import org.apache.hadoop.yarn.api.records.ContainerId;
4440
import org.apache.hadoop.yarn.api.records.ContainerReport;
4541
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
46-
import org.apache.hadoop.yarn.api.records.Resource;
47-
import org.apache.hadoop.yarn.api.records.ResourceRequest;
4842
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
4943
import org.apache.hadoop.yarn.conf.YarnConfiguration;
5044
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
@@ -53,10 +47,8 @@
5347
import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
5448
import org.apache.hadoop.yarn.util.Apps;
5549
import org.apache.hadoop.yarn.util.Times;
56-
import org.apache.hadoop.yarn.util.resource.Resources;
5750
import org.apache.hadoop.yarn.webapp.YarnWebParams;
5851
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
59-
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
6052
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
6153
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
6254
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
@@ -69,9 +61,11 @@ public class AppBlock extends HtmlBlock {
6961
private static final Log LOG = LogFactory.getLog(AppBlock.class);
7062
protected ApplicationBaseProtocol appBaseProt;
7163
protected Configuration conf;
64+
protected ApplicationId appID = null;
7265

7366
@Inject
74-
AppBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx, Configuration conf) {
67+
protected AppBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx,
68+
Configuration conf) {
7569
super(ctx);
7670
this.appBaseProt = appBaseProt;
7771
this.conf = conf;
@@ -86,7 +80,6 @@ protected void render(Block html) {
8680
return;
8781
}
8882

89-
ApplicationId appID = null;
9083
try {
9184
appID = Apps.toAppID(aid);
9285
} catch (Exception e) {
@@ -213,31 +206,7 @@ public Collection<ApplicationAttemptReport> run() throws Exception {
213206
return;
214207
}
215208

216-
//TODO:YARN-3284
217-
//The preemption metrics will be exposed from ApplicationReport
218-
// and ApplicationAttemptReport
219-
ApplicationResourceUsageReport usageReport =
220-
appReport.getApplicationResourceUsageReport();
221-
DIV<Hamlet> pdiv = html.
222-
_(InfoBlock.class).
223-
div(_INFO_WRAP);
224-
info("Application Overview").clear();
225-
info("Application Metrics")
226-
._("Total Resource Preempted:",
227-
Resources.none()) // TODO: YARN-3284
228-
._("Total Number of Non-AM Containers Preempted:",
229-
String.valueOf(0)) // TODO: YARN-3284
230-
._("Total Number of AM Containers Preempted:",
231-
String.valueOf(0)) // TODO: YARN-3284
232-
._("Resource Preempted from Current Attempt:",
233-
Resources.none()) // TODO: YARN-3284
234-
._("Number of Non-AM Containers Preempted from Current Attempt:",
235-
0) // TODO: YARN-3284
236-
._("Aggregate Resource Allocation:",
237-
String.format("%d MB-seconds, %d vcore-seconds", usageReport == null
238-
? 0 : usageReport.getMemorySeconds(), usageReport == null ? 0
239-
: usageReport.getVcoreSeconds()));
240-
pdiv._();
209+
createApplicationMetricsTable(html);
241210

242211
html._(InfoBlock.class);
243212

@@ -319,49 +288,6 @@ public ContainerReport run() throws Exception {
319288
._("var attemptsTableData=" + attemptsTableData)._();
320289

321290
tbody._()._();
322-
323-
if (webUiType != null && webUiType.equals(YarnWebParams.RM_WEB_UI)) {
324-
createResourceRequestsTable(html, null); // TODO:YARN-3284
325-
}
326-
}
327-
328-
//TODO:YARN-3284
329-
//The resource requests metrics will be exposed from attemptReport
330-
private void createResourceRequestsTable(Block html, List<ResourceRequest> resouceRequests) {
331-
TBODY<TABLE<Hamlet>> tbody =
332-
html.table("#ResourceRequests").thead().tr()
333-
.th(".priority", "Priority")
334-
.th(".resourceName", "ResourceName")
335-
.th(".totalResource", "Capability")
336-
.th(".numContainers", "NumContainers")
337-
.th(".relaxLocality", "RelaxLocality")
338-
.th(".nodeLabelExpression", "NodeLabelExpression")._()._().tbody();
339-
340-
Resource totalResource = Resource.newInstance(0, 0);
341-
if (resouceRequests != null) {
342-
for (ResourceRequest request : resouceRequests) {
343-
if (request.getNumContainers() == 0) {
344-
continue;
345-
}
346-
347-
tbody.tr()
348-
.td(String.valueOf(request.getPriority()))
349-
.td(request.getResourceName())
350-
.td(String.valueOf(request.getCapability()))
351-
.td(String.valueOf(request.getNumContainers()))
352-
.td(String.valueOf(request.getRelaxLocality()))
353-
.td(request.getNodeLabelExpression() == null ? "N/A" : request
354-
.getNodeLabelExpression())._();
355-
if (request.getResourceName().equals(ResourceRequest.ANY)) {
356-
Resources.addTo(totalResource,
357-
Resources.multiply(request.getCapability(),
358-
request.getNumContainers()));
359-
}
360-
}
361-
}
362-
html.div().$class("totalResourceRequests")
363-
.h3("Total Outstanding Resource Requests: " + totalResource)._();
364-
tbody._()._();
365291
}
366292

367293
private String clarifyAppState(YarnApplicationState state) {
@@ -389,4 +315,9 @@ private String clairfyAppFinalStatus(FinalApplicationStatus status) {
389315
}
390316
return status.toString();
391317
}
318+
319+
// The preemption metrics only need to be shown in RM WebUI
320+
protected void createApplicationMetricsTable(Block html) {
321+
322+
}
392323
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
2424
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
2525

26-
import org.apache.hadoop.yarn.server.webapp.AppAttemptBlock;
2726
import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
2827
import org.apache.hadoop.yarn.webapp.SubView;
2928
import org.apache.hadoop.yarn.webapp.YarnWebParams;
@@ -51,7 +50,7 @@ protected void preHead(Page.HTML<_> html) {
5150

5251
@Override
5352
protected Class<? extends SubView> content() {
54-
return AppAttemptBlock.class;
53+
return RMAppAttemptBlock.class;
5554
}
5655

5756
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
2424
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
2525

26-
import org.apache.hadoop.yarn.server.webapp.AppBlock;
2726
import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
2827
import org.apache.hadoop.yarn.webapp.SubView;
2928
import org.apache.hadoop.yarn.webapp.YarnWebParams;
@@ -50,6 +49,6 @@ protected void preHead(Page.HTML<_> html) {
5049

5150
@Override
5251
protected Class<? extends SubView> content() {
53-
return AppBlock.class;
52+
return RMAppBlock.class;
5453
}
5554
}

0 commit comments

Comments
 (0)