diff options
10 files changed, 698 insertions, 302 deletions
diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html new file mode 100644 index 0000000000..64ea719141 --- /dev/null +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html @@ -0,0 +1,105 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +<script id="executors-summary-template" type="text/html"> + <h4 style="clear: left; display: inline-block;">Summary</h4> + <div class="container-fluid"> + <div class="container-fluid"> + <table id="summary-execs-table" class="table table-striped compact"> + <thead> + <th></th> + <th>RDD Blocks</th> + <th><span data-toggle="tooltip" + title="Memory used / total available memory for storage of data like RDD partitions cached in memory. ">Storage Memory</span> + </th> + <th>Disk Used</th> + <th>Cores</th> + <th>Active Tasks</th> + <th>Failed Tasks</th> + <th>Complete Tasks</th> + <th>Total Tasks</th> + <th><span data-toggle="tooltip" + title="Shaded red when garbage collection (GC) time is over 10% of task time"> + Task Time (GC Time)</span></th> + <th><span data-toggle="tooltip" + title="Bytes and records read from Hadoop or from Spark storage.">Input</span></th> + <th><span data-toggle="tooltip" + title="Total shuffle bytes and records read (includes both data read locally and data read from remote executors)."> + Shuffle Read</span></th> + <th> + <span data-toggle="tooltip" data-placement="left" + title="Bytes and records written to disk in order to be read by a shuffle in a future stage."> + Shuffle Write</span> + </th> + </thead> + <tbody> + </tbody> + </table> + </div> + </div> + <h4 style="clear: left; display: inline-block;">Executors</h4> + <div class="container-fluid"> + <div class="container-fluid"> + <table id="active-executors-table" class="table table-striped compact"> + <thead> + <tr> + <th> + <span data-toggle="tooltip" data-placement="right" title="ID of the executor">Executor ID</span></th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Address">Address</span></th> + <th><span data-toggle="tooltip" data-placement="top" title="Status">Status</span></th> + <th> + <span data-toggle="tooltip" data-placement="top" title="RDD Blocks">RDD Blocks</span></th> + <th> + <span data-toggle="tooltip" data-placement="top" + title="Memory used / total available memory for storage of data like RDD partitions cached in memory."> + Storage Memory</span></th> + <th><span data-toggle="tooltip" data-placement="top" title="Disk Used">Disk Used</span></th> + <th><span data-toggle="tooltip" data-placement="top" title="Cores">Cores</span></th> + <th><span data-toggle="tooltip" data-placement="top" title="Active Tasks">Active Tasks</span></th> + <th><span data-toggle="tooltip" data-placement="top" title="Failed Tasks">Failed Tasks</span></th> + <th><span data-toggle="tooltip" data-placement="top" title="Complete Tasks">Complete Tasks</span></th> + <th><span data-toggle="tooltip" data-placement="top" title="Total Tasks">Total Tasks</span></th> + <th> + <scan data-toggle="tooltip" data-placement="top" + title="Shaded red when garbage collection (GC) time is over 10% of task time"> + Task Time (GC Time) + </scan> + </th> + <th><span data-toggle="tooltip" data-placement="top" + title="Bytes and records read from Hadoop or from Spark storage.">Input</span></th> + <th> + <span data-toggle="tooltip" data-placement="top" + title="Total shuffle bytes and records read (includes both data read locally and data read from remote executors)."> + Shuffle Read</span></th> + <th> + <!-- Place the shuffle write tooltip on the left (rather than the default position + of on top) because the shuffle write column is the last column on the right side and + the tooltip is wider than the column, so it doesn't fit on top. --> + <span data-toggle="tooltip" data-placement="left" + title="Bytes and records written to disk in order to be read by a shuffle in a future stage."> + Shuffle Write</span></th> + <th><span data-toggle="tooltip" data-placement="left" title="Logs">Logs</span></th> + <th><span data-toggle="tooltip" data-placement="left" title="Thread Dump">Thread Dump</span></th> + </tr> + </thead> + <tbody> + </tbody> + </table> + </div> + </div> +</script> diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js new file mode 100644 index 0000000000..b2b2363d3a --- /dev/null +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js @@ -0,0 +1,470 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +function formatStatus(status, type) { + if (type !== 'display') return status; + if (status) { + return "Active" + } else { + return "Dead" + } +} + +jQuery.extend(jQuery.fn.dataTableExt.oSort, { + "title-numeric-pre": function (a) { + var x = a.match(/title="*(-?[0-9\.]+)/)[1]; + return parseFloat(x); + }, + + "title-numeric-asc": function (a, b) { + return ((a < b) ? -1 : ((a > b) ? 1 : 0)); + }, + + "title-numeric-desc": function (a, b) { + return ((a < b) ? 1 : ((a > b) ? -1 : 0)); + } +}); + +$(document).ajaxStop($.unblockUI); +$(document).ajaxStart(function () { + $.blockUI({message: '<h3>Loading Executors Page...</h3>'}); +}); + +function createTemplateURI(appId) { + var words = document.baseURI.split('/'); + var ind = words.indexOf("proxy"); + if (ind > 0) { + var baseURI = words.slice(0, ind + 1).join('/') + '/' + appId + '/static/executorspage-template.html'; + return baseURI; + } + ind = words.indexOf("history"); + if(ind > 0) { + var baseURI = words.slice(0, ind).join('/') + '/static/executorspage-template.html'; + return baseURI; + } + return location.origin + "/static/executorspage-template.html"; +} + +function getStandAloneppId(cb) { + var words = document.baseURI.split('/'); + var ind = words.indexOf("proxy"); + if (ind > 0) { + var appId = words[ind + 1]; + cb(appId); + return; + } + ind = words.indexOf("history"); + if (ind > 0) { + var appId = words[ind + 1]; + cb(appId); + return; + } + //Looks like Web UI is running in standalone mode + //Let's get application-id using REST End Point + $.getJSON(location.origin + "/api/v1/applications", function(response, status, jqXHR) { + if (response && response.length > 0) { + var appId = response[0].id + cb(appId); + return; + } + }); +} + +function createRESTEndPoint(appId) { + var words = document.baseURI.split('/'); + var ind = words.indexOf("proxy"); + if (ind > 0) { + var appId = words[ind + 1]; + var newBaseURI = words.slice(0, ind + 2).join('/'); + return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors" + } + ind = words.indexOf("history"); + if (ind > 0) { + var appId = words[ind + 1]; + var attemptId = words[ind + 2]; + var newBaseURI = words.slice(0, ind).join('/'); + if (isNaN(attemptId)) { + return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors"; + } else { + return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/allexecutors"; + } + } + return location.origin + "/api/v1/applications/" + appId + "/allexecutors"; +} + +function formatLogsCells(execLogs, type) { + if (type !== 'display') return Object.keys(execLogs); + if (!execLogs) return; + var result = ''; + $.each(execLogs, function (logName, logUrl) { + result += '<div><a href=' + logUrl + '>' + logName + '</a></div>' + }); + return result; +} + +// Determine Color Opacity from 0.5-1 +// activeTasks range from 0 to maxTasks +function activeTasksAlpha(activeTasks, maxTasks) { + return maxTasks > 0 ? ((activeTasks / maxTasks) * 0.5 + 0.5) : 1; +} + +function activeTasksStyle(activeTasks, maxTasks) { + return activeTasks > 0 ? ("hsla(240, 100%, 50%, " + activeTasksAlpha(activeTasks, maxTasks) + ")") : ""; +} + +// failedTasks range max at 10% failure, alpha max = 1 +function failedTasksAlpha(failedTasks, totalTasks) { + return totalTasks > 0 ? + (Math.min(10 * failedTasks / totalTasks, 1) * 0.5 + 0.5) : 1; +} + +function failedTasksStyle(failedTasks, totalTasks) { + return failedTasks > 0 ? + ("hsla(0, 100%, 50%, " + failedTasksAlpha(failedTasks, totalTasks) + ")") : ""; +} + +// totalDuration range from 0 to 50% GC time, alpha max = 1 +function totalDurationAlpha(totalGCTime, totalDuration) { + return totalDuration > 0 ? + (Math.min(totalGCTime / totalDuration + 0.5, 1)) : 1; +} + +function totalDurationStyle(totalGCTime, totalDuration) { + // Red if GC time over GCTimePercent of total time + // When GCTimePercent is edited change ToolTips.TASK_TIME to match + var GCTimePercent = 0.1; + return (totalGCTime > GCTimePercent * totalDuration) ? + ("hsla(0, 100%, 50%, " + totalDurationAlpha(totalGCTime, totalDuration) + ")") : ""; +} + +function totalDurationColor(totalGCTime, totalDuration) { + // Red if GC time over GCTimePercent of total time + // When GCTimePercent is edited change ToolTips.TASK_TIME to match + var GCTimePercent = 0.1; + return (totalGCTime > GCTimePercent * totalDuration) ? "white" : "black"; +} + +$(document).ready(function () { + $.extend($.fn.dataTable.defaults, { + stateSave: true, + lengthMenu: [[20, 40, 60, 100, -1], [20, 40, 60, 100, "All"]], + pageLength: 20 + }); + + executorsSummary = $("#active-executors"); + + getStandAloneppId(function (appId) { + + var endPoint = createRESTEndPoint(appId); + $.getJSON(endPoint, function (response, status, jqXHR) { + var summary = []; + var allExecCnt = 0; + var allRDDBlocks = 0; + var allMemoryUsed = 0; + var allMaxMemory = 0; + var allDiskUsed = 0; + var allTotalCores = 0; + var allMaxTasks = 0; + var allActiveTasks = 0; + var allFailedTasks = 0; + var allCompletedTasks = 0; + var allTotalTasks = 0; + var allTotalDuration = 0; + var allTotalGCTime = 0; + var allTotalInputBytes = 0; + var allTotalShuffleRead = 0; + var allTotalShuffleWrite = 0; + + var activeExecCnt = 0; + var activeRDDBlocks = 0; + var activeMemoryUsed = 0; + var activeMaxMemory = 0; + var activeDiskUsed = 0; + var activeTotalCores = 0; + var activeMaxTasks = 0; + var activeActiveTasks = 0; + var activeFailedTasks = 0; + var activeCompletedTasks = 0; + var activeTotalTasks = 0; + var activeTotalDuration = 0; + var activeTotalGCTime = 0; + var activeTotalInputBytes = 0; + var activeTotalShuffleRead = 0; + var activeTotalShuffleWrite = 0; + + var deadExecCnt = 0; + var deadRDDBlocks = 0; + var deadMemoryUsed = 0; + var deadMaxMemory = 0; + var deadDiskUsed = 0; + var deadTotalCores = 0; + var deadMaxTasks = 0; + var deadActiveTasks = 0; + var deadFailedTasks = 0; + var deadCompletedTasks = 0; + var deadTotalTasks = 0; + var deadTotalDuration = 0; + var deadTotalGCTime = 0; + var deadTotalInputBytes = 0; + var deadTotalShuffleRead = 0; + var deadTotalShuffleWrite = 0; + + response.forEach(function (exec) { + allExecCnt += 1; + allRDDBlocks += exec.rddBlocks; + allMemoryUsed += exec.memoryUsed; + allMaxMemory += exec.maxMemory; + allDiskUsed += exec.diskUsed; + allTotalCores += exec.totalCores; + allMaxTasks += exec.maxTasks; + allActiveTasks += exec.activeTasks; + allFailedTasks += exec.failedTasks; + allCompletedTasks += exec.completedTasks; + allTotalTasks += exec.totalTasks; + allTotalDuration += exec.totalDuration; + allTotalGCTime += exec.totalGCTime; + allTotalInputBytes += exec.totalInputBytes; + allTotalShuffleRead += exec.totalShuffleRead; + allTotalShuffleWrite += exec.totalShuffleWrite; + if (exec.isActive) { + activeExecCnt += 1; + activeRDDBlocks += exec.rddBlocks; + activeMemoryUsed += exec.memoryUsed; + activeMaxMemory += exec.maxMemory; + activeDiskUsed += exec.diskUsed; + activeTotalCores += exec.totalCores; + activeMaxTasks += exec.maxTasks; + activeActiveTasks += exec.activeTasks; + activeFailedTasks += exec.failedTasks; + activeCompletedTasks += exec.completedTasks; + activeTotalTasks += exec.totalTasks; + activeTotalDuration += exec.totalDuration; + activeTotalGCTime += exec.totalGCTime; + activeTotalInputBytes += exec.totalInputBytes; + activeTotalShuffleRead += exec.totalShuffleRead; + activeTotalShuffleWrite += exec.totalShuffleWrite; + } else { + deadExecCnt += 1; + deadRDDBlocks += exec.rddBlocks; + deadMemoryUsed += exec.memoryUsed; + deadMaxMemory += exec.maxMemory; + deadDiskUsed += exec.diskUsed; + deadTotalCores += exec.totalCores; + deadMaxTasks += exec.maxTasks; + deadActiveTasks += exec.activeTasks; + deadFailedTasks += exec.failedTasks; + deadCompletedTasks += exec.completedTasks; + deadTotalTasks += exec.totalTasks; + deadTotalDuration += exec.totalDuration; + deadTotalGCTime += exec.totalGCTime; + deadTotalInputBytes += exec.totalInputBytes; + deadTotalShuffleRead += exec.totalShuffleRead; + deadTotalShuffleWrite += exec.totalShuffleWrite; + } + }); + + var totalSummary = { + "execCnt": ( "Total(" + allExecCnt + ")"), + "allRDDBlocks": allRDDBlocks, + "allMemoryUsed": allMemoryUsed, + "allMaxMemory": allMaxMemory, + "allDiskUsed": allDiskUsed, + "allTotalCores": allTotalCores, + "allMaxTasks": allMaxTasks, + "allActiveTasks": allActiveTasks, + "allFailedTasks": allFailedTasks, + "allCompletedTasks": allCompletedTasks, + "allTotalTasks": allTotalTasks, + "allTotalDuration": allTotalDuration, + "allTotalGCTime": allTotalGCTime, + "allTotalInputBytes": allTotalInputBytes, + "allTotalShuffleRead": allTotalShuffleRead, + "allTotalShuffleWrite": allTotalShuffleWrite + }; + var activeSummary = { + "execCnt": ( "Active(" + activeExecCnt + ")"), + "allRDDBlocks": activeRDDBlocks, + "allMemoryUsed": activeMemoryUsed, + "allMaxMemory": activeMaxMemory, + "allDiskUsed": activeDiskUsed, + "allTotalCores": activeTotalCores, + "allMaxTasks": activeMaxTasks, + "allActiveTasks": activeActiveTasks, + "allFailedTasks": activeFailedTasks, + "allCompletedTasks": activeCompletedTasks, + "allTotalTasks": activeTotalTasks, + "allTotalDuration": activeTotalDuration, + "allTotalGCTime": activeTotalGCTime, + "allTotalInputBytes": activeTotalInputBytes, + "allTotalShuffleRead": activeTotalShuffleRead, + "allTotalShuffleWrite": activeTotalShuffleWrite + }; + var deadSummary = { + "execCnt": ( "Dead(" + deadExecCnt + ")" ), + "allRDDBlocks": deadRDDBlocks, + "allMemoryUsed": deadMemoryUsed, + "allMaxMemory": deadMaxMemory, + "allDiskUsed": deadDiskUsed, + "allTotalCores": deadTotalCores, + "allMaxTasks": deadMaxTasks, + "allActiveTasks": deadActiveTasks, + "allFailedTasks": deadFailedTasks, + "allCompletedTasks": deadCompletedTasks, + "allTotalTasks": deadTotalTasks, + "allTotalDuration": deadTotalDuration, + "allTotalGCTime": deadTotalGCTime, + "allTotalInputBytes": deadTotalInputBytes, + "allTotalShuffleRead": deadTotalShuffleRead, + "allTotalShuffleWrite": deadTotalShuffleWrite + }; + + var data = {executors: response, "execSummary": [activeSummary, deadSummary, totalSummary]}; + $.get(createTemplateURI(appId), function (template) { + + executorsSummary.append(Mustache.render($(template).filter("#executors-summary-template").html(), data)); + var selector = "#active-executors-table"; + var conf = { + "data": response, + "columns": [ + { + data: function (row, type) { + return type !== 'display' ? (isNaN(row.id) ? 0 : row.id ) : row.id; + } + }, + {data: 'hostPort'}, + {data: 'isActive', render: formatStatus}, + {data: 'rddBlocks'}, + { + data: function (row, type) { + return type === 'display' ? (formatBytes(row.memoryUsed, type) + ' / ' + formatBytes(row.maxMemory, type)) : row.memoryUsed; + } + }, + {data: 'diskUsed', render: formatBytes}, + {data: 'totalCores'}, + { + data: 'activeTasks', + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if (sData > 0) { + $(nTd).css('color', 'white'); + $(nTd).css('background', activeTasksStyle(oData.activeTasks, oData.maxTasks)); + } + } + }, + { + data: 'failedTasks', + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if (sData > 0) { + $(nTd).css('color', 'white'); + $(nTd).css('background', failedTasksStyle(oData.failedTasks, oData.totalTasks)); + } + } + }, + {data: 'completedTasks'}, + {data: 'totalTasks'}, + { + data: function (row, type) { + return type === 'display' ? (formatDuration(row.totalDuration) + ' (' + formatDuration(row.totalGCTime) + ')') : row.totalDuration + }, + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if (oData.totalDuration > 0) { + $(nTd).css('color', totalDurationColor(oData.totalGCTime, oData.totalDuration)); + $(nTd).css('background', totalDurationStyle(oData.totalGCTime, oData.totalDuration)); + } + } + }, + {data: 'totalInputBytes', render: formatBytes}, + {data: 'totalShuffleRead', render: formatBytes}, + {data: 'totalShuffleWrite', render: formatBytes}, + {data: 'executorLogs', render: formatLogsCells}, + { + data: 'id', render: function (data, type) { + return type === 'display' ? ("<a href='threadDump/?executorId=" + data + "'>Thread Dump</a>" ) : data; + } + } + ], + "order": [[0, "asc"]] + }; + + $(selector).DataTable(conf); + $('#active-executors [data-toggle="tooltip"]').tooltip(); + + var sumSelector = "#summary-execs-table"; + var sumConf = { + "data": [activeSummary, deadSummary, totalSummary], + "columns": [ + { + data: 'execCnt', + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + $(nTd).css('font-weight', 'bold'); + } + }, + {data: 'allRDDBlocks'}, + { + data: function (row, type) { + return type === 'display' ? (formatBytes(row.allMemoryUsed, type) + ' / ' + formatBytes(row.allMaxMemory, type)) : row.allMemoryUsed; + } + }, + {data: 'allDiskUsed', render: formatBytes}, + {data: 'allTotalCores'}, + { + data: 'allActiveTasks', + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if (sData > 0) { + $(nTd).css('color', 'white'); + $(nTd).css('background', activeTasksStyle(oData.allActiveTasks, oData.allMaxTasks)); + } + } + }, + { + data: 'allFailedTasks', + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if (sData > 0) { + $(nTd).css('color', 'white'); + $(nTd).css('background', failedTasksStyle(oData.allFailedTasks, oData.allTotalTasks)); + } + } + }, + {data: 'allCompletedTasks'}, + {data: 'allTotalTasks'}, + { + data: function (row, type) { + return type === 'display' ? (formatDuration(row.allTotalDuration, type) + ' (' + formatDuration(row.allTotalGCTime, type) + ')') : row.allTotalDuration + }, + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if (oData.allTotalDuration > 0) { + $(nTd).css('color', totalDurationColor(oData.allTotalGCTime, oData.allTotalDuration)); + $(nTd).css('background', totalDurationStyle(oData.allTotalGCTime, oData.allTotalDuration)); + } + } + }, + {data: 'allTotalInputBytes', render: formatBytes}, + {data: 'allTotalShuffleRead', render: formatBytes}, + {data: 'allTotalShuffleWrite', render: formatBytes} + ], + "paging": false, + "searching": false, + "info": false + + }; + + $(sumSelector).DataTable(sumConf); + $('#execSummary [data-toggle="tooltip"]').tooltip(); + + }); + }); + }); +}); diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index d2161662d5..5b9afb59ef 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -15,28 +15,6 @@ * limitations under the License. */ -// this function works exactly the same as UIUtils.formatDuration -function formatDuration(milliseconds) { - if (milliseconds < 100) { - return milliseconds + " ms"; - } - var seconds = milliseconds * 1.0 / 1000; - if (seconds < 1) { - return seconds.toFixed(1) + " s"; - } - if (seconds < 60) { - return seconds.toFixed(0) + " s"; - } - var minutes = seconds / 60; - if (minutes < 10) { - return minutes.toFixed(1) + " min"; - } else if (minutes < 60) { - return minutes.toFixed(0) + " min"; - } - var hours = minutes / 60; - return hours.toFixed(1) + " h"; -} - function makeIdNumeric(id) { var strs = id.split("_"); if (strs.length < 3) { diff --git a/core/src/main/resources/org/apache/spark/ui/static/utils.js b/core/src/main/resources/org/apache/spark/ui/static/utils.js new file mode 100644 index 0000000000..edc0ee2ce1 --- /dev/null +++ b/core/src/main/resources/org/apache/spark/ui/static/utils.js @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// this function works exactly the same as UIUtils.formatDuration +function formatDuration(milliseconds) { + if (milliseconds < 100) { + return milliseconds + " ms"; + } + var seconds = milliseconds * 1.0 / 1000; + if (seconds < 1) { + return seconds.toFixed(1) + " s"; + } + if (seconds < 60) { + return seconds.toFixed(0) + " s"; + } + var minutes = seconds / 60; + if (minutes < 10) { + return minutes.toFixed(1) + " min"; + } else if (minutes < 60) { + return minutes.toFixed(0) + " min"; + } + var hours = minutes / 60; + return hours.toFixed(1) + " h"; +} + +function formatBytes(bytes, type) { + if (type !== 'display') return bytes; + if (bytes == 0) return '0.0 B'; + var k = 1000; + var dm = 1; + var sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']; + var i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; +} diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala index 2fad1120cd..74f78021ed 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala @@ -43,8 +43,9 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("") { if (allAppsSize > 0) { <script src={UIUtils.prependBaseUri("/static/dataTables.rowsGroup.js")}></script> ++ - <div id="history-summary" class="span12 pagination"></div> ++ - <script src={UIUtils.prependBaseUri("/static/historypage.js")}> </script> + <div id="history-summary" class="span12 pagination"></div> ++ + <script src={UIUtils.prependBaseUri("/static/utils.js")}></script> ++ + <script src={UIUtils.prependBaseUri("/static/historypage.js")}></script> } else if (requestedIncomplete) { <h4>No incomplete applications found!</h4> } else { diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/AllExecutorListResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/AllExecutorListResource.scala new file mode 100644 index 0000000000..01f2a18122 --- /dev/null +++ b/core/src/main/scala/org/apache/spark/status/api/v1/AllExecutorListResource.scala @@ -0,0 +1,41 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.spark.status.api.v1 + +import javax.ws.rs.{GET, Produces} +import javax.ws.rs.core.MediaType + +import org.apache.spark.ui.SparkUI +import org.apache.spark.ui.exec.ExecutorsPage + +@Produces(Array(MediaType.APPLICATION_JSON)) +private[v1] class AllExecutorListResource(ui: SparkUI) { + + @GET + def executorList(): Seq[ExecutorSummary] = { + val listener = ui.executorsListener + listener.synchronized { + // The follow codes should be protected by `listener` to make sure no executors will be + // removed before we query their status. See SPARK-12784. + (0 until listener.activeStorageStatusList.size).map { statusId => + ExecutorsPage.getExecInfo(listener, statusId, isActive = true) + } ++ (0 until listener.deadStorageStatusList.size).map { statusId => + ExecutorsPage.getExecInfo(listener, statusId, isActive = false) + } + } + } +} diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala index 681f295006..de927117e1 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala @@ -91,6 +91,13 @@ private[v1] class ApiRootResource extends UIRootFromServletContext { } } + @Path("applications/{appId}/allexecutors") + def getAllExecutors(@PathParam("appId") appId: String): AllExecutorListResource = { + uiRoot.withSparkUI(appId, None) { ui => + new AllExecutorListResource(ui) + } + } + @Path("applications/{appId}/{attemptId}/executors") def getExecutors( @PathParam("appId") appId: String, @@ -100,6 +107,15 @@ private[v1] class ApiRootResource extends UIRootFromServletContext { } } + @Path("applications/{appId}/{attemptId}/allexecutors") + def getAllExecutors( + @PathParam("appId") appId: String, + @PathParam("attemptId") attemptId: String): AllExecutorListResource = { + uiRoot.withSparkUI(appId, Some(attemptId)) { ui => + new AllExecutorListResource(ui) + } + } + @Path("applications/{appId}/stages") def getStages(@PathParam("appId") appId: String): AllStagesResource = { diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 740f5e5f7f..2b6c538485 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -201,7 +201,8 @@ private[spark] object UIUtils extends Logging { activeTab: SparkUITab, refreshInterval: Option[Int] = None, helpText: Option[String] = None, - showVisualization: Boolean = false): Seq[Node] = { + showVisualization: Boolean = false, + useDataTables: Boolean = false): Seq[Node] = { val appName = activeTab.appName val shortAppName = if (appName.length < 36) appName else appName.take(32) + "..." @@ -216,6 +217,7 @@ private[spark] object UIUtils extends Logging { <head> {commonHeaderNodes} {if (showVisualization) vizHeaderNodes else Seq.empty} + {if (useDataTables) dataTablesHeaderNodes else Seq.empty} <title>{appName} - {title}</title> </head> <body> diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index 67deb7b14b..287390b87b 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -20,7 +20,6 @@ package org.apache.spark.ui.exec import java.net.URLEncoder import javax.servlet.http.HttpServletRequest -import scala.util.Try import scala.xml.Node import org.apache.spark.status.api.v1.ExecutorSummary @@ -54,285 +53,17 @@ private[ui] class ExecutorsPage( // When GCTimePercent is edited change ToolTips.TASK_TIME to match private val GCTimePercent = 0.1 - // a safe String to Int for sorting ids (converts non-numeric Strings to -1) - private def idStrToInt(str: String) : Int = Try(str.toInt).getOrElse(-1) - def render(request: HttpServletRequest): Seq[Node] = { - val (activeExecutorInfo, deadExecutorInfo) = listener.synchronized { - // The follow codes should be protected by `listener` to make sure no executors will be - // removed before we query their status. See SPARK-12784. - val _activeExecutorInfo = { - for (statusId <- 0 until listener.activeStorageStatusList.size) - yield ExecutorsPage.getExecInfo(listener, statusId, isActive = true) - } - val _deadExecutorInfo = { - for (statusId <- 0 until listener.deadStorageStatusList.size) - yield ExecutorsPage.getExecInfo(listener, statusId, isActive = false) - } - (_activeExecutorInfo, _deadExecutorInfo) - } - - val execInfo = activeExecutorInfo ++ deadExecutorInfo - implicit val idOrder = Ordering[Int].on((s: String) => idStrToInt(s)).reverse - val execInfoSorted = execInfo.sortBy(_.id) - val logsExist = execInfo.filter(_.executorLogs.nonEmpty).nonEmpty - - val execTable = { - <table class={UIUtils.TABLE_CLASS_STRIPED_SORTABLE}> - <thead> - <th class="sorttable_numeric">Executor ID</th> - <th>Address</th> - <th>Status</th> - <th>RDD Blocks</th> - <th><span data-toggle="tooltip" title={ToolTips.STORAGE_MEMORY}>Storage Memory</span></th> - <th>Disk Used</th> - <th>Cores</th> - <th>Active Tasks</th> - <th>Failed Tasks</th> - <th>Complete Tasks</th> - <th>Total Tasks</th> - <th><span data-toggle="tooltip" title={ToolTips.TASK_TIME}>Task Time (GC Time)</span></th> - <th><span data-toggle="tooltip" title={ToolTips.INPUT}>Input</span></th> - <th><span data-toggle="tooltip" title={ToolTips.SHUFFLE_READ}>Shuffle Read</span></th> - <th> - <!-- Place the shuffle write tooltip on the left (rather than the default position - of on top) because the shuffle write column is the last column on the right side and - the tooltip is wider than the column, so it doesn't fit on top. --> - <span data-toggle="tooltip" data-placement="left" title={ToolTips.SHUFFLE_WRITE}> - Shuffle Write - </span> - </th> - {if (logsExist) <th class="sorttable_nosort">Logs</th> else Seq.empty} - {if (threadDumpEnabled) <th class="sorttable_nosort">Thread Dump</th> else Seq.empty} - </thead> - <tbody> - {execInfoSorted.map(execRow(_, logsExist))} - </tbody> - </table> - } - val content = - <div class="row"> - <div class="span12"> - <h4>Summary</h4> - {execSummary(activeExecutorInfo, deadExecutorInfo)} - </div> - </div> - <div class = "row"> - <div class="span12"> - <h4>Executors</h4> - {execTable} - </div> - </div>; - - UIUtils.headerSparkPage("Executors", content, parent) - } - - /** Render an HTML row representing an executor */ - private def execRow(info: ExecutorSummary, logsExist: Boolean): Seq[Node] = { - val maximumMemory = info.maxMemory - val memoryUsed = info.memoryUsed - val diskUsed = info.diskUsed - val executorStatus = - if (info.isActive) { - "Active" - } else { - "Dead" - } - - <tr> - <td sorttable_customkey={idStrToInt(info.id).toString}>{info.id}</td> - <td>{info.hostPort}</td> - <td sorttable_customkey={executorStatus.toString}> - {executorStatus} - </td> - <td>{info.rddBlocks}</td> - <td sorttable_customkey={memoryUsed.toString}> - {Utils.bytesToString(memoryUsed)} / - {Utils.bytesToString(maximumMemory)} - </td> - <td sorttable_customkey={diskUsed.toString}> - {Utils.bytesToString(diskUsed)} - </td> - <td>{info.totalCores}</td> - {taskData(info.maxTasks, info.activeTasks, info.failedTasks, info.completedTasks, - info.totalTasks, info.totalDuration, info.totalGCTime)} - <td sorttable_customkey={info.totalInputBytes.toString}> - {Utils.bytesToString(info.totalInputBytes)} - </td> - <td sorttable_customkey={info.totalShuffleRead.toString}> - {Utils.bytesToString(info.totalShuffleRead)} - </td> - <td sorttable_customkey={info.totalShuffleWrite.toString}> - {Utils.bytesToString(info.totalShuffleWrite)} - </td> - { - if (logsExist) { - <td> - { - info.executorLogs.map { case (logName, logUrl) => - <div> - <a href={logUrl}> - {logName} - </a> - </div> - } - } - </td> + <div> + { + <div id="active-executors"></div> ++ + <script src={UIUtils.prependBaseUri("/static/utils.js")}></script> ++ + <script src={UIUtils.prependBaseUri("/static/executorspage.js")}></script> } - } - { - if (threadDumpEnabled) { - if (info.isActive) { - val encodedId = URLEncoder.encode(info.id, "UTF-8") - <td> - <a href={s"threadDump/?executorId=${encodedId}"}>Thread Dump</a> - </td> - } else { - <td> </td> - } - } else { - Seq.empty - } - } - </tr> - } - - private def execSummaryRow(execInfo: Seq[ExecutorSummary], rowName: String): Seq[Node] = { - val maximumMemory = execInfo.map(_.maxMemory).sum - val memoryUsed = execInfo.map(_.memoryUsed).sum - val diskUsed = execInfo.map(_.diskUsed).sum - val totalCores = execInfo.map(_.totalCores).sum - val totalInputBytes = execInfo.map(_.totalInputBytes).sum - val totalShuffleRead = execInfo.map(_.totalShuffleRead).sum - val totalShuffleWrite = execInfo.map(_.totalShuffleWrite).sum - - <tr> - <td><b>{rowName}({execInfo.size})</b></td> - <td>{execInfo.map(_.rddBlocks).sum}</td> - <td sorttable_customkey={memoryUsed.toString}> - {Utils.bytesToString(memoryUsed)} / - {Utils.bytesToString(maximumMemory)} - </td> - <td sorttable_customkey={diskUsed.toString}> - {Utils.bytesToString(diskUsed)} - </td> - <td>{totalCores}</td> - {taskData(execInfo.map(_.maxTasks).sum, - execInfo.map(_.activeTasks).sum, - execInfo.map(_.failedTasks).sum, - execInfo.map(_.completedTasks).sum, - execInfo.map(_.totalTasks).sum, - execInfo.map(_.totalDuration).sum, - execInfo.map(_.totalGCTime).sum)} - <td sorttable_customkey={totalInputBytes.toString}> - {Utils.bytesToString(totalInputBytes)} - </td> - <td sorttable_customkey={totalShuffleRead.toString}> - {Utils.bytesToString(totalShuffleRead)} - </td> - <td sorttable_customkey={totalShuffleWrite.toString}> - {Utils.bytesToString(totalShuffleWrite)} - </td> - </tr> - } - - private def execSummary(activeExecInfo: Seq[ExecutorSummary], deadExecInfo: Seq[ExecutorSummary]): - Seq[Node] = { - val totalExecInfo = activeExecInfo ++ deadExecInfo - val activeRow = execSummaryRow(activeExecInfo, "Active"); - val deadRow = execSummaryRow(deadExecInfo, "Dead"); - val totalRow = execSummaryRow(totalExecInfo, "Total"); - - <table class={UIUtils.TABLE_CLASS_STRIPED}> - <thead> - <th></th> - <th>RDD Blocks</th> - <th><span data-toggle="tooltip" title={ToolTips.STORAGE_MEMORY}>Storage Memory</span></th> - <th>Disk Used</th> - <th>Cores</th> - <th>Active Tasks</th> - <th>Failed Tasks</th> - <th>Complete Tasks</th> - <th>Total Tasks</th> - <th><span data-toggle="tooltip" title={ToolTips.TASK_TIME}>Task Time (GC Time)</span></th> - <th><span data-toggle="tooltip" title={ToolTips.INPUT}>Input</span></th> - <th><span data-toggle="tooltip" title={ToolTips.SHUFFLE_READ}>Shuffle Read</span></th> - <th> - <span data-toggle="tooltip" data-placement="left" title={ToolTips.SHUFFLE_WRITE}> - Shuffle Write - </span> - </th> - </thead> - <tbody> - {activeRow} - {deadRow} - {totalRow} - </tbody> - </table> - } - - private def taskData( - maxTasks: Int, - activeTasks: Int, - failedTasks: Int, - completedTasks: Int, - totalTasks: Int, - totalDuration: Long, - totalGCTime: Long): Seq[Node] = { - // Determine Color Opacity from 0.5-1 - // activeTasks range from 0 to maxTasks - val activeTasksAlpha = - if (maxTasks > 0) { - (activeTasks.toDouble / maxTasks) * 0.5 + 0.5 - } else { - 1 - } - // failedTasks range max at 10% failure, alpha max = 1 - val failedTasksAlpha = - if (totalTasks > 0) { - math.min(10 * failedTasks.toDouble / totalTasks, 1) * 0.5 + 0.5 - } else { - 1 - } - // totalDuration range from 0 to 50% GC time, alpha max = 1 - val totalDurationAlpha = - if (totalDuration > 0) { - math.min(totalGCTime.toDouble / totalDuration + 0.5, 1) - } else { - 1 - } - - val tableData = - <td style={ - if (activeTasks > 0) { - "background:hsla(240, 100%, 50%, " + activeTasksAlpha + ");color:white" - } else { - "" - } - }>{activeTasks}</td> - <td style={ - if (failedTasks > 0) { - "background:hsla(0, 100%, 50%, " + failedTasksAlpha + ");color:white" - } else { - "" - } - }>{failedTasks}</td> - <td>{completedTasks}</td> - <td>{totalTasks}</td> - <td sorttable_customkey={totalDuration.toString} style={ - // Red if GC time over GCTimePercent of total time - if (totalGCTime > GCTimePercent * totalDuration) { - "background:hsla(0, 100%, 50%, " + totalDurationAlpha + ");color:white" - } else { - "" - } - }> - {Utils.msDurationToString(totalDuration)} - ({Utils.msDurationToString(totalGCTime)}) - </td>; + </div>; - tableData + UIUtils.headerSparkPage("Executors", content, parent, useDataTables = true) } } diff --git a/docs/monitoring.md b/docs/monitoring.md index ee932cfc6d..c8694762ff 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -289,7 +289,11 @@ can be identified by their `[attempt-id]`. In the API listed below, when running </tr> <tr> <td><code>/applications/[app-id]/executors</code></td> - <td>A list of all executors for the given application.</td> + <td>A list of all active executors for the given application.</td> + </tr> + <tr> + <td><code>/applications/[app-id]/allexecutors</code></td> + <td>A list of all(active and dead) executors for the given application.</td> </tr> <tr> <td><code>/applications/[app-id]/storage/rdd</code></td> |