From f15b79c9093127ca3992f55d9eb594dbff7c6e1c Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Thu, 21 May 2026 19:37:30 +0000 Subject: [PATCH 1/4] Added overview tables to main page Added tables that show general instance information plus global ingest, scan, and compaction activity. --- .../accumulo/monitor/next/Endpoints.java | 11 +- .../monitor/next/SystemInformation.java | 312 +++++++++++++++++- .../monitor/resources/js/functions.js | 13 +- .../accumulo/monitor/resources/js/overview.js | 90 ++++- .../accumulo/monitor/templates/overview.ftl | 42 ++- 5 files changed, 441 insertions(+), 27 deletions(-) diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java index d48173f821d..1e0c2d32b09 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java @@ -59,6 +59,7 @@ import org.apache.accumulo.monitor.next.SystemInformation.CompactionGroupSummary; import org.apache.accumulo.monitor.next.SystemInformation.CompactionTableSummary; import org.apache.accumulo.monitor.next.SystemInformation.FateTransaction; +import org.apache.accumulo.monitor.next.SystemInformation.InstanceOverview; import org.apache.accumulo.monitor.next.SystemInformation.MessageCategory; import org.apache.accumulo.monitor.next.SystemInformation.MessagePriority; import org.apache.accumulo.monitor.next.SystemInformation.RecoveryInformation; @@ -186,7 +187,7 @@ public MonitorStatus getStatus() { } @GET - @Path("instance") + @Path("instance/info") @Produces(MediaType.APPLICATION_JSON) @Description("Returns the instance name, instance id, version, zookeepers, and volumes") public InstanceSummary getInstanceSummary() { @@ -198,6 +199,14 @@ public InstanceSummary getInstanceSummary() { Constants.VERSION); } + @GET + @Path("instance/overview") + @Produces(MediaType.APPLICATION_JSON) + @Description("Returns an overview of instance level activity") + public InstanceOverview getInstanceOverview() { + return monitor.getInformationFetcher().getSummaryForEndpoint().getInstanceOverview(); + } + @GET @Path("compactors/detail/{" + GROUP_PARAM_KEY + "}") @Produces(MediaType.APPLICATION_JSON) diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java index e27991799b0..dde0d0786a9 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java @@ -55,6 +55,8 @@ import java.util.stream.Stream; import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.admin.TabletAvailability; import org.apache.accumulo.core.client.admin.TabletInformation; @@ -111,6 +113,159 @@ public class SystemInformation { + public static class InstanceOverview { + private final AtomicLong numKVs = new AtomicLong(0); + private final AtomicLong numFiles = new AtomicLong(0); + private final AtomicLong numTables = new AtomicLong(0); + private final AtomicLong numNamespaces = new AtomicLong(0); + private final AtomicLong numTablets = new AtomicLong(0); + private final AtomicLong totalFileSize = new AtomicLong(0); + private final AtomicLong tabletsAssignedToDeadTServers = new AtomicLong(0); + private final AtomicLong totalSuspendedTablets = new AtomicLong(0); + private final AtomicLong tabletsNeedingRecovery = new AtomicLong(0); + private final AtomicLong compactionsQueued = new AtomicLong(0); + private final AtomicLong compactionsDequeued = new AtomicLong(0); + private final AtomicLong compactionsRunning = new AtomicLong(0); + private final AtomicLong compactionsFailed = new AtomicLong(0); + private final AtomicLong scansTotalInProgress = new AtomicLong(0); + private final AtomicLong scanTotalOpenFiles = new AtomicLong(0); + private final AtomicLong scansTotalKvScanned = new AtomicLong(0); + private final AtomicLong scansTotalKvReturned = new AtomicLong(0); + private final AtomicLong scansTotalKvReturnedBytes = new AtomicLong(0); + private final AtomicLong ingestTotalEntries = new AtomicLong(0); + private final AtomicLong ingestTotalEntriesBytes = new AtomicLong(0); + private final AtomicLong ingestNumTServersHolding = new AtomicLong(0); + private final AtomicLong ingestTotalEntriesInMem = new AtomicLong(0); + private final AtomicLong ingestBulkImportQueued = new AtomicLong(0); + private final AtomicLong ingestBulkImportRunning = new AtomicLong(0); + private final AtomicLong totalMinCQueued = new AtomicLong(0); + private final AtomicLong totalMinCRunning = new AtomicLong(0); + private final AtomicLong totalMinCCompleted = new AtomicLong(0); + private final AtomicLong totalFateSubmitted = new AtomicLong(0); + private final AtomicLong totalFateRunning = new AtomicLong(0); + private final AtomicLong totalServersLowMem = new AtomicLong(0); + + public AtomicLong getNumKVs() { + return numKVs; + } + + public AtomicLong getNumFiles() { + return numFiles; + } + + public AtomicLong getNumTablets() { + return numTablets; + } + + public AtomicLong getNumTables() { + return numTables; + } + + public AtomicLong getNumNamespaces() { + return numNamespaces; + } + + public AtomicLong getTotalFileSize() { + return totalFileSize; + } + + public AtomicLong getTabletsAssignedToDeadTServers() { + return tabletsAssignedToDeadTServers; + } + + public AtomicLong getTotalSuspendedTablets() { + return totalSuspendedTablets; + } + + public AtomicLong getTabletsNeedingRecovery() { + return tabletsNeedingRecovery; + } + + public AtomicLong getCompactionsQueued() { + return compactionsQueued; + } + + public AtomicLong getCompactionsDequeued() { + return compactionsDequeued; + } + + public AtomicLong getCompactionsRunning() { + return compactionsRunning; + } + + public AtomicLong getCompactionsFailed() { + return compactionsFailed; + } + + public AtomicLong getScansTotalInProgress() { + return scansTotalInProgress; + } + + public AtomicLong getScanTotalOpenFiles() { + return scanTotalOpenFiles; + } + + public AtomicLong getScansTotalKvScanned() { + return scansTotalKvScanned; + } + + public AtomicLong getScansTotalKvReturned() { + return scansTotalKvReturned; + } + + public AtomicLong getScansTotalKvReturnedBytes() { + return scansTotalKvReturnedBytes; + } + + public AtomicLong getIngestTotalEntries() { + return ingestTotalEntries; + } + + public AtomicLong getIngestTotalEntriesBytes() { + return ingestTotalEntriesBytes; + } + + public AtomicLong getIngestNumTServersHolding() { + return ingestNumTServersHolding; + } + + public AtomicLong getIngestTotalEntriesInMem() { + return ingestTotalEntriesInMem; + } + + public AtomicLong getIngestBulkImportQueued() { + return ingestBulkImportQueued; + } + + public AtomicLong getIngestBulkImportRunning() { + return ingestBulkImportRunning; + } + + public AtomicLong getTotalMinCQueued() { + return totalMinCQueued; + } + + public AtomicLong getTotalMinCRunning() { + return totalMinCRunning; + } + + public AtomicLong getTotalMinCCompleted() { + return totalMinCCompleted; + } + + public AtomicLong getTotalFateSubmitted() { + return totalFateSubmitted; + } + + public AtomicLong getTotalFateRunning() { + return totalFateRunning; + } + + public AtomicLong getTotalServersLowMem() { + return totalServersLowMem; + } + } + public static class ObfuscatedTabletId extends TabletIdImpl { public ObfuscatedTabletId(KeyExtent ke) { @@ -261,12 +416,16 @@ public String getTableName() { return tableName; } - public void addTablet(TabletInformation info) { + public void addTablet(TabletInformation info, InstanceOverview io) { totalEntries.addAndGet(info.getEstimatedEntries()); + io.getNumKVs().addAndGet(info.getEstimatedEntries()); totalSizeOnDisk.addAndGet(info.getEstimatedSize()); + io.getTotalFileSize().addAndGet(info.getEstimatedSize()); totalFiles.addAndGet(info.getNumFiles()); + io.getNumFiles().addAndGet(info.getNumFiles()); totalWals.addAndGet(info.getNumWalLogs()); totalTablets.addAndGet(1); + io.getNumTablets().incrementAndGet(); switch (info.getTabletAvailability()) { case HOSTED: availableAlways.addAndGet(1); @@ -288,12 +447,14 @@ public void addTablet(TabletInformation info) { break; case ASSIGNED_TO_DEAD_SERVER: totalAssignedToDeadServerTablets.addAndGet(1); + io.getTabletsAssignedToDeadTServers().incrementAndGet(); break; case HOSTED: totalHostedTablets.addAndGet(1); break; case SUSPENDED: totalSuspendedTablets.addAndGet(1); + io.getTotalSuspendedTablets().incrementAndGet(); break; case UNASSIGNED: totalUnassignedTablets.addAndGet(1); @@ -549,6 +710,8 @@ public record FateTransaction(FateInstanceType type, FateOperation op, String id private String managerGoalState; private final int rgLongRunningCompactionSize; + private final InstanceOverview instanceOverview = new InstanceOverview(); + public SystemInformation(Cache allMetrics, ServerContext ctx) { this.allMetrics = allMetrics; this.ctx = ctx; @@ -803,21 +966,48 @@ public void processResponse(final ServerId server, final MetricResponse response deployment.computeIfAbsent(server.getResourceGroup(), g -> new ConcurrentHashMap<>()) .computeIfAbsent(server.getType(), t -> new ProcessSummary()).addResponded(server); captureRecoveriesInProgress(server, response); + FMetric flatbuffer = new FMetric(); switch (response.serverType) { case COMPACTOR: compactors .computeIfAbsent(response.getResourceGroup(), (rg) -> ConcurrentHashMap.newKeySet()) .add(server); updateAggregates(response, totalCompactorMetrics, rgCompactorMetrics); + for (ByteBuffer binary : response.getMetrics()) { + flatbuffer = FMetric.getRootAsFMetric(binary, flatbuffer); + final String metricName = flatbuffer.name(); + if (metricName.equals(Metric.COMPACTOR_MAJC_IN_PROGRESS.getName())) { + long running = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getCompactionsRunning().addAndGet(running); + } else if (metricName.equals(Metric.COMPACTOR_MAJC_FAILED.getName())) { + long failed = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getCompactionsFailed().addAndGet(failed); + } else if (metricName.equals(Metric.LOW_MEMORY.getName())) { + long lowMem = getMetricValue(flatbuffer).longValue(); + if (lowMem > 0) { + this.instanceOverview.getTotalServersLowMem().incrementAndGet(); + } + } + } break; case GARBAGE_COLLECTOR: if (gc.get() == null || !gc.get().equals(server)) { gc.set(server); } + for (ByteBuffer binary : response.getMetrics()) { + flatbuffer = FMetric.getRootAsFMetric(binary, flatbuffer); + final String metricName = flatbuffer.name(); + if (metricName.equals(Metric.LOW_MEMORY.getName())) { + long lowMem = getMetricValue(flatbuffer).longValue(); + if (lowMem > 0) { + this.instanceOverview.getTotalServersLowMem().incrementAndGet(); + } + break; + } + } break; case MANAGER: managers.add(server); - FMetric flatbuffer = new FMetric(); for (ByteBuffer binary : response.getMetrics()) { flatbuffer = FMetric.getRootAsFMetric(binary, flatbuffer); final String metricName = flatbuffer.name(); @@ -825,11 +1015,13 @@ public void processResponse(final ServerId server, final MetricResponse response boolean recovering = getMetricValue(flatbuffer).longValue() > 0; this.recoveries.getOverview().setRootTabletRecovering(recovering); if (recovering) { + this.instanceOverview.getTabletsNeedingRecovery().incrementAndGet(); addMessage(Critical, Table, "The root table requires recovery"); } } else if (metricName.equals(Metric.MANAGER_META_TGW_RECOVERY.getName())) { long tablets = getMetricValue(flatbuffer).longValue(); this.recoveries.getOverview().setMetadataTabletsRecovering(tablets); + this.instanceOverview.getTabletsNeedingRecovery().addAndGet(tablets); if (tablets > 0) { addMessage(Critical, Table, "At least " + tablets + " metadata table tablets require recovery"); @@ -837,10 +1029,23 @@ public void processResponse(final ServerId server, final MetricResponse response } else if (metricName.equals(Metric.MANAGER_USER_TGW_RECOVERY.getName())) { long tablets = getMetricValue(flatbuffer).longValue(); this.recoveries.getOverview().setUserTabletsRecovering(tablets); + this.instanceOverview.getTabletsNeedingRecovery().addAndGet(tablets); if (tablets > 0) { addMessage(High, Table, "At least " + tablets + " user table tablets require recovery"); } + } else if (metricName.equals(Metric.COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_QUEUED.getName())) { + long queued = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getCompactionsQueued().addAndGet(queued); + } else if (metricName + .equals(Metric.COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_DEQUEUED.getName())) { + long dequeued = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getCompactionsDequeued().addAndGet(dequeued); + } else if (metricName.equals(Metric.LOW_MEMORY.getName())) { + long lowMem = getMetricValue(flatbuffer).longValue(); + if (lowMem > 0) { + this.instanceOverview.getTotalServersLowMem().incrementAndGet(); + } } } break; @@ -848,11 +1053,84 @@ public void processResponse(final ServerId server, final MetricResponse response sservers.computeIfAbsent(response.getResourceGroup(), (rg) -> ConcurrentHashMap.newKeySet()) .add(server); updateAggregates(response, totalSServerMetrics, rgSServerMetrics); + for (ByteBuffer binary : response.getMetrics()) { + flatbuffer = FMetric.getRootAsFMetric(binary, flatbuffer); + final String metricName = flatbuffer.name(); + if (metricName.equals(Metric.SCAN_CONTINUE.getName())) { + long inProgress = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getScansTotalInProgress().addAndGet(inProgress); + } else if (metricName.equals(Metric.SCAN_OPEN_FILES.getName())) { + long files = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getScanTotalOpenFiles().addAndGet(files); + } else if (metricName.equals(Metric.SCAN_SCANNED_ENTRIES.getName())) { + long scanned = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getScansTotalKvScanned().addAndGet(scanned); + } else if (metricName.equals(Metric.SCAN_QUERY_SCAN_RESULTS.getName())) { + long results = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getScansTotalKvReturned().addAndGet(results); + } else if (metricName.equals(Metric.SCAN_QUERY_SCAN_RESULTS_BYTES.getName())) { + long resultSize = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getScansTotalKvReturnedBytes().addAndGet(resultSize); + } else if (metricName.equals(Metric.LOW_MEMORY.getName())) { + long lowMem = getMetricValue(flatbuffer).longValue(); + if (lowMem > 0) { + this.instanceOverview.getTotalServersLowMem().incrementAndGet(); + } + } + } break; case TABLET_SERVER: tservers.computeIfAbsent(response.getResourceGroup(), (rg) -> ConcurrentHashMap.newKeySet()) .add(server); updateAggregates(response, totalTServerMetrics, rgTServerMetrics); + for (ByteBuffer binary : response.getMetrics()) { + flatbuffer = FMetric.getRootAsFMetric(binary, flatbuffer); + final String metricName = flatbuffer.name(); + if (metricName.equals(Metric.SCAN_CONTINUE.getName())) { + long inProgress = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getScansTotalInProgress().addAndGet(inProgress); + } else if (metricName.equals(Metric.SCAN_OPEN_FILES.getName())) { + long files = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getScanTotalOpenFiles().addAndGet(files); + } else if (metricName.equals(Metric.SCAN_SCANNED_ENTRIES.getName())) { + long scanned = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getScansTotalKvScanned().addAndGet(scanned); + } else if (metricName.equals(Metric.SCAN_QUERY_SCAN_RESULTS.getName())) { + long results = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getScansTotalKvReturned().addAndGet(results); + } else if (metricName.equals(Metric.SCAN_QUERY_SCAN_RESULTS_BYTES.getName())) { + long resultSize = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getScansTotalKvReturnedBytes().addAndGet(resultSize); + } else if (metricName.equals(Metric.TSERVER_HOLD.getName())) { + long held = getMetricValue(flatbuffer).longValue(); + if (held > 0) { + this.instanceOverview.ingestNumTServersHolding.incrementAndGet(); + } + } else if (metricName.equals(Metric.TSERVER_INGEST_ENTRIES.getName())) { + long ingestEntries = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getIngestTotalEntries().addAndGet(ingestEntries); + } else if (metricName.equals(Metric.TSERVER_INGEST_BYTES.getName())) { + long ingestEntriesBytes = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getIngestTotalEntriesBytes().addAndGet(ingestEntriesBytes); + } else if (metricName.equals(Metric.TSERVER_MEM_ENTRIES.getName())) { + long entriesInMem = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getIngestTotalEntriesInMem().addAndGet(entriesInMem); + } else if (metricName.equals(Metric.TSERVER_MINC_QUEUED.getName())) { + long mincQueued = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getTotalMinCQueued().addAndGet(mincQueued); + } else if (metricName.equals(Metric.TSERVER_MINC_RUNNING.getName())) { + long mincRunning = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getTotalMinCRunning().addAndGet(mincRunning); + } else if (metricName.equals(Metric.TSERVER_MINC_TOTAL.getName())) { + long mincCompleted = getMetricValue(flatbuffer).longValue(); + this.instanceOverview.getTotalMinCCompleted().addAndGet(mincCompleted); + } else if (metricName.equals(Metric.LOW_MEMORY.getName())) { + long lowMem = getMetricValue(flatbuffer).longValue(); + if (lowMem > 0) { + this.instanceOverview.getTotalServersLowMem().incrementAndGet(); + } + } + } break; default: LOG.error("Unhandled server type in fetch metric response: {}", response.serverType); @@ -884,7 +1162,8 @@ public void processTabletInformation(TableId tableId, String tableName, TabletIn final SanitizedTabletInformation sti = new SanitizedTabletInformation(info); tablets.computeIfAbsent(tableId, (t) -> Collections.synchronizedList(new ArrayList<>())) .add(sti); - tables.computeIfAbsent(tableId, (t) -> new TableSummary(tableName)).addTablet(sti); + tables.computeIfAbsent(tableId, (t) -> new TableSummary(tableName)).addTablet(sti, + instanceOverview); if (sti.getNumWalLogs() > 0) { String loc = sti.getLocation().orElse(""); int idx = loc.indexOf(':'); @@ -926,9 +1205,22 @@ public void processTabletInformation(TableId tableId, String tableName, TabletIn public void processFateTransactions(List transactions) { transactions.forEach(t -> { + FateOperation op = t.getFateOp(); + TStatus status = t.getStatus(); + if (status == TStatus.SUBMITTED) { + instanceOverview.getTotalFateSubmitted().incrementAndGet(); + if (op == FateOperation.TABLE_BULK_IMPORT2) { + instanceOverview.getIngestBulkImportQueued().incrementAndGet(); + } + } else if (status == TStatus.IN_PROGRESS) { + instanceOverview.getTotalFateRunning().incrementAndGet(); + if (op == FateOperation.TABLE_BULK_IMPORT2) { + instanceOverview.getIngestBulkImportRunning().incrementAndGet(); + } + } fateTransactions - .add(new FateTransaction(t.getInstanceType(), t.getFateOp(), t.getFateId().getTxUUIDStr(), - t.getStatus(), t.getTimeCreated(), t.getHeldLocks(), t.getWaitingLocks(), + .add(new FateTransaction(t.getInstanceType(), op, t.getFateId().getTxUUIDStr(), status, + t.getTimeCreated(), t.getHeldLocks(), t.getWaitingLocks(), t.getLockRange().isInfinite() ? LockRangeType.FULL : LockRangeType.PARTIAL)); }); } @@ -1226,6 +1518,12 @@ public void finish(final List failures, break; } } + try { + this.instanceOverview.getNumNamespaces().set(ctx.namespaceOperations().list().size()); + } catch (AccumuloException | AccumuloSecurityException e1) { + LOG.error("Error getting list of namespaces for Monitor overview", e1); + } + this.instanceOverview.getNumTables().set(this.tables.size()); deploymentOverview = DeploymentOverview.fromSummary(deployment, timestamp); computeMessageCounts(); } @@ -1411,6 +1709,10 @@ public long getTimestamp() { return this.timestamp.get(); } + public InstanceOverview getInstanceOverview() { + return this.instanceOverview; + } + /** * Cache a ServersView for the given table and set of servers. */ diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js index 05b0613ee43..c4cb2197bc8 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js @@ -43,6 +43,7 @@ const MESSAGES = 'messages'; const FATE = 'fate'; const MESSAGE_COUNTS = 'messageCounts' const RECOVERY = 'recovery'; +const INSTANCE_OVERVIEW = 'instanceOverview'; // Override Length Menu options for dataTables if ($.fn && $.fn.dataTable) { @@ -779,11 +780,19 @@ function getTserversSummary() { } /** - * REST GET call for /instance, + * REST GET call for /instance/info, * stores it on a sessionStorage variable */ function getInstanceInfo() { - return getJSONForTable(REST_V2_PREFIX + '/instance', 'instance'); + return getJSONForTable(REST_V2_PREFIX + '/instance/info', 'instance'); +} + +/** + * REST GET call for /instance/overview, + * stores it on a sessionStorage variable + */ +function getInstanceOverview() { + return getJSONForTable(REST_V2_PREFIX + '/instance/overview', INSTANCE_OVERVIEW); } /** diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/overview.js b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/overview.js index e0f7bc43900..8ae21ec0010 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/overview.js +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/overview.js @@ -35,7 +35,10 @@ $(function () { * Makes the REST calls, generates the table with the new information */ function refreshOverview() { - refreshDeploymentTables(); + $.when(getDeployment(), getInstanceOverview()).then(function () { + refreshInstanceOverviewTables(); + refreshDeploymentTables(); + }); } /** @@ -45,24 +48,83 @@ function refresh() { refreshOverview(); } +function refreshInstanceOverviewTables() { + var data = getStoredJson(INSTANCE_OVERVIEW, {}); + + var table = $('#instance-overview-table'); + $(table).find('thead').remove(); + var body = $(table).find('tbody'); + body.empty(); + body.append(createRow("Namespaces", bigNumberForQuantity(data.numNamespaces || 0))); + body.append(createRow("Tables", bigNumberForQuantity(data.numTables || 0))); + body.append(createRow("Tablets", bigNumberForQuantity(data.numTablets || 0))); + body.append(createRow("Entries", bigNumberForQuantity(data.numKVs || 0))); + body.append(createRow("Files", bigNumberForQuantity(data.numFiles || 0))); + body.append(createRow("File Size", bigNumberForSize(data.totalFileSize || 0))); + body.append(createRow("Tablets Assigned To Dead TServers", bigNumberForQuantity(data.tabletsAssignedToDeadTServers || 0))); + body.append(createRow("Tablets Suspended", bigNumberForQuantity(data.totalSuspendedTablets || 0))); + body.append(createRow("Tablets Requiring Recovery", bigNumberForQuantity(data.tabletsNeedingRecovery || 0))); + body.append(createRow("Fate Tx Queued", bigNumberForQuantity(data.totalFateSubmitted || 0))); + body.append(createRow("Fate Tx Running", bigNumberForQuantity(data.totalFateRunning || 0))); + body.append(createRow("Servers Low On Memory", bigNumberForQuantity(data.totalServersLowMem || 0))); + + var table = $('#ingest-overview-table'); + $(table).find('thead').remove(); + var body = $(table).find('tbody'); + body.empty(); + body.append(createRow("Entries", bigNumberForQuantity(data.ingestTotalEntries || 0))); + body.append(createRow("Bytes", bigNumberForSize(data.ingestTotalEntriesBytes || 0))); + body.append(createRow("Entries In Memory", bigNumberForQuantity(data.ingestTotalEntriesInMem || 0))); + body.append(createRow("TServers Holding For MinC", bigNumberForQuantity(data.ingestNumTServersHolding || 0))); + body.append(createRow("MinC Queued", bigNumberForQuantity(data.totalMinCQueued || 0))); + body.append(createRow("MinC Running", bigNumberForQuantity(data.totalMinCRunning || 0))); + body.append(createRow("MinC Completed", bigNumberForQuantity(data.totalMinCCompleted || 0))); + body.append(createRow("Bulk Imports Queued", bigNumberForQuantity(data.ingestBulkImportQueued || 0))); + body.append(createRow("Bulk Imports Running", bigNumberForQuantity(data.ingestBulkImportRunning || 0))); + + var table = $('#scan-overview-table'); + $(table).find('thead').remove(); + var body = $(table).find('tbody'); + body.empty(); + body.append(createRow("Scans In Progress", bigNumberForQuantity(data.scansTotalInProgress || 0))); + body.append(createRow("Entries Scanned", bigNumberForQuantity(data.scansTotalKvScanned || 0))); + body.append(createRow("Entries Returned", bigNumberForQuantity(data.scansTotalKvReturned || 0))); + body.append(createRow("Bytes Returned", bigNumberForSize(data.scansTotalKvReturnedBytes || 0))); + body.append(createRow("Open Files", bigNumberForQuantity(data.scanTotalOpenFiles || 0))); + + var table = $('#compaction-overview-table'); + $(table).find('thead').remove(); + var body = $(table).find('tbody'); + body.empty(); + body.append(createRow("MajC Queued", bigNumberForQuantity(data.compactionsQueued || 0))); + body.append(createRow("MajC Dequeued", bigNumberForQuantity(data.compactionsDequeued || 0))); + body.append(createRow("MajC Running", bigNumberForQuantity(data.compactionsRunning || 0))); + body.append(createRow("MajC Failed", bigNumberForQuantity(data.compactionsFailed || 0))); +} + +function createRow(name, value) { + var row = $(document.createElement("tr")); + row.append(createFirstCell(name, name)) + row.append(createRightCell(value, value)); + return row; +} + /** * Refreshes the deployment overview tables */ function refreshDeploymentTables() { - getDeployment().then(function () { - var data = JSON.parse(sessionStorage.deployment); - var breakdown = Array.isArray(data.breakdown) ? data.breakdown : []; - deploymentBreakdown = breakdown; - - if (breakdown.length === 0) { - $('#deploymentWarning').html(''); - } else { - $('#deploymentWarning').empty(); - } + var data = JSON.parse(sessionStorage.deployment); + var breakdown = Array.isArray(data.breakdown) ? data.breakdown : []; + deploymentBreakdown = breakdown; - renderDeploymentMatrix(breakdown); - }); + if (breakdown.length === 0) { + $('#deploymentWarning').html(''); + } else { + $('#deploymentWarning').empty(); + } + + renderDeploymentMatrix(breakdown); } function renderDeploymentMatrix(breakdown) { diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/overview.ftl b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/overview.ftl index ada148dddfb..bba8a88d4d0 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/overview.ftl +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/overview.ftl @@ -24,13 +24,45 @@
-
-
-
-
- +
+
+
+
+
+
+ <#include "table_loading.ftl" >
Instance Overview
+
+ + + <#include "table_loading.ftl" > +
Compaction Overview
+
+
+ + + <#include "table_loading.ftl" > +
Ingest Overview
+
+ + + <#include "table_loading.ftl" > +
Scan Overview
+
+
+
+
+
+
+
+
+
+ +
Server Deployment Overview
+ <#include "table_loading.ftl" > +
+
From e490d74b6657902e71513cf76ac0d8b5655c914e Mon Sep 17 00:00:00 2001 From: "Dom G." Date: Thu, 28 May 2026 13:11:14 -0400 Subject: [PATCH 2/4] Replace tables with cards to overview stats (#66) --- .../accumulo/monitor/resources/js/overview.js | 120 ++++++++++-------- .../accumulo/monitor/templates/overview.ftl | 102 ++++++++++----- 2 files changed, 133 insertions(+), 89 deletions(-) diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/overview.js b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/overview.js index 8ae21ec0010..0ceade4d694 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/overview.js +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/overview.js @@ -51,62 +51,74 @@ function refresh() { function refreshInstanceOverviewTables() { var data = getStoredJson(INSTANCE_OVERVIEW, {}); - var table = $('#instance-overview-table'); - $(table).find('thead').remove(); - var body = $(table).find('tbody'); - body.empty(); - body.append(createRow("Namespaces", bigNumberForQuantity(data.numNamespaces || 0))); - body.append(createRow("Tables", bigNumberForQuantity(data.numTables || 0))); - body.append(createRow("Tablets", bigNumberForQuantity(data.numTablets || 0))); - body.append(createRow("Entries", bigNumberForQuantity(data.numKVs || 0))); - body.append(createRow("Files", bigNumberForQuantity(data.numFiles || 0))); - body.append(createRow("File Size", bigNumberForSize(data.totalFileSize || 0))); - body.append(createRow("Tablets Assigned To Dead TServers", bigNumberForQuantity(data.tabletsAssignedToDeadTServers || 0))); - body.append(createRow("Tablets Suspended", bigNumberForQuantity(data.totalSuspendedTablets || 0))); - body.append(createRow("Tablets Requiring Recovery", bigNumberForQuantity(data.tabletsNeedingRecovery || 0))); - body.append(createRow("Fate Tx Queued", bigNumberForQuantity(data.totalFateSubmitted || 0))); - body.append(createRow("Fate Tx Running", bigNumberForQuantity(data.totalFateRunning || 0))); - body.append(createRow("Servers Low On Memory", bigNumberForQuantity(data.totalServersLowMem || 0))); - - var table = $('#ingest-overview-table'); - $(table).find('thead').remove(); - var body = $(table).find('tbody'); - body.empty(); - body.append(createRow("Entries", bigNumberForQuantity(data.ingestTotalEntries || 0))); - body.append(createRow("Bytes", bigNumberForSize(data.ingestTotalEntriesBytes || 0))); - body.append(createRow("Entries In Memory", bigNumberForQuantity(data.ingestTotalEntriesInMem || 0))); - body.append(createRow("TServers Holding For MinC", bigNumberForQuantity(data.ingestNumTServersHolding || 0))); - body.append(createRow("MinC Queued", bigNumberForQuantity(data.totalMinCQueued || 0))); - body.append(createRow("MinC Running", bigNumberForQuantity(data.totalMinCRunning || 0))); - body.append(createRow("MinC Completed", bigNumberForQuantity(data.totalMinCCompleted || 0))); - body.append(createRow("Bulk Imports Queued", bigNumberForQuantity(data.ingestBulkImportQueued || 0))); - body.append(createRow("Bulk Imports Running", bigNumberForQuantity(data.ingestBulkImportRunning || 0))); - - var table = $('#scan-overview-table'); - $(table).find('thead').remove(); - var body = $(table).find('tbody'); - body.empty(); - body.append(createRow("Scans In Progress", bigNumberForQuantity(data.scansTotalInProgress || 0))); - body.append(createRow("Entries Scanned", bigNumberForQuantity(data.scansTotalKvScanned || 0))); - body.append(createRow("Entries Returned", bigNumberForQuantity(data.scansTotalKvReturned || 0))); - body.append(createRow("Bytes Returned", bigNumberForSize(data.scansTotalKvReturnedBytes || 0))); - body.append(createRow("Open Files", bigNumberForQuantity(data.scanTotalOpenFiles || 0))); - - var table = $('#compaction-overview-table'); - $(table).find('thead').remove(); - var body = $(table).find('tbody'); - body.empty(); - body.append(createRow("MajC Queued", bigNumberForQuantity(data.compactionsQueued || 0))); - body.append(createRow("MajC Dequeued", bigNumberForQuantity(data.compactionsDequeued || 0))); - body.append(createRow("MajC Running", bigNumberForQuantity(data.compactionsRunning || 0))); - body.append(createRow("MajC Failed", bigNumberForQuantity(data.compactionsFailed || 0))); + renderOverviewList('#instance-overview-list', [ + ["Namespaces", bigNumberForQuantity(data.numNamespaces || 0)], + ["Tables", bigNumberForQuantity(data.numTables || 0)], + ["Tablets", bigNumberForQuantity(data.numTablets || 0)], + ["Entries", bigNumberForQuantity(data.numKVs || 0)], + ["Files", bigNumberForQuantity(data.numFiles || 0)], + ["File Size", bigNumberForSize(data.totalFileSize || 0)], + ["Dead TServer Tablets", bigNumberForQuantity(data.tabletsAssignedToDeadTServers || 0)], + ["Suspended Tablets", bigNumberForQuantity(data.totalSuspendedTablets || 0)], + ["Recovery Tablets", bigNumberForQuantity(data.tabletsNeedingRecovery || 0)], + ["Fate Tx Queued", bigNumberForQuantity(data.totalFateSubmitted || 0)], + ["Fate Tx Running", bigNumberForQuantity(data.totalFateRunning || 0)], + ["Servers Low On Memory", bigNumberForQuantity(data.totalServersLowMem || 0)] + ]); + + renderOverviewList('#ingest-overview-list', [ + ["Entries", bigNumberForQuantity(data.ingestTotalEntries || 0)], + ["Bytes", bigNumberForSize(data.ingestTotalEntriesBytes || 0)], + ["Entries In Memory", bigNumberForQuantity(data.ingestTotalEntriesInMem || 0)], + ["TServers Holding For MinC", bigNumberForQuantity(data.ingestNumTServersHolding || 0)], + ["MinC Queued", bigNumberForQuantity(data.totalMinCQueued || 0)], + ["MinC Running", bigNumberForQuantity(data.totalMinCRunning || 0)], + ["MinC Completed", bigNumberForQuantity(data.totalMinCCompleted || 0)], + ["Bulk Imports Queued", bigNumberForQuantity(data.ingestBulkImportQueued || 0)], + ["Bulk Imports Running", bigNumberForQuantity(data.ingestBulkImportRunning || 0)] + ]); + + renderOverviewList('#scan-overview-list', [ + ["Scans In Progress", bigNumberForQuantity(data.scansTotalInProgress || 0)], + ["Entries Scanned", bigNumberForQuantity(data.scansTotalKvScanned || 0)], + ["Entries Returned", bigNumberForQuantity(data.scansTotalKvReturned || 0)], + ["Bytes Returned", bigNumberForSize(data.scansTotalKvReturnedBytes || 0)], + ["Open Files", bigNumberForQuantity(data.scanTotalOpenFiles || 0)] + ]); + + renderOverviewList('#compaction-overview-list', [ + ["MajC Queued", bigNumberForQuantity(data.compactionsQueued || 0)], + ["MajC Dequeued", bigNumberForQuantity(data.compactionsDequeued || 0)], + ["MajC Running", bigNumberForQuantity(data.compactionsRunning || 0)], + ["MajC Failed", bigNumberForQuantity(data.compactionsFailed || 0)] + ]); } -function createRow(name, value) { - var row = $(document.createElement("tr")); - row.append(createFirstCell(name, name)) - row.append(createRightCell(value, value)); - return row; +function renderOverviewList(selector, rows) { + var list = $(selector); + list.empty(); + + list.append(rows.map(function (row) { + const name = row[0]; + const value = row[1]; + return createOverviewListItem(name, value); + })); +} + +function createOverviewListItem(name, rowValue) { + var item = $(document.createElement("li")); + item.addClass("list-group-item d-flex justify-content-between align-items-center"); + + var label = $(document.createElement("span")); + label.text(name); + item.append(label); + + var value = $(document.createElement("strong")); + value.addClass("text-end text-nowrap"); + value.text(rowValue); + item.append(value); + + return item; } /** diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/overview.ftl b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/overview.ftl index bba8a88d4d0..2d39bb0deb3 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/overview.ftl +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/overview.ftl @@ -25,43 +25,75 @@
-
-
-
-
- - - <#include "table_loading.ftl" > -
Instance Overview
-
- - - <#include "table_loading.ftl" > -
Compaction Overview
-
-
- - - <#include "table_loading.ftl" > -
Ingest Overview
-
- - - <#include "table_loading.ftl" > -
Scan Overview
+
+
+
+
+
Server Deployment
+
+
+
+ + <#include "table_loading.ftl" > +
+
+
+
+
-
-
-
-
-
-
-
- -
Server Deployment Overview
- <#include "table_loading.ftl" > -
+ +
+
+
+
Instance
+
    +
  • + + Loading... +
    • +
+
+
+
+
+
Ingest
+
    +
  • + + Loading... +
    • +
+
+
+
+
+
Scan
+
    +
  • + + Loading... +
    • +
+
+
+
+
+
Compaction
+
    +
  • + + Loading... +
    • +
From 842b71dc97dac501e4f05afda4e190a49940bea0 Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Thu, 28 May 2026 14:33:38 -0400 Subject: [PATCH 3/4] Add smallest two cards to single larger parent card --- .../org/apache/accumulo/monitor/templates/overview.ftl | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/overview.ftl b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/overview.ftl index 2d39bb0deb3..a06db211050 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/overview.ftl +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/overview.ftl @@ -43,7 +43,7 @@
-
+
Instance
@@ -70,8 +70,8 @@
-
-
+
+
Scan
  • @@ -82,9 +82,7 @@
-
-
-
+
Compaction
  • From 3100750dc2ada04af71fa44c24e13fda5f9c71c5 Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Thu, 28 May 2026 20:19:27 +0000 Subject: [PATCH 4/4] Get scans in progress from active scans --- .../accumulo/monitor/next/SystemInformation.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java index 961ce397e75..0f2006fd6c0 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java @@ -1083,10 +1083,7 @@ public void processResponse(final ServerId server, final MetricResponse response for (ByteBuffer binary : response.getMetrics()) { flatbuffer = FMetric.getRootAsFMetric(binary, flatbuffer); final String metricName = flatbuffer.name(); - if (metricName.equals(Metric.SCAN_CONTINUE.getName())) { - long inProgress = getMetricValue(flatbuffer).longValue(); - this.instanceOverview.getScansTotalInProgress().addAndGet(inProgress); - } else if (metricName.equals(Metric.SCAN_OPEN_FILES.getName())) { + if (metricName.equals(Metric.SCAN_OPEN_FILES.getName())) { long files = getMetricValue(flatbuffer).longValue(); this.instanceOverview.getScanTotalOpenFiles().addAndGet(files); } else if (metricName.equals(Metric.SCAN_SCANNED_ENTRIES.getName())) { @@ -1113,10 +1110,7 @@ public void processResponse(final ServerId server, final MetricResponse response for (ByteBuffer binary : response.getMetrics()) { flatbuffer = FMetric.getRootAsFMetric(binary, flatbuffer); final String metricName = flatbuffer.name(); - if (metricName.equals(Metric.SCAN_CONTINUE.getName())) { - long inProgress = getMetricValue(flatbuffer).longValue(); - this.instanceOverview.getScansTotalInProgress().addAndGet(inProgress); - } else if (metricName.equals(Metric.SCAN_OPEN_FILES.getName())) { + if (metricName.equals(Metric.SCAN_OPEN_FILES.getName())) { long files = getMetricValue(flatbuffer).longValue(); this.instanceOverview.getScanTotalOpenFiles().addAndGet(files); } else if (metricName.equals(Metric.SCAN_SCANNED_ENTRIES.getName())) { @@ -1267,6 +1261,7 @@ public void processActiveScans(ServerId server, List scans) { activeScans.add(new Scan(server.toHostPortString(), server.getType().name(), server.getResourceGroup().canonical(), s.getTableId(), s.getScanId(), s.getClient(), s.getUser(), s.getState().name(), s.getType().name(), s.getAge(), s.getIdleTime())); + instanceOverview.getScansTotalInProgress().incrementAndGet(); }); }