From 0d71ae635ac5aea507852d9af17c5b0e40cbf986 Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Thu, 14 May 2026 17:17:16 +0000 Subject: [PATCH] Added monitor activity page for Fate transaction details --- .../accumulo/monitor/next/Endpoints.java | 9 ++ .../monitor/next/InformationFetcher.java | 56 ++++++++- .../monitor/next/SystemInformation.java | 23 ++++ .../accumulo/monitor/view/WebViews.java | 18 +++ .../accumulo/monitor/resources/js/fate.js | 107 ++++++++++++++++++ .../monitor/resources/js/functions.js | 9 ++ .../accumulo/monitor/templates/fate.ftl | 30 +++++ .../accumulo/monitor/templates/navbar.ftl | 1 + 8 files changed, 250 insertions(+), 3 deletions(-) create mode 100644 server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/fate.js create mode 100644 server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/fate.ftl diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java index 360829b328e..5b63c08d86e 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java @@ -57,6 +57,7 @@ import org.apache.accumulo.monitor.next.InformationFetcher.InstanceSummary; import org.apache.accumulo.monitor.next.SystemInformation.CompactionGroupSummary; import org.apache.accumulo.monitor.next.SystemInformation.CompactionTableSummary; +import org.apache.accumulo.monitor.next.SystemInformation.FateTransaction; import org.apache.accumulo.monitor.next.SystemInformation.MessageCategory; import org.apache.accumulo.monitor.next.SystemInformation.MessagePriority; import org.apache.accumulo.monitor.next.SystemInformation.TableSummary; @@ -385,6 +386,14 @@ public CompactorsSummary getExternalCompactors() { return new CompactorsSummary(summary.getCompactorServers(), summary.getTimestamp()); } + @GET + @Path("fate") + @Produces(MediaType.APPLICATION_JSON) + @Description("Returns a list of fate transaction details") + public List getFateTransactions() { + return monitor.getInformationFetcher().getSummaryForEndpoint().getFateTransactions(); + } + @GET @Path("tables") @Produces(MediaType.APPLICATION_JSON) diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java index 666bfe32969..837cc18ee36 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java @@ -22,8 +22,10 @@ import java.time.Duration; import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; @@ -44,6 +46,15 @@ import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.RowRange; import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.fate.AdminUtil; +import org.apache.accumulo.core.fate.AdminUtil.FateStatus; +import org.apache.accumulo.core.fate.FateId; +import org.apache.accumulo.core.fate.FateInstanceType; +import org.apache.accumulo.core.fate.ReadOnlyFateStore; +import org.apache.accumulo.core.fate.user.UserFateStore; +import org.apache.accumulo.core.fate.zookeeper.LockRange; +import org.apache.accumulo.core.fate.zookeeper.MetaFateStore; +import org.apache.accumulo.core.metadata.SystemTables; import org.apache.accumulo.core.process.thrift.MetricResponse; import org.apache.accumulo.core.process.thrift.ServerProcessService.Client; import org.apache.accumulo.core.rpc.ThriftUtil; @@ -54,6 +65,8 @@ import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.compaction.CompactionPluginUtils; +import org.apache.accumulo.server.util.adminCommand.Fate; +import org.apache.zookeeper.KeeperException; import org.checkerframework.checker.nullness.qual.Nullable; import org.eclipse.jetty.util.NanoTime; import org.slf4j.Logger; @@ -191,6 +204,24 @@ public void run() { } } + private class FateTransactionFetcher implements Runnable { + + private final SystemInformation summary; + private Map> emptyLocks = new HashMap<>(); + private Map emptyRanges = new HashMap<>(); + + public FateTransactionFetcher(SystemInformation summary) { + this.summary = summary; + } + + @Override + public void run() { + FateStatus status = AdminUtil.getTransactionStatus(stores, null, null, null, emptyLocks, + emptyLocks, emptyRanges); + summary.processFateTransactions(status.getTransactions()); + } + } + private final String poolName = "MonitorMetricsThreadPool"; private final ThreadPoolExecutor pool = ThreadPools.getServerThreadPools() .getPoolBuilder(poolName).numCoreThreads(10).withTimeOut(30, SECONDS).build(); @@ -201,6 +232,9 @@ public void run() { private final Cache allMetrics; private final Cache retainedProblemServers; private final AtomicReference summaryRef = new AtomicReference<>(); + private final ReadOnlyFateStore readOnlyMFS; + private final ReadOnlyFateStore readOnlyUFS; + private final Map> stores; public InformationFetcher(ServerContext ctx, Supplier connectionCount) { this.ctx = ctx; @@ -209,6 +243,13 @@ public InformationFetcher(ServerContext ctx, Supplier connectionCount) { .expireAfterWrite(Duration.ofMinutes(10)).evictionListener(this::onRemoval).build(); this.retainedProblemServers = Caffeine.newBuilder().executor(pool) .scheduler(Scheduler.systemScheduler()).expireAfterWrite(Duration.ofMinutes(10)).build(); + try { + this.readOnlyMFS = new MetaFateStore<>(ctx.getZooSession(), null, null); + } catch (KeeperException | InterruptedException e) { + throw new RuntimeException("Exception creating MetaFateStore", e); + } + this.readOnlyUFS = new UserFateStore<>(ctx, SystemTables.FATE.tableName(), null, null); + this.stores = Map.of(FateInstanceType.META, readOnlyMFS, FateInstanceType.USER, readOnlyUFS); } public void newConnectionEvent() { @@ -284,9 +325,17 @@ public void run() { final List> futures = new ArrayList<>(); final SystemInformation summary = new SystemInformation(allMetrics, this.ctx); - Set compactors = this.ctx.instanceOperations().getServers(Type.COMPACTOR); - summary.processExternalCompactionInventory(compactors); + // Fetch set of registered compactors + futures.add(this.pool.submit(() -> { + Set compactors = this.ctx.instanceOperations().getServers(Type.COMPACTOR); + summary.processExternalCompactionInventory(compactors); + })); + + // Fetch Fate transaction information + futures.add(this.pool.submit(new FateTransactionFetcher(summary))); + + // Fetch metrics from the server processes for (ServerId.Type type : ServerId.Type.values()) { if (type == Type.MONITOR) { continue; @@ -295,7 +344,6 @@ public void run() { futures.add(this.pool.submit(new MetricFetcher(this.ctx, server, summary))); } } - ThreadPools.resizePool(pool, () -> Math.max(20, (futures.size() / 20)), poolName); // Fetch external compaction information from the Compactors futures.add(this.pool.submit(new RunningCompactionFetcher(summary, pool))); @@ -314,6 +362,8 @@ public void run() { } })); + ThreadPools.resizePool(pool, () -> Math.max(20, (futures.size() / 20)), poolName); + final long monitorFetchTimeout = ctx.getConfiguration().getTimeInMillis(Property.MONITOR_FETCH_TIMEOUT); final long allFuturesAdded = NanoTime.now(); diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java index b45bad0f3cf..2940a0b498b 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java @@ -64,6 +64,10 @@ import org.apache.accumulo.core.data.TabletId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.dataImpl.TabletIdImpl; +import org.apache.accumulo.core.fate.AdminUtil.TransactionStatus; +import org.apache.accumulo.core.fate.Fate.FateOperation; +import org.apache.accumulo.core.fate.FateInstanceType; +import org.apache.accumulo.core.fate.ReadOnlyFateStore.TStatus; import org.apache.accumulo.core.metadata.SystemTables; import org.apache.accumulo.core.metadata.TabletState; import org.apache.accumulo.core.metrics.Metric; @@ -379,6 +383,10 @@ public enum MessageCategory { Configuration, Table; } + public record FateTransaction(FateInstanceType type, FateOperation op, String id, TStatus status, + long created, List heldLocks, List waitingLocks, String lockRange) { + } + private static final Logger LOG = LoggerFactory.getLogger(SystemInformation.class); private final DistributionStatisticConfig DSC = @@ -445,6 +453,8 @@ public enum MessageCategory { private final Set configuredCompactionResourceGroups = ConcurrentHashMap.newKeySet(); + private final List fateTransactions = new ArrayList<>(); + private final AtomicLong timestamp = new AtomicLong(0); private final EnumMap componentStatuses = new EnumMap<>(ServerId.Type.class); @@ -491,6 +501,7 @@ public void clear() { componentStatuses.clear(); managerGoalState = null; serverMetricsView.clear(); + fateTransactions.clear(); } private void addMessage(MessagePriority pri, MessageCategory cat, String msg) { @@ -682,6 +693,14 @@ public void processTabletInformation(TableId tableId, String tableName, TabletIn } } + public void processFateTransactions(List transactions) { + transactions.forEach(t -> { + fateTransactions.add(new FateTransaction(t.getInstanceType(), t.getFateOp(), + t.getFateId().getTxUUIDStr(), t.getStatus(), t.getTimeCreated(), t.getHeldLocks(), + t.getWaitingLocks(), t.getLockRange().toString())); + }); + } + public void processError(ServerId server) { problemHosts.add(server); } @@ -987,6 +1006,10 @@ public Map>> getMessages() { return this.messages; } + public List getFateTransactions() { + return this.fateTransactions; + } + public long getTimestamp() { return this.timestamp.get(); } diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java index bc09781c954..06fd61e860f 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java @@ -328,6 +328,24 @@ public Map getBulkImports() { return model; } + /** + * Returns the garbage collector template + * + * @return GC model + */ + @GET + @Path("fate") + @Template(name = "/default.ftl") + public Map getFate() { + + Map model = getModel(); + model.put("title", "Fate Transaction Details"); + model.put("template", "fate.ftl"); + model.put("js", "fate.js"); + + return model; + } + /** * Returns the garbage collector template * diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/fate.js b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/fate.js new file mode 100644 index 00000000000..6d207183ece --- /dev/null +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/fate.js @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +"use strict"; + +const fateHtmlTable = '#fateTable'; + +var dataTableRef; + +function getTableData() { + return getStoredArray(FATE); +} + +function createDataTable() { + $(fateHtmlTable).find('thead').remove(); + $(fateHtmlTable).find('tbody').remove(); + dataTableRef = $(fateHtmlTable).DataTable({ + "autoWidth": false, + "ajax": function (data, callback) { + callback({ + data: getTableData() + }); + }, + "stateSave": true, + "colReorder": true, + "columnDefs": [{ + targets: '_all', + defaultContent: '-' + }], + "columns": [{ + "data": "type", + "title": "Type" + }, + { + "data": "op", + "title": "Operation" + }, + { + "data": "id", + "title": "Id" + }, + { + "data": "status", + "title": "State" + }, + { + "data": "created", + "title": "Created", + "render": function (data, type, row) { + if (type === 'display') data = dateFormat(data); + return data; + } + }, + { + "data": "created", + "title": "Age", + "render": function (data, type, row) { + var dur = Date.now() - data; + if (type === 'display') dur = timeDuration(dur); + return dur; + } + }, + { + "data": "heldLocks", + "title": "Locks Held" + }, + { + "data": "waitingLocks", + "title": "Locks Waiting On" + }, + { + "data": "lockRange", + "title": "Lock Range" + } + ] + }); +} + +function refresh() { + return getFate().then(function () { + if (dataTableRef) { + ajaxReloadTable(dataTableRef); + } + }); +} + + +$(function () { + getFate().then(function () { + createDataTable(); + }); +}); diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js index 38ef5981e77..b789360ba69 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js @@ -40,6 +40,7 @@ const RUNNING_COMPACTIONS_BY_GROUP = 'runningCompactionsByGroup'; const AUTO_REFRESH_KEY = 'auto-refresh'; const MESSAGE_CATEGORIES = 'messageCategories'; const MESSAGES = 'messages'; +const FATE = 'fate'; // Override Length Menu options for dataTables if ($.fn && $.fn.dataTable) { @@ -704,6 +705,14 @@ function getDeployment() { return getJSONForTable(REST_V2_PREFIX + '/deployment', 'deployment'); } +/** + * REST GET call for /fate, + * stores it on a sessionStorage variable + */ +function getFate() { + return getJSONForTable(REST_V2_PREFIX + '/fate', FATE); +} + function getServerProcessView(table, storageKey) { var url = REST_V2_PREFIX + '/servers/view;table=' + table; return getJSONForTable(url, storageKey); diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/fate.ftl b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/fate.ftl new file mode 100644 index 00000000000..b460c4a9478 --- /dev/null +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/fate.ftl @@ -0,0 +1,30 @@ +<#-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +
+
+ + + <#include "table_loading.ftl" > +
Fate Transaction Details
+ The table contains the last known Fate transaction status.
+
+
+
diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/navbar.ftl b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/navbar.ftl index d29518c380c..d76f6ff20fd 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/navbar.ftl +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/navbar.ftl @@ -57,6 +57,7 @@
  • Bulk Imports
  • Compaction Overview
  • Compaction Details
    • +
  • Fate Tx Details
  • Scans