diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java index 72dfc4a1360..56eaefae79a 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java @@ -57,6 +57,7 @@ import org.apache.accumulo.monitor.next.InformationFetcher.InstanceSummary; import org.apache.accumulo.monitor.next.SystemInformation.CompactionGroupSummary; import org.apache.accumulo.monitor.next.SystemInformation.CompactionTableSummary; +import org.apache.accumulo.monitor.next.SystemInformation.FateTransaction; import org.apache.accumulo.monitor.next.SystemInformation.MessageCategory; import org.apache.accumulo.monitor.next.SystemInformation.MessagePriority; import org.apache.accumulo.monitor.next.SystemInformation.RecoveryInformation; @@ -386,6 +387,14 @@ public CompactorsSummary getExternalCompactors() { return new CompactorsSummary(summary.getCompactorServers(), summary.getTimestamp()); } + @GET + @Path("fate") + @Produces(MediaType.APPLICATION_JSON) + @Description("Returns a list of fate transaction details") + public List getFateTransactions() { + return monitor.getInformationFetcher().getSummaryForEndpoint().getFateTransactions(); + } + @GET @Path("tables") @Produces(MediaType.APPLICATION_JSON) diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java index 5e0aec350b9..55b5567c256 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java @@ -28,8 +28,10 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.CancellationException; @@ -53,6 +55,14 @@ import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.RowRange; import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.fate.AdminUtil; +import org.apache.accumulo.core.fate.AdminUtil.FateStatus; +import org.apache.accumulo.core.fate.FateId; +import org.apache.accumulo.core.fate.FateInstanceType; +import org.apache.accumulo.core.fate.ReadOnlyFateStore; +import org.apache.accumulo.core.fate.user.UserFateStore; +import org.apache.accumulo.core.fate.zookeeper.LockRange; +import org.apache.accumulo.core.fate.zookeeper.MetaFateStore; import org.apache.accumulo.core.lock.ServiceLockPaths.AddressSelector; import org.apache.accumulo.core.lock.ServiceLockPaths.ResourceGroupPredicate; import org.apache.accumulo.core.lock.ServiceLockPaths.ServiceLockPath; @@ -74,6 +84,8 @@ import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.compaction.CompactionPluginUtils; +import org.apache.accumulo.server.util.adminCommand.Fate; +import org.apache.zookeeper.KeeperException; import org.checkerframework.checker.nullness.qual.Nullable; import org.eclipse.jetty.util.NanoTime; import org.slf4j.Logger; @@ -173,7 +185,7 @@ void add(UpdateTaskFuture f) { } enum UpdateType { - COMPACTION, COMPACTION_RGS, METRIC, TABLE; + COMPACTION, COMPACTION_RGS, FATE, METRIC, TABLE; } interface UpdateTask extends Runnable, Comparable> { @@ -412,6 +424,67 @@ public void run() { } } + class FateTransactionFetcher implements UpdateTask { + + private final SystemInformation summary; + private Map> emptyLocks = new HashMap<>(); + private Map emptyRanges = new HashMap<>(); + + public FateTransactionFetcher(SystemInformation summary) { + this.summary = summary; + } + + @Override + public void run() { + FateStatus status = AdminUtil.getTransactionStatus(stores, null, null, null, emptyLocks, + emptyLocks, emptyRanges); + summary.processFateTransactions(status.getTransactions()); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Objects.hash(getType()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + FateTransactionFetcher other = (FateTransactionFetcher) obj; + return Objects.equals(getType(), other.getType()); + } + + @Override + public int compareTo(UpdateTask other) { + return this.getType().compareTo(other.getType()); + } + + @Override + public UpdateType getType() { + return UpdateType.FATE; + } + + @Override + public Void getResource() { + return null; + } + + @Override + public String getFailureMessage() { + return "Error fetching fate transaction details"; + } + } + class ConfiguredCompactionResourceGroupFetcher implements UpdateTask { private final SystemInformation summary; @@ -486,6 +559,9 @@ public String getFailureMessage() { private final Cache allMetrics; private final Cache retainedProblemServers; private final AtomicReference summaryRef = new AtomicReference<>(); + private final ReadOnlyFateStore readOnlyMFS; + private final ReadOnlyFateStore readOnlyUFS; + private final Map> stores; private final TabletMetadataFilter noLocation = new NoCurrentLocationFilter(); public InformationFetcher(ServerContext ctx, Supplier connectionCount) { @@ -495,6 +571,13 @@ public InformationFetcher(ServerContext ctx, Supplier connectionCount) { .expireAfterWrite(Duration.ofMinutes(10)).evictionListener(this::onRemoval).build(); this.retainedProblemServers = Caffeine.newBuilder().executor(pool) .scheduler(Scheduler.systemScheduler()).expireAfterWrite(Duration.ofMinutes(10)).build(); + try { + this.readOnlyMFS = new MetaFateStore<>(ctx.getZooSession(), null, null); + } catch (KeeperException | InterruptedException e) { + throw new RuntimeException("Exception creating MetaFateStore", e); + } + this.readOnlyUFS = new UserFateStore<>(ctx, SystemTables.FATE.tableName(), null, null); + this.stores = Map.of(FateInstanceType.META, readOnlyMFS, FateInstanceType.USER, readOnlyUFS); } public void newConnectionEvent() { @@ -659,9 +742,16 @@ public void run() { final UpdateTasks futures = new UpdateTasks(); final SystemInformation summary = new SystemInformation(allMetrics, this.ctx); + + // Fetch set of registered compactors Set compactors = this.ctx.instanceOperations().getServers(Type.COMPACTOR); summary.processExternalCompactionInventory(compactors); + // Fetch Fate transaction information + FateTransactionFetcher fateFetcher = new FateTransactionFetcher(summary); + Future fff = this.pool.submit(fateFetcher); + futures.add(new UpdateTaskFuture(fff, fateFetcher)); + // Fetch metrics from the other server processes. This // makes an RPC call to AbstractServer.getMetrics for (ServerId.Type type : ServerId.Type.values()) { @@ -674,7 +764,6 @@ public void run() { futures.add(new UpdateTaskFuture(mff, mf)); } } - ThreadPools.resizePool(pool, () -> Math.max(20, (futures.size() / 20)), poolName); // Fetch external compaction information from the Compactors RunningCompactionFetcher rcf = new RunningCompactionFetcher(summary, pool); @@ -692,6 +781,8 @@ public void run() { Future f = this.pool.submit(r); futures.add(new UpdateTaskFuture(f, r)); + ThreadPools.resizePool(pool, () -> Math.max(20, (futures.size() / 20)), poolName); + final long monitorFetchTimeout = ctx.getConfiguration().getTimeInMillis(Property.MONITOR_FETCH_TIMEOUT); final long allFuturesAdded = NanoTime.now(); diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java index e5d7491f58c..3a1a15938c5 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java @@ -67,6 +67,10 @@ import org.apache.accumulo.core.data.TabletId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.dataImpl.TabletIdImpl; +import org.apache.accumulo.core.fate.AdminUtil.TransactionStatus; +import org.apache.accumulo.core.fate.Fate.FateOperation; +import org.apache.accumulo.core.fate.FateInstanceType; +import org.apache.accumulo.core.fate.ReadOnlyFateStore.TStatus; import org.apache.accumulo.core.lock.ServiceLockPaths.AddressSelector; import org.apache.accumulo.core.lock.ServiceLockPaths.ResourceGroupPredicate; import org.apache.accumulo.core.lock.ServiceLockPaths.ServiceLockPath; @@ -457,6 +461,10 @@ public List getTabletsNeedingRecovery() { } } + public record FateTransaction(FateInstanceType type, FateOperation op, String id, TStatus status, + long created, List heldLocks, List waitingLocks, String lockRange) { + } + private static final Logger LOG = LoggerFactory.getLogger(SystemInformation.class); private final DistributionStatisticConfig DSC = @@ -524,6 +532,8 @@ public List getTabletsNeedingRecovery() { private final Set configuredCompactionResourceGroups = ConcurrentHashMap.newKeySet(); + private final List fateTransactions = new ArrayList<>(); + private final AtomicLong timestamp = new AtomicLong(0); private final EnumMap componentStatuses = new EnumMap<>(ServerId.Type.class); @@ -570,6 +580,7 @@ public void clear() { componentStatuses.clear(); managerGoalState = null; serverMetricsView.clear(); + fateTransactions.clear(); } public void addMessage(MessagePriority pri, MessageCategory cat, String msg) { @@ -887,6 +898,14 @@ public void processTabletInformation(TableId tableId, String tableName, TabletIn } } + public void processFateTransactions(List transactions) { + transactions.forEach(t -> { + fateTransactions.add(new FateTransaction(t.getInstanceType(), t.getFateOp(), + t.getFateId().getTxUUIDStr(), t.getStatus(), t.getTimeCreated(), t.getHeldLocks(), + t.getWaitingLocks(), t.getLockRange().toString())); + }); + } + public void processError(ServerId server) { problemHosts.add(server); } @@ -1352,6 +1371,10 @@ public Map>> getMessages() { return this.messages; } + public List getFateTransactions() { + return this.fateTransactions; + } + public long getTimestamp() { return this.timestamp.get(); } diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java index f373ea24636..07edfca496e 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java @@ -346,6 +346,24 @@ public Map getBulkImports() { return model; } + /** + * Returns the garbage collector template + * + * @return GC model + */ + @GET + @Path("fate") + @Template(name = "/default.ftl") + public Map getFate() { + + Map model = getModel(); + model.put("title", "Fate Transaction Details"); + model.put("template", "fate.ftl"); + model.put("js", "fate.js"); + + return model; + } + /** * Returns the garbage collector template * diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/fate.js b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/fate.js new file mode 100644 index 00000000000..6d207183ece --- /dev/null +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/fate.js @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +"use strict"; + +const fateHtmlTable = '#fateTable'; + +var dataTableRef; + +function getTableData() { + return getStoredArray(FATE); +} + +function createDataTable() { + $(fateHtmlTable).find('thead').remove(); + $(fateHtmlTable).find('tbody').remove(); + dataTableRef = $(fateHtmlTable).DataTable({ + "autoWidth": false, + "ajax": function (data, callback) { + callback({ + data: getTableData() + }); + }, + "stateSave": true, + "colReorder": true, + "columnDefs": [{ + targets: '_all', + defaultContent: '-' + }], + "columns": [{ + "data": "type", + "title": "Type" + }, + { + "data": "op", + "title": "Operation" + }, + { + "data": "id", + "title": "Id" + }, + { + "data": "status", + "title": "State" + }, + { + "data": "created", + "title": "Created", + "render": function (data, type, row) { + if (type === 'display') data = dateFormat(data); + return data; + } + }, + { + "data": "created", + "title": "Age", + "render": function (data, type, row) { + var dur = Date.now() - data; + if (type === 'display') dur = timeDuration(dur); + return dur; + } + }, + { + "data": "heldLocks", + "title": "Locks Held" + }, + { + "data": "waitingLocks", + "title": "Locks Waiting On" + }, + { + "data": "lockRange", + "title": "Lock Range" + } + ] + }); +} + +function refresh() { + return getFate().then(function () { + if (dataTableRef) { + ajaxReloadTable(dataTableRef); + } + }); +} + + +$(function () { + getFate().then(function () { + createDataTable(); + }); +}); diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js index 093d2d1d75e..60644748349 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js @@ -40,6 +40,7 @@ const RUNNING_COMPACTIONS_BY_GROUP = 'runningCompactionsByGroup'; const AUTO_REFRESH_KEY = 'auto-refresh'; const MESSAGE_CATEGORIES = 'messageCategories'; const MESSAGES = 'messages'; +const FATE = 'fate'; const RECOVERY = 'recovery'; // Override Length Menu options for dataTables @@ -706,6 +707,14 @@ function getDeployment() { return getJSONForTable(REST_V2_PREFIX + '/deployment', 'deployment'); } +/** + * REST GET call for /fate, + * stores it on a sessionStorage variable + */ +function getFate() { + return getJSONForTable(REST_V2_PREFIX + '/fate', FATE); +} + function getServerProcessView(table, storageKey) { var url = REST_V2_PREFIX + '/servers/view;table=' + table; return getJSONForTable(url, storageKey); diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/fate.ftl b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/fate.ftl new file mode 100644 index 00000000000..b460c4a9478 --- /dev/null +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/fate.ftl @@ -0,0 +1,30 @@ +<#-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +
+
+ + + <#include "table_loading.ftl" > +
Fate Transaction Details
+ The table contains the last known Fate transaction status.
+
+
+
diff --git a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/navbar.ftl b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/navbar.ftl index fc188ca6187..35ced56b9ab 100644 --- a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/navbar.ftl +++ b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/navbar.ftl @@ -57,6 +57,7 @@
  • Bulk Imports
  • Compaction Overview
  • Compaction Details
  • +
  • Fate Tx Details
  • Scans
  • Tablet Recoveries