Skip to content

Automatically include PG locks and active queries in thread dumps #7343

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
labkey-jeckels wants to merge 5 commits into
base: develop
Choose a base branch
from
Open

Automatically include PG locks and active queries in thread dumps #7343

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
fcbe9b2
Automatically include PG locks and active queries in thread dumps
labkey-jeckels Jan 20, 2026
10ba5b2
Another file
labkey-jeckels Jan 20, 2026
05c3356
Merge branch 'develop' into fb_713_pgInfoDump
labkey-jeckels Jan 22, 2026
fc7c226
Include time info and fix HTML rendering
labkey-jeckels Jan 22, 2026
a284e81
Share date format constants
labkey-jeckels Jan 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,10 @@ public abstract class BasePostgreSqlDialect extends SqlDialect
// Issue 52190: Expose troubleshooting data that supports postgreSQL-specific analysis
public static final String POSTGRES_SCHEMA_NAME = "postgres";

public static final String POSTGRES_STAT_ACTIVITY_TABLE_NAME = "pg_stat_activity";
public static final String POSTGRES_LOCKS_TABLE_NAME = "pg_locks";
public static final String POSTGRES_TABLE_SIZES_TABLE_NAME = "pg_tablesizes";

private final Map<String, Integer> _domainScaleMap = new CopyOnWriteHashMap<>();

private HtmlString _adminWarning = null;
Expand Down
12 changes: 6 additions & 6 deletions api/src/org/labkey/api/util/DateUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,12 +87,12 @@ private DateUtil()
private static final Locale _localeDefault = Locale.getDefault();
private static final TimeZone _timezoneDefault = TimeZone.getDefault();

private static final String ISO_DATE_FORMAT_STRING = "yyyy-MM-dd";
private static final String ISO_SHORT_TIME_FORMAT_STRING = "HH:mm";
private static final String ISO_DATE_SHORT_TIME_FORMAT_STRING = ISO_DATE_FORMAT_STRING + " " + ISO_SHORT_TIME_FORMAT_STRING;
private static final String ISO_TIME_FORMAT_STRING = "HH:mm:ss";
private static final String ISO_LONG_TIME_FORMAT_STRING = "HH:mm:ss.SSS";
private static final String ISO_DATE_TIME_FORMAT_STRING = ISO_DATE_FORMAT_STRING + " " + ISO_LONG_TIME_FORMAT_STRING;
public static final String ISO_DATE_FORMAT_STRING = "yyyy-MM-dd";
public static final String ISO_SHORT_TIME_FORMAT_STRING = "HH:mm";
public static final String ISO_DATE_SHORT_TIME_FORMAT_STRING = ISO_DATE_FORMAT_STRING + " " + ISO_SHORT_TIME_FORMAT_STRING;
public static final String ISO_TIME_FORMAT_STRING = "HH:mm:ss";
public static final String ISO_LONG_TIME_FORMAT_STRING = "HH:mm:ss.SSS";
public static final String ISO_DATE_TIME_FORMAT_STRING = ISO_DATE_FORMAT_STRING + " " + ISO_LONG_TIME_FORMAT_STRING;

// SimpleDataFormat does not support microseconds, it can only support up to milliseconds
private static final Pattern NON_SIMPLE_PRECISION_TIME_PATTERN = Pattern.compile(".*([0-5][0-9]):([0-5][0-9])\\.(\\d{4,6}).*");
Expand Down
53 changes: 52 additions & 1 deletion api/src/org/labkey/api/util/DebugInfoDumper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,33 @@
import org.apache.commons.collections4.multimap.HashSetValuedHashMap;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.Logger;
import org.labkey.api.action.NullSafeBindException;
import org.labkey.api.data.ConnectionWrapper;
import org.labkey.api.data.ContainerManager;
import org.labkey.api.data.DbScope;
import org.labkey.api.data.TSVWriter;
import org.labkey.api.data.TransactionFilter;
import org.labkey.api.data.dialect.BasePostgreSqlDialect;
import org.labkey.api.files.FileSystemDirectoryListener;
import org.labkey.api.files.FileSystemWatchers;
import org.labkey.api.miniprofiler.MiniProfiler;
import org.labkey.api.module.ModuleLoader;
import org.labkey.api.query.QueryForm;
import org.labkey.api.query.QueryService;
import org.labkey.api.query.QueryView;
import org.labkey.api.query.UserSchema;
import org.labkey.api.security.User;
import org.labkey.api.util.logging.LogHelper;
import org.labkey.api.view.ActionURL;
import org.labkey.api.view.HttpView;
import org.labkey.api.view.ViewContext;
import org.labkey.api.writer.PrintWriters;
import org.labkey.vfs.FileLike;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.lang.management.ManagementFactory;
import java.lang.management.OperatingSystemMXBean;
import java.lang.reflect.InvocationTargetException;
Expand Down Expand Up @@ -151,7 +164,7 @@ record ThreadExtraContext(String context, StackTraceElement[] stack, long startT
*/
public static _PopAutoCloseable pushThreadDumpContext(String context)
{
final var arr = _threadDumpExtraContext.computeIfAbsent(Thread.currentThread(), (p1) -> Collections.synchronizedList(new ArrayList<>()));
final var arr = _threadDumpExtraContext.computeIfAbsent(Thread.currentThread(), (_) -> Collections.synchronizedList(new ArrayList<>()));
int size = arr.size();
arr.add(new ThreadExtraContext(context, MiniProfiler.getTroubleshootingStackTrace(), System.currentTimeMillis()));
return new _PopAutoCloseable(size);
Expand Down Expand Up @@ -393,6 +406,44 @@ public static synchronized void dumpThreads(LoggerWriter logWriter)
logWriter.debug("Completed dump of all open connections");
logWriter.debug("*********************************************");
}

// GitHib Issue 713: Automatically include PG locks and active queries in thread dumps
UserSchema schema = QueryService.get().getUserSchema(User.getAdminServiceUser(), ContainerManager.getRoot(), BasePostgreSqlDialect.POSTGRES_SCHEMA_NAME);
// Schema won't exist on SQLServer
if (schema != null)
{
writeTable(logWriter, schema, BasePostgreSqlDialect.POSTGRES_STAT_ACTIVITY_TABLE_NAME, "Postgres activity");
writeTable(logWriter, schema, BasePostgreSqlDialect.POSTGRES_LOCKS_TABLE_NAME, "Postgres locks");
}
}

private static void writeTable(LoggerWriter logWriter, UserSchema schema, String tableName, String header)
{
QueryForm form = new QueryForm();
try (var _ = ViewContext.pushMockViewContext(schema.getUser(), schema.getContainer(), new ActionURL()))
{
form.setViewContext(HttpView.currentContext());
form.setSchemaName(schema.getName());
form.setQueryName(tableName);
QueryView view = QueryView.create(form, new NullSafeBindException(new Object(), "form"));
logWriter.debug("Starting dump of " + header);
logWriter.debug("*********************************************");
try (TSVWriter writer = view.getTsvWriter())
{
StringWriter stringWriter = new StringWriter();
PrintWriter printWriter = new PrintWriter(stringWriter);
writer.write(printWriter);
printWriter.flush();
logWriter.debug("\n" + stringWriter.toString());
}
catch (IOException e)
{
logWriter.error("Failed to write " + header, e);
}
logWriter.debug("*********************************************");
logWriter.debug("Completed dump of " + header);
logWriter.debug("*********************************************");
}
}

static private final Set<String> skipMethods = Set.of("pushThreadDumpContext", "beginTransaction", "ensureTransaction", "execute", "getTroubleshootingStackTrace");
Expand Down
7 changes: 4 additions & 3 deletions core/src/org/labkey/core/admin/AdminController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@
import org.labkey.api.data.TableInfo;
import org.labkey.api.data.TransactionFilter;
import org.labkey.api.data.WorkbookContainerType;
import org.labkey.api.data.dialect.BasePostgreSqlDialect;
import org.labkey.api.data.dialect.SqlDialect.ExecutionPlanType;
import org.labkey.api.data.queryprofiler.QueryProfiler;
import org.labkey.api.data.queryprofiler.QueryProfiler.QueryStatTsvWriter;
Expand Down Expand Up @@ -2645,7 +2646,7 @@ public class PostgresStatActivityAction extends AbstractPostgresAction
{
public PostgresStatActivityAction()
{
super(PostgresUserSchema.POSTGRES_STAT_ACTIVITY_TABLE_NAME);
super(BasePostgreSqlDialect.POSTGRES_STAT_ACTIVITY_TABLE_NAME);
}
}

Expand All @@ -2654,7 +2655,7 @@ public class PostgresLocksAction extends AbstractPostgresAction
{
public PostgresLocksAction()
{
super(PostgresUserSchema.POSTGRES_LOCKS_TABLE_NAME);
super(BasePostgreSqlDialect.POSTGRES_LOCKS_TABLE_NAME);
}
}

Expand All @@ -2663,7 +2664,7 @@ public class PostgresTableSizesAction extends AbstractPostgresAction
{
public PostgresTableSizesAction()
{
super(PostgresUserSchema.POSTGRES_TABLE_SIZES_TABLE_NAME);
super(BasePostgreSqlDialect.POSTGRES_TABLE_SIZES_TABLE_NAME);
}
}

Expand Down
5 changes: 3 additions & 2 deletions core/src/org/labkey/core/query/PostgresLocksTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,15 @@
import org.labkey.api.data.BaseColumnInfo;
import org.labkey.api.data.JdbcType;
import org.labkey.api.data.SQLFragment;
import org.labkey.api.data.dialect.BasePostgreSqlDialect;
import org.labkey.api.query.QueryForeignKey;

/** Backed by pg_locks view */
public class PostgresLocksTable extends AbstractPostgresAdminOnlyTable
{
public PostgresLocksTable(@NotNull PostgresUserSchema userSchema)
{
super(PostgresUserSchema.POSTGRES_LOCKS_TABLE_NAME, userSchema);
super(BasePostgreSqlDialect.POSTGRES_LOCKS_TABLE_NAME, userSchema);

setDescription("Shows info about the currently held Postgres locks");

Expand All @@ -28,7 +29,7 @@ public PostgresLocksTable(@NotNull PostgresUserSchema userSchema)
addColumn(new BaseColumnInfo("objsubid", this, JdbcType.INTEGER));
addColumn(new BaseColumnInfo("virtualtransaction", this, JdbcType.VARCHAR));
addColumn(new BaseColumnInfo("pid", this, JdbcType.INTEGER)).
setFk(new QueryForeignKey.Builder(userSchema, null).table(PostgresUserSchema.POSTGRES_STAT_ACTIVITY_TABLE_NAME).raw(true));
setFk(new QueryForeignKey.Builder(userSchema, null).table(BasePostgreSqlDialect.POSTGRES_STAT_ACTIVITY_TABLE_NAME).raw(true));
addColumn(new BaseColumnInfo("mode", this, JdbcType.VARCHAR));
addColumn(new BaseColumnInfo("granted", this, JdbcType.BOOLEAN));
addColumn(new BaseColumnInfo("fastpath", this, JdbcType.BOOLEAN));
Expand Down
15 changes: 10 additions & 5 deletions core/src/org/labkey/core/query/PostgresStatActivityTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.labkey.api.data.TableInfo;
import org.labkey.api.data.TableSelector;
import org.labkey.api.data.TransactionFilter;
import org.labkey.api.data.dialect.BasePostgreSqlDialect;
import org.labkey.api.query.AbstractQueryUpdateService;
import org.labkey.api.query.ExprColumn;
import org.labkey.api.query.FieldKey;
Expand All @@ -29,6 +30,8 @@
import org.labkey.api.security.permissions.ApplicationAdminPermission;
import org.labkey.api.security.permissions.DeletePermission;
import org.labkey.api.security.permissions.Permission;
import org.labkey.api.util.DateUtil;
import org.labkey.api.util.HtmlString;
import org.labkey.api.util.LinkBuilder;
import org.labkey.api.util.logging.LogHelper;
import org.labkey.api.view.ActionURL;
Expand All @@ -47,7 +50,7 @@ public class PostgresStatActivityTable extends AbstractPostgresAdminOnlyTable

public PostgresStatActivityTable(@NotNull PostgresUserSchema userSchema)
{
super(PostgresUserSchema.POSTGRES_STAT_ACTIVITY_TABLE_NAME, userSchema);
super(BasePostgreSqlDialect.POSTGRES_STAT_ACTIVITY_TABLE_NAME, userSchema);

setDescription("Shows info about the active Postgres connections and their activity");

Expand All @@ -68,8 +71,10 @@ public PostgresStatActivityTable(@NotNull PostgresUserSchema userSchema)
addColumn(new BaseColumnInfo("client_port", this, JdbcType.INTEGER));
addColumn(new BaseColumnInfo("backend_start", this, JdbcType.TIMESTAMP));
addColumn(new BaseColumnInfo("xact_start", this, JdbcType.TIMESTAMP));
addColumn(new BaseColumnInfo("query_start", this, JdbcType.TIMESTAMP));
addColumn(new BaseColumnInfo("state_change", this, JdbcType.TIMESTAMP));
addColumn(new BaseColumnInfo("query_start", this, JdbcType.TIMESTAMP)).
setFormat(DateUtil.ISO_DATE_TIME_FORMAT_STRING);
addColumn(new BaseColumnInfo("state_change", this, JdbcType.TIMESTAMP)).
setFormat(DateUtil.ISO_DATE_TIME_FORMAT_STRING);
addColumn(new BaseColumnInfo("wait_event_type", this, JdbcType.VARCHAR));
addColumn(new BaseColumnInfo("wait_event", this, JdbcType.VARCHAR));
addColumn(new BaseColumnInfo("state", this, JdbcType.VARCHAR));
Expand Down Expand Up @@ -231,14 +236,14 @@ public void renderGridCellContents(RenderContext ctx, HtmlWriter out)
}
}
}
String separator = "";
HtmlString separator = HtmlString.EMPTY_STRING;
for (Thread thread : threads)
{
out.write(separator);
ActionURL url = new ActionURL(AdminController.ShowThreadsAction.class, ContainerManager.getRoot());
url.setFragment(thread.getName());
out.write(LinkBuilder.labkeyLink(thread.getName(), url).target("_blank"));
separator = "\n<br/>";
separator = HtmlString.BR;

// Check for HTTP threads and their async counterparts to tie queries to the request that spawned them
var request = TransactionFilter.getRequestSummary(thread);
Expand Down
3 changes: 2 additions & 1 deletion core/src/org/labkey/core/query/PostgresTableSizesTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,14 @@
import org.labkey.api.data.BaseColumnInfo;
import org.labkey.api.data.JdbcType;
import org.labkey.api.data.SQLFragment;
import org.labkey.api.data.dialect.BasePostgreSqlDialect;

/** Backed by pg_locks view */
public class PostgresTableSizesTable extends AbstractPostgresAdminOnlyTable
{
public PostgresTableSizesTable(@NotNull PostgresUserSchema userSchema)
{
super(PostgresUserSchema.POSTGRES_TABLE_SIZES_TABLE_NAME, userSchema);
super(BasePostgreSqlDialect.POSTGRES_TABLE_SIZES_TABLE_NAME, userSchema);

setDescription("Shows info Postgres table sizes");

Expand Down
16 changes: 6 additions & 10 deletions core/src/org/labkey/core/query/PostgresUserSchema.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,6 @@
/** Issue 52190: Expose troubleshooting data that supports postgreSQL-specific analysis */
public class PostgresUserSchema extends UserSchema
{
public static final String POSTGRES_STAT_ACTIVITY_TABLE_NAME = "pg_stat_activity";
public static final String POSTGRES_LOCKS_TABLE_NAME = "pg_locks";
public static final String POSTGRES_TABLE_SIZES_TABLE_NAME = "pg_tablesizes";

public PostgresUserSchema(User user, Container container)
{
super(BasePostgreSqlDialect.POSTGRES_SCHEMA_NAME, "Postgres-specific internal views for database troubleshooting", user, container, CoreSchema.getInstance().getSchema());
Expand All @@ -33,11 +29,11 @@ public boolean canReadSchema()
@Override
public @Nullable TableInfo createTable(String name, ContainerFilter cf)
{
if (POSTGRES_STAT_ACTIVITY_TABLE_NAME.equalsIgnoreCase(name))
if (BasePostgreSqlDialect.POSTGRES_STAT_ACTIVITY_TABLE_NAME.equalsIgnoreCase(name))
return new PostgresStatActivityTable(this);
if (POSTGRES_LOCKS_TABLE_NAME.equalsIgnoreCase(name))
if (BasePostgreSqlDialect.POSTGRES_LOCKS_TABLE_NAME.equalsIgnoreCase(name))
return new PostgresLocksTable(this);
if (POSTGRES_TABLE_SIZES_TABLE_NAME.equalsIgnoreCase(name))
if (BasePostgreSqlDialect.POSTGRES_TABLE_SIZES_TABLE_NAME.equalsIgnoreCase(name))
return new PostgresTableSizesTable(this);

return null;
Expand All @@ -47,8 +43,8 @@ public boolean canReadSchema()
public Set<String> getTableNames()
{
return Set.of(
POSTGRES_LOCKS_TABLE_NAME,
POSTGRES_STAT_ACTIVITY_TABLE_NAME,
POSTGRES_TABLE_SIZES_TABLE_NAME);
BasePostgreSqlDialect.POSTGRES_LOCKS_TABLE_NAME,
BasePostgreSqlDialect.POSTGRES_STAT_ACTIVITY_TABLE_NAME,
BasePostgreSqlDialect.POSTGRES_TABLE_SIZES_TABLE_NAME);
}
}