diff --git a/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java b/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java index 364e1affefa..0ecd40466c3 100644 --- a/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java +++ b/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java @@ -70,6 +70,10 @@ public abstract class BasePostgreSqlDialect extends SqlDialect // Issue 52190: Expose troubleshooting data that supports postgreSQL-specific analysis public static final String POSTGRES_SCHEMA_NAME = "postgres"; + public static final String POSTGRES_STAT_ACTIVITY_TABLE_NAME = "pg_stat_activity"; + public static final String POSTGRES_LOCKS_TABLE_NAME = "pg_locks"; + public static final String POSTGRES_TABLE_SIZES_TABLE_NAME = "pg_tablesizes"; + private final Map _domainScaleMap = new CopyOnWriteHashMap<>(); private HtmlString _adminWarning = null; diff --git a/api/src/org/labkey/api/util/DateUtil.java b/api/src/org/labkey/api/util/DateUtil.java index e4db870da6a..af369e4bed9 100644 --- a/api/src/org/labkey/api/util/DateUtil.java +++ b/api/src/org/labkey/api/util/DateUtil.java @@ -87,12 +87,12 @@ private DateUtil() private static final Locale _localeDefault = Locale.getDefault(); private static final TimeZone _timezoneDefault = TimeZone.getDefault(); - private static final String ISO_DATE_FORMAT_STRING = "yyyy-MM-dd"; - private static final String ISO_SHORT_TIME_FORMAT_STRING = "HH:mm"; - private static final String ISO_DATE_SHORT_TIME_FORMAT_STRING = ISO_DATE_FORMAT_STRING + " " + ISO_SHORT_TIME_FORMAT_STRING; - private static final String ISO_TIME_FORMAT_STRING = "HH:mm:ss"; - private static final String ISO_LONG_TIME_FORMAT_STRING = "HH:mm:ss.SSS"; - private static final String ISO_DATE_TIME_FORMAT_STRING = ISO_DATE_FORMAT_STRING + " " + ISO_LONG_TIME_FORMAT_STRING; + public static final String ISO_DATE_FORMAT_STRING = "yyyy-MM-dd"; + public static final String ISO_SHORT_TIME_FORMAT_STRING = "HH:mm"; + public static final String ISO_DATE_SHORT_TIME_FORMAT_STRING = ISO_DATE_FORMAT_STRING + " " + ISO_SHORT_TIME_FORMAT_STRING; + public static final String ISO_TIME_FORMAT_STRING = "HH:mm:ss"; + public static final String ISO_LONG_TIME_FORMAT_STRING = "HH:mm:ss.SSS"; + public static final String ISO_DATE_TIME_FORMAT_STRING = ISO_DATE_FORMAT_STRING + " " + ISO_LONG_TIME_FORMAT_STRING; // SimpleDataFormat does not support microseconds, it can only support up to milliseconds private static final Pattern NON_SIMPLE_PRECISION_TIME_PATTERN = Pattern.compile(".*([0-5][0-9]):([0-5][0-9])\\.(\\d{4,6}).*"); diff --git a/api/src/org/labkey/api/util/DebugInfoDumper.java b/api/src/org/labkey/api/util/DebugInfoDumper.java index 2b74f17e54d..c252ab34bb2 100644 --- a/api/src/org/labkey/api/util/DebugInfoDumper.java +++ b/api/src/org/labkey/api/util/DebugInfoDumper.java @@ -18,20 +18,33 @@ import org.apache.commons.collections4.multimap.HashSetValuedHashMap; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.Logger; +import org.labkey.api.action.NullSafeBindException; import org.labkey.api.data.ConnectionWrapper; +import org.labkey.api.data.ContainerManager; import org.labkey.api.data.DbScope; +import org.labkey.api.data.TSVWriter; import org.labkey.api.data.TransactionFilter; +import org.labkey.api.data.dialect.BasePostgreSqlDialect; import org.labkey.api.files.FileSystemDirectoryListener; import org.labkey.api.files.FileSystemWatchers; import org.labkey.api.miniprofiler.MiniProfiler; import org.labkey.api.module.ModuleLoader; +import org.labkey.api.query.QueryForm; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.ViewContext; import org.labkey.api.writer.PrintWriters; import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; import java.io.PrintWriter; +import java.io.StringWriter; import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; import java.lang.reflect.InvocationTargetException; @@ -151,7 +164,7 @@ record ThreadExtraContext(String context, StackTraceElement[] stack, long startT */ public static _PopAutoCloseable pushThreadDumpContext(String context) { - final var arr = _threadDumpExtraContext.computeIfAbsent(Thread.currentThread(), (p1) -> Collections.synchronizedList(new ArrayList<>())); + final var arr = _threadDumpExtraContext.computeIfAbsent(Thread.currentThread(), (_) -> Collections.synchronizedList(new ArrayList<>())); int size = arr.size(); arr.add(new ThreadExtraContext(context, MiniProfiler.getTroubleshootingStackTrace(), System.currentTimeMillis())); return new _PopAutoCloseable(size); @@ -393,6 +406,44 @@ public static synchronized void dumpThreads(LoggerWriter logWriter) logWriter.debug("Completed dump of all open connections"); logWriter.debug("*********************************************"); } + + // GitHib Issue 713: Automatically include PG locks and active queries in thread dumps + UserSchema schema = QueryService.get().getUserSchema(User.getAdminServiceUser(), ContainerManager.getRoot(), BasePostgreSqlDialect.POSTGRES_SCHEMA_NAME); + // Schema won't exist on SQLServer + if (schema != null) + { + writeTable(logWriter, schema, BasePostgreSqlDialect.POSTGRES_STAT_ACTIVITY_TABLE_NAME, "Postgres activity"); + writeTable(logWriter, schema, BasePostgreSqlDialect.POSTGRES_LOCKS_TABLE_NAME, "Postgres locks"); + } + } + + private static void writeTable(LoggerWriter logWriter, UserSchema schema, String tableName, String header) + { + QueryForm form = new QueryForm(); + try (var _ = ViewContext.pushMockViewContext(schema.getUser(), schema.getContainer(), new ActionURL())) + { + form.setViewContext(HttpView.currentContext()); + form.setSchemaName(schema.getName()); + form.setQueryName(tableName); + QueryView view = QueryView.create(form, new NullSafeBindException(new Object(), "form")); + logWriter.debug("Starting dump of " + header); + logWriter.debug("*********************************************"); + try (TSVWriter writer = view.getTsvWriter()) + { + StringWriter stringWriter = new StringWriter(); + PrintWriter printWriter = new PrintWriter(stringWriter); + writer.write(printWriter); + printWriter.flush(); + logWriter.debug("\n" + stringWriter.toString()); + } + catch (IOException e) + { + logWriter.error("Failed to write " + header, e); + } + logWriter.debug("*********************************************"); + logWriter.debug("Completed dump of " + header); + logWriter.debug("*********************************************"); + } } static private final Set skipMethods = Set.of("pushThreadDumpContext", "beginTransaction", "ensureTransaction", "execute", "getTroubleshootingStackTrace"); diff --git a/core/src/org/labkey/core/admin/AdminController.java b/core/src/org/labkey/core/admin/AdminController.java index 19266ce4987..90a29aff303 100644 --- a/core/src/org/labkey/core/admin/AdminController.java +++ b/core/src/org/labkey/core/admin/AdminController.java @@ -133,6 +133,7 @@ import org.labkey.api.data.TableInfo; import org.labkey.api.data.TransactionFilter; import org.labkey.api.data.WorkbookContainerType; +import org.labkey.api.data.dialect.BasePostgreSqlDialect; import org.labkey.api.data.dialect.SqlDialect.ExecutionPlanType; import org.labkey.api.data.queryprofiler.QueryProfiler; import org.labkey.api.data.queryprofiler.QueryProfiler.QueryStatTsvWriter; @@ -2645,7 +2646,7 @@ public class PostgresStatActivityAction extends AbstractPostgresAction { public PostgresStatActivityAction() { - super(PostgresUserSchema.POSTGRES_STAT_ACTIVITY_TABLE_NAME); + super(BasePostgreSqlDialect.POSTGRES_STAT_ACTIVITY_TABLE_NAME); } } @@ -2654,7 +2655,7 @@ public class PostgresLocksAction extends AbstractPostgresAction { public PostgresLocksAction() { - super(PostgresUserSchema.POSTGRES_LOCKS_TABLE_NAME); + super(BasePostgreSqlDialect.POSTGRES_LOCKS_TABLE_NAME); } } @@ -2663,7 +2664,7 @@ public class PostgresTableSizesAction extends AbstractPostgresAction { public PostgresTableSizesAction() { - super(PostgresUserSchema.POSTGRES_TABLE_SIZES_TABLE_NAME); + super(BasePostgreSqlDialect.POSTGRES_TABLE_SIZES_TABLE_NAME); } } diff --git a/core/src/org/labkey/core/query/PostgresLocksTable.java b/core/src/org/labkey/core/query/PostgresLocksTable.java index d2c473d3cdc..c64371daabc 100644 --- a/core/src/org/labkey/core/query/PostgresLocksTable.java +++ b/core/src/org/labkey/core/query/PostgresLocksTable.java @@ -4,6 +4,7 @@ import org.labkey.api.data.BaseColumnInfo; import org.labkey.api.data.JdbcType; import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.dialect.BasePostgreSqlDialect; import org.labkey.api.query.QueryForeignKey; /** Backed by pg_locks view */ @@ -11,7 +12,7 @@ public class PostgresLocksTable extends AbstractPostgresAdminOnlyTable { public PostgresLocksTable(@NotNull PostgresUserSchema userSchema) { - super(PostgresUserSchema.POSTGRES_LOCKS_TABLE_NAME, userSchema); + super(BasePostgreSqlDialect.POSTGRES_LOCKS_TABLE_NAME, userSchema); setDescription("Shows info about the currently held Postgres locks"); @@ -28,7 +29,7 @@ public PostgresLocksTable(@NotNull PostgresUserSchema userSchema) addColumn(new BaseColumnInfo("objsubid", this, JdbcType.INTEGER)); addColumn(new BaseColumnInfo("virtualtransaction", this, JdbcType.VARCHAR)); addColumn(new BaseColumnInfo("pid", this, JdbcType.INTEGER)). - setFk(new QueryForeignKey.Builder(userSchema, null).table(PostgresUserSchema.POSTGRES_STAT_ACTIVITY_TABLE_NAME).raw(true)); + setFk(new QueryForeignKey.Builder(userSchema, null).table(BasePostgreSqlDialect.POSTGRES_STAT_ACTIVITY_TABLE_NAME).raw(true)); addColumn(new BaseColumnInfo("mode", this, JdbcType.VARCHAR)); addColumn(new BaseColumnInfo("granted", this, JdbcType.BOOLEAN)); addColumn(new BaseColumnInfo("fastpath", this, JdbcType.BOOLEAN)); diff --git a/core/src/org/labkey/core/query/PostgresStatActivityTable.java b/core/src/org/labkey/core/query/PostgresStatActivityTable.java index 3adc8b2fea6..14af25c7683 100644 --- a/core/src/org/labkey/core/query/PostgresStatActivityTable.java +++ b/core/src/org/labkey/core/query/PostgresStatActivityTable.java @@ -18,6 +18,7 @@ import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.data.TransactionFilter; +import org.labkey.api.data.dialect.BasePostgreSqlDialect; import org.labkey.api.query.AbstractQueryUpdateService; import org.labkey.api.query.ExprColumn; import org.labkey.api.query.FieldKey; @@ -29,6 +30,8 @@ import org.labkey.api.security.permissions.ApplicationAdminPermission; import org.labkey.api.security.permissions.DeletePermission; import org.labkey.api.security.permissions.Permission; +import org.labkey.api.util.DateUtil; +import org.labkey.api.util.HtmlString; import org.labkey.api.util.LinkBuilder; import org.labkey.api.util.logging.LogHelper; import org.labkey.api.view.ActionURL; @@ -47,7 +50,7 @@ public class PostgresStatActivityTable extends AbstractPostgresAdminOnlyTable public PostgresStatActivityTable(@NotNull PostgresUserSchema userSchema) { - super(PostgresUserSchema.POSTGRES_STAT_ACTIVITY_TABLE_NAME, userSchema); + super(BasePostgreSqlDialect.POSTGRES_STAT_ACTIVITY_TABLE_NAME, userSchema); setDescription("Shows info about the active Postgres connections and their activity"); @@ -68,8 +71,10 @@ public PostgresStatActivityTable(@NotNull PostgresUserSchema userSchema) addColumn(new BaseColumnInfo("client_port", this, JdbcType.INTEGER)); addColumn(new BaseColumnInfo("backend_start", this, JdbcType.TIMESTAMP)); addColumn(new BaseColumnInfo("xact_start", this, JdbcType.TIMESTAMP)); - addColumn(new BaseColumnInfo("query_start", this, JdbcType.TIMESTAMP)); - addColumn(new BaseColumnInfo("state_change", this, JdbcType.TIMESTAMP)); + addColumn(new BaseColumnInfo("query_start", this, JdbcType.TIMESTAMP)). + setFormat(DateUtil.ISO_DATE_TIME_FORMAT_STRING); + addColumn(new BaseColumnInfo("state_change", this, JdbcType.TIMESTAMP)). + setFormat(DateUtil.ISO_DATE_TIME_FORMAT_STRING); addColumn(new BaseColumnInfo("wait_event_type", this, JdbcType.VARCHAR)); addColumn(new BaseColumnInfo("wait_event", this, JdbcType.VARCHAR)); addColumn(new BaseColumnInfo("state", this, JdbcType.VARCHAR)); @@ -231,14 +236,14 @@ public void renderGridCellContents(RenderContext ctx, HtmlWriter out) } } } - String separator = ""; + HtmlString separator = HtmlString.EMPTY_STRING; for (Thread thread : threads) { out.write(separator); ActionURL url = new ActionURL(AdminController.ShowThreadsAction.class, ContainerManager.getRoot()); url.setFragment(thread.getName()); out.write(LinkBuilder.labkeyLink(thread.getName(), url).target("_blank")); - separator = "\n
"; + separator = HtmlString.BR; // Check for HTTP threads and their async counterparts to tie queries to the request that spawned them var request = TransactionFilter.getRequestSummary(thread); diff --git a/core/src/org/labkey/core/query/PostgresTableSizesTable.java b/core/src/org/labkey/core/query/PostgresTableSizesTable.java index 7f31ba96b9b..ea8da83cbc0 100644 --- a/core/src/org/labkey/core/query/PostgresTableSizesTable.java +++ b/core/src/org/labkey/core/query/PostgresTableSizesTable.java @@ -4,13 +4,14 @@ import org.labkey.api.data.BaseColumnInfo; import org.labkey.api.data.JdbcType; import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.dialect.BasePostgreSqlDialect; /** Backed by pg_locks view */ public class PostgresTableSizesTable extends AbstractPostgresAdminOnlyTable { public PostgresTableSizesTable(@NotNull PostgresUserSchema userSchema) { - super(PostgresUserSchema.POSTGRES_TABLE_SIZES_TABLE_NAME, userSchema); + super(BasePostgreSqlDialect.POSTGRES_TABLE_SIZES_TABLE_NAME, userSchema); setDescription("Shows info Postgres table sizes"); diff --git a/core/src/org/labkey/core/query/PostgresUserSchema.java b/core/src/org/labkey/core/query/PostgresUserSchema.java index 6ca8a60c00b..f058b00f030 100644 --- a/core/src/org/labkey/core/query/PostgresUserSchema.java +++ b/core/src/org/labkey/core/query/PostgresUserSchema.java @@ -15,10 +15,6 @@ /** Issue 52190: Expose troubleshooting data that supports postgreSQL-specific analysis */ public class PostgresUserSchema extends UserSchema { - public static final String POSTGRES_STAT_ACTIVITY_TABLE_NAME = "pg_stat_activity"; - public static final String POSTGRES_LOCKS_TABLE_NAME = "pg_locks"; - public static final String POSTGRES_TABLE_SIZES_TABLE_NAME = "pg_tablesizes"; - public PostgresUserSchema(User user, Container container) { super(BasePostgreSqlDialect.POSTGRES_SCHEMA_NAME, "Postgres-specific internal views for database troubleshooting", user, container, CoreSchema.getInstance().getSchema()); @@ -33,11 +29,11 @@ public boolean canReadSchema() @Override public @Nullable TableInfo createTable(String name, ContainerFilter cf) { - if (POSTGRES_STAT_ACTIVITY_TABLE_NAME.equalsIgnoreCase(name)) + if (BasePostgreSqlDialect.POSTGRES_STAT_ACTIVITY_TABLE_NAME.equalsIgnoreCase(name)) return new PostgresStatActivityTable(this); - if (POSTGRES_LOCKS_TABLE_NAME.equalsIgnoreCase(name)) + if (BasePostgreSqlDialect.POSTGRES_LOCKS_TABLE_NAME.equalsIgnoreCase(name)) return new PostgresLocksTable(this); - if (POSTGRES_TABLE_SIZES_TABLE_NAME.equalsIgnoreCase(name)) + if (BasePostgreSqlDialect.POSTGRES_TABLE_SIZES_TABLE_NAME.equalsIgnoreCase(name)) return new PostgresTableSizesTable(this); return null; @@ -47,8 +43,8 @@ public boolean canReadSchema() public Set getTableNames() { return Set.of( - POSTGRES_LOCKS_TABLE_NAME, - POSTGRES_STAT_ACTIVITY_TABLE_NAME, - POSTGRES_TABLE_SIZES_TABLE_NAME); + BasePostgreSqlDialect.POSTGRES_LOCKS_TABLE_NAME, + BasePostgreSqlDialect.POSTGRES_STAT_ACTIVITY_TABLE_NAME, + BasePostgreSqlDialect.POSTGRES_TABLE_SIZES_TABLE_NAME); } }