Skip to content

Perf: DB query optimization #29

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cchwala merged 5 commits into from
Mar 30, 2026
Merged

Perf: DB query optimization #29

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d5ac22d
perf: tune PostgreSQL memory settings for 8 GB VM
cchwala Mar 30, 2026
4e018bb
perf: adaptive $__interval bucketing, three-tier auto/raw query split
cchwala Mar 30, 2026
0db2807
perf: optimize SQL query for CML time series data with min, max, and …
cchwala Mar 30, 2026
1b5053c
perf: change timeseries panel to never show points for improved clarity
cchwala Mar 30, 2026
0e9af2c
perf: enable TimescaleDB compression after 7 days, increase shared_bu…
cchwala Mar 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 23 additions & 1 deletion database/init.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,4 +112,26 @@ SELECT add_continuous_aggregate_policy('cml_data_1h',
start_offset => INTERVAL '2 days',
end_offset => INTERVAL '1 hour',
schedule_interval => INTERVAL '1 hour'
);
);

-- ---------------------------------------------------------------------------
-- Compression for cml_data chunks older than 7 days.
--
-- compress_segmentby: each compressed segment contains one (cml_id, sublink_id)
-- pair, so a query filtered to a single CML decompresses only ~1/728th of a
-- chunk — not the whole thing.
-- compress_orderby: matches the query pattern (time range scans), allowing
-- skip-scan decompression for narrow time windows within a segment.
--
-- At ~10-20x compression ratio, the last month of data fits in shared_buffers
-- after a single cache warm-up, regardless of how many new streams are added.
-- The current uncompressed week chunk is left untouched so real-time ingestion
-- and detail-view queries on recent data have no decompression overhead.
-- ---------------------------------------------------------------------------
ALTER TABLE cml_data SET (
timescaledb.compress,
timescaledb.compress_segmentby = 'cml_id, sublink_id',
timescaledb.compress_orderby = 'time DESC'
);

SELECT add_compression_policy('cml_data', INTERVAL '7 days');
12 changes: 12 additions & 0 deletions docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,18 @@ services:
build: ./database
ports:
- "5432:5432"
# Tune PostgreSQL memory for an 8 GB VM.
# shared_buffers: keep recently-used chunks in RAM (default is only 128 MB).
# effective_cache_size: hints to the planner how much OS page cache is available.
# work_mem: memory per sort/hash operation; speeds up ORDER BY on large result sets.
# random_page_cost: tell the planner data is effectively cached, prefer index scans.
command: >
postgres
-c shared_buffers=2GB
-c effective_cache_size=4GB
-c work_mem=64MB
-c maintenance_work_mem=256MB
-c random_page_cost=1.1
healthcheck:
test: ["CMD-SHELL", "pg_isready -U myuser -d mydatabase"]
interval: 5s
Expand Down
34 changes: 27 additions & 7 deletions grafana/provisioning/dashboards/definitions/cml-realtime.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
Expand Down Expand Up @@ -373,7 +373,7 @@
},
"format": "time_series",
"rawQuery": true,
"rawSql": "SELECT\n bucket AS \"time\",\n sublink_id || ' min' AS metric,\n rsl_min AS value\nFROM cml_data_1h\nWHERE cml_id = '${cml_id}'\n AND bucket >= $__timeFrom()::timestamptz\n AND bucket <= $__timeTo()::timestamptz\nORDER BY 1 ASC",
"rawSql": "SELECT\n bucket AS \"time\",\n sublink_id || ' min' AS metric,\n rsl_min AS value\nFROM cml_data_1h\nWHERE cml_id = '${cml_id}'\n AND '${interval}' = 'auto'\n AND EXTRACT(EPOCH FROM ($__timeTo()::timestamptz - $__timeFrom()::timestamptz)) > 259200\n AND bucket >= $__timeFrom()::timestamptz\n AND bucket <= $__timeTo()::timestamptz\nORDER BY 1 ASC",
"refId": "A"
},
{
Expand All @@ -383,7 +383,7 @@
},
"format": "time_series",
"rawQuery": true,
"rawSql": "SELECT\n bucket AS \"time\",\n sublink_id || ' max' AS metric,\n rsl_max AS value\nFROM cml_data_1h\nWHERE cml_id = '${cml_id}'\n AND bucket >= $__timeFrom()::timestamptz\n AND bucket <= $__timeTo()::timestamptz\nORDER BY 1 ASC",
"rawSql": "SELECT\n bucket AS \"time\",\n sublink_id || ' max' AS metric,\n rsl_max AS value\nFROM cml_data_1h\nWHERE cml_id = '${cml_id}'\n AND '${interval}' = 'auto'\n AND EXTRACT(EPOCH FROM ($__timeTo()::timestamptz - $__timeFrom()::timestamptz)) > 259200\n AND bucket >= $__timeFrom()::timestamptz\n AND bucket <= $__timeTo()::timestamptz\nORDER BY 1 ASC",
"refId": "B"
},
{
Expand All @@ -403,8 +403,18 @@
},
"format": "time_series",
"rawQuery": true,
"rawSql": "SELECT\n time AS \"time\",\n sublink_id AS metric,\n rsl AS value\nFROM cml_data\nWHERE cml_id = '${cml_id}'\n AND (\n ('${interval}' = 'auto' AND EXTRACT(EPOCH FROM ($__timeTo()::timestamptz - $__timeFrom()::timestamptz)) <= 259200)\n OR '${interval}' = 'raw'\n )\n AND time >= $__timeFrom()::timestamptz\n AND time <= $__timeTo()::timestamptz\nORDER BY 1 ASC",
"rawSql": "WITH bucketed AS (\n SELECT\n time_bucket('$__interval', time) AS bucket,\n sublink_id,\n MIN(rsl) AS rsl_min,\n MAX(rsl) AS rsl_max,\n AVG(rsl) AS rsl_avg\n FROM cml_data\n WHERE cml_id = '${cml_id}'\n AND '${interval}' = 'auto'\n AND EXTRACT(EPOCH FROM ($__timeTo()::timestamptz - $__timeFrom()::timestamptz)) <= 259200\n AND time >= $__timeFrom()::timestamptz\n AND time <= $__timeTo()::timestamptz\n GROUP BY 1, 2\n)\nSELECT bucket AS \"time\", sublink_id || ' min' AS metric, rsl_min AS value FROM bucketed\nUNION ALL\nSELECT bucket AS \"time\", sublink_id || ' max' AS metric, rsl_max AS value FROM bucketed\nUNION ALL\nSELECT bucket AS \"time\", sublink_id || ' avg' AS metric, rsl_avg AS value FROM bucketed\nORDER BY 1 ASC",
"refId": "D"
},
{
"datasource": {
"type": "grafana-postgresql-datasource",
"uid": "PostgreSQL"
},
"format": "time_series",
"rawQuery": true,
"rawSql": "SELECT\n time AS \"time\",\n sublink_id AS metric,\n rsl AS value\nFROM cml_data\nWHERE cml_id = '${cml_id}'\n AND '${interval}' = 'raw'\n AND time >= $__timeFrom()::timestamptz\n AND time <= $__timeTo()::timestamptz\nORDER BY 1 ASC",
"refId": "E"
}
],
"title": "CML Time Series - Received Signal Level",
Expand Down Expand Up @@ -708,7 +718,7 @@
},
"format": "time_series",
"rawQuery": true,
"rawSql": "SELECT\n bucket AS \"time\",\n sublink_id || ' min' AS metric,\n tsl_min AS value\nFROM cml_data_1h\nWHERE cml_id = '${cml_id}'\n AND bucket >= $__timeFrom()::timestamptz\n AND bucket <= $__timeTo()::timestamptz\nORDER BY 1 ASC",
"rawSql": "SELECT\n bucket AS \"time\",\n sublink_id || ' min' AS metric,\n tsl_min AS value\nFROM cml_data_1h\nWHERE cml_id = '${cml_id}'\n AND '${interval}' = 'auto'\n AND EXTRACT(EPOCH FROM ($__timeTo()::timestamptz - $__timeFrom()::timestamptz)) > 259200\n AND bucket >= $__timeFrom()::timestamptz\n AND bucket <= $__timeTo()::timestamptz\nORDER BY 1 ASC",
"refId": "A"
},
{
Expand All @@ -718,7 +728,7 @@
},
"format": "time_series",
"rawQuery": true,
"rawSql": "SELECT\n bucket AS \"time\",\n sublink_id || ' max' AS metric,\n tsl_max AS value\nFROM cml_data_1h\nWHERE cml_id = '${cml_id}'\n AND bucket >= $__timeFrom()::timestamptz\n AND bucket <= $__timeTo()::timestamptz\nORDER BY 1 ASC",
"rawSql": "SELECT\n bucket AS \"time\",\n sublink_id || ' max' AS metric,\n tsl_max AS value\nFROM cml_data_1h\nWHERE cml_id = '${cml_id}'\n AND '${interval}' = 'auto'\n AND EXTRACT(EPOCH FROM ($__timeTo()::timestamptz - $__timeFrom()::timestamptz)) > 259200\n AND bucket >= $__timeFrom()::timestamptz\n AND bucket <= $__timeTo()::timestamptz\nORDER BY 1 ASC",
"refId": "B"
},
{
Expand All @@ -738,8 +748,18 @@
},
"format": "time_series",
"rawQuery": true,
"rawSql": "SELECT\n time AS \"time\",\n sublink_id AS metric,\n tsl AS value\nFROM cml_data\nWHERE cml_id = '${cml_id}'\n AND (\n ('${interval}' = 'auto' AND EXTRACT(EPOCH FROM ($__timeTo()::timestamptz - $__timeFrom()::timestamptz)) <= 259200)\n OR '${interval}' = 'raw'\n )\n AND time >= $__timeFrom()::timestamptz\n AND time <= $__timeTo()::timestamptz\nORDER BY 1 ASC",
"rawSql": "SELECT\n time_bucket('$__interval', time) AS \"time\",\n sublink_id AS metric,\n AVG(tsl) AS value\nFROM cml_data\nWHERE cml_id = '${cml_id}'\n AND '${interval}' = 'auto'\n AND EXTRACT(EPOCH FROM ($__timeTo()::timestamptz - $__timeFrom()::timestamptz)) <= 259200\n AND time >= $__timeFrom()::timestamptz\n AND time <= $__timeTo()::timestamptz\nGROUP BY 1, 2\nORDER BY 1 ASC",
"refId": "D"
},
{
"datasource": {
"type": "grafana-postgresql-datasource",
"uid": "PostgreSQL"
},
"format": "time_series",
"rawQuery": true,
"rawSql": "SELECT\n time AS \"time\",\n sublink_id AS metric,\n tsl AS value\nFROM cml_data\nWHERE cml_id = '${cml_id}'\n AND '${interval}' = 'raw'\n AND time >= $__timeFrom()::timestamptz\n AND time <= $__timeTo()::timestamptz\nORDER BY 1 ASC",
"refId": "E"
}
],
"title": "CML Time Series - Transmitted Signal Level",
Expand Down
Loading