Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,18 @@
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

/**
* Usage time series of one node (contract op {@code getAdminNodeMetrics}),
* read live from the hypervisor's RRD store like the per-VM series. Unlike a
* VM, a node has no "not provisioned yet" state: a node row exists because the
* host exists, so a host that cannot be asked is an outage and answers 503.
*
* <p>Deliberately not {@code @Transactional}, for the same reason as the per-VM
* series: the hypervisor read can run to the client read timeout and must not
* hold a pooled database connection while it does. The one database read here
* carries its own transaction and only basic columns of the node are read
* afterwards.
*/
@Service
public class AdminNodeMetricsService {
Expand All @@ -39,7 +44,6 @@ public AdminNodeMetricsService(NodeRepository nodeRepository, ProxmoxClient prox
this.clock = clock;
}

@Transactional(readOnly = true)
public NodeMetricsResponse metrics(long nodeId, RrdTimeframe timeframe) {
Node node = nodeRepository.findById(nodeId).orElseThrow(() -> new ApiException(
HttpStatus.NOT_FOUND, ErrorCodes.RESOURCE_NOT_FOUND, "리소스를 찾을 수 없습니다",
Expand All @@ -48,9 +52,13 @@ public NodeMetricsResponse metrics(long nodeId, RrdTimeframe timeframe) {
try {
points = proxmoxClient.nodeRrdData(node.getApiHost(), node.getName(), timeframe)
.stream()
// A row with no timestamp has no place on the axis.
.filter(sample -> sample.time() != null)
.map(NodeMetricPointResponse::from)
.toList();
} catch (ProxmoxApiException e) {
} catch (ProxmoxApiException | IllegalStateException e) {
// An unconfigured API token refuses before the request leaves, which
// is the same "PVE cannot be asked" as a transport failure.
log.warn("Node {} usage read failed: {}", node.getName(), e.getMessage());
throw new ApiException(HttpStatus.SERVICE_UNAVAILABLE, ErrorCodes.METRICS_UNAVAILABLE,
"사용량 데이터를 불러오지 못했습니다", "잠시 후 다시 시도해 주세요.");
Expand Down
62 changes: 52 additions & 10 deletions src/main/java/kr/ac/pusan/pickle/admin/AdminSummaryService.java
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import kr.ac.pusan.pickle.config.ClockConfig;
import kr.ac.pusan.pickle.inventory.Node;
import kr.ac.pusan.pickle.inventory.NodeRepository;
import kr.ac.pusan.pickle.inventory.NodeStatus;
import kr.ac.pusan.pickle.ipam.IpPoolRepository;
import kr.ac.pusan.pickle.ipam.IpamService;
import kr.ac.pusan.pickle.orgs.OrgRepository;
Expand Down Expand Up @@ -158,7 +159,16 @@ and status not in ('DELETED', 'DELETING')
published, expiring30d, attention);
}

@Transactional(readOnly = true)
/**
* Platform panel. Deliberately not {@code @Transactional}: it ends in a
* live hypervisor probe per node, and a shared transaction would pin a
* pooled database connection for the whole of that — long enough, with a
* stalled pveproxy, for a refreshing dashboard to drain the pool and take
* unrelated endpoints down with it. Nothing here needs one transaction:
* every tile is an independent counter, each query and each collaborator
* carries its own read transaction, and the entities read afterwards are
* touched on basic columns only.
*/
public SystemDashboardSummaryResponse systemSummary() {
List<NodeRatio> nodes = adminNodeQueryService.listNodes().stream()
.map(AdminSummaryService::toNodeRatio)
Expand Down Expand Up @@ -212,24 +222,56 @@ select count(*) from vms v
* show. Both refusals the client can raise before a reply are caught for
* that reason — the HTTP/transport failure and the unconfigured-token
* refusal, which is the same "cannot ask PVE" from the operator's side.
*
* <p>The two probes are separate answers. Status decides reachability;
* storage is a second call behind a second right ({@code Datastore.Audit}),
* so losing it leaves the two storage fields null on a node that is still
* reachable rather than blanking a tile that did answer.
*
* <p>An OFFLINE node is not probed at all — the operator has already said
* the host is down, and asking it only buys a timeout per dashboard load
* (the status poller skips OFFLINE for the same reason). It still gets a
* row, so the node count stays whole, and that row is
* {@code reachable: false}: no live numbers are known either way, and the
* node's own status is already on the panel beside it in the ratio list,
* which is where "the operator parked it" is said.
*/
private List<NodeLiveResponse> nodesLive() {
List<NodeLiveResponse> live = new ArrayList<>();
for (Node node : nodeRepository.findAll(org.springframework.data.domain.Sort.by("id"))) {
if (node.getStatus() == NodeStatus.OFFLINE) {
live.add(NodeLiveResponse.unreachable(node.getId(), node.getName()));
continue;
}
NodeStatusInfo status;
try {
NodeStatusInfo status = proxmoxClient.nodeStatus(node.getApiHost(), node.getName());
NodeStorageStatus storage = guestStorage(node);
live.add(new NodeLiveResponse(node.getId(), node.getName(), true,
status.memory() == null ? null : status.memory().total(),
status.memory() == null ? null : status.memory().used(),
status.cpu(),
storage == null ? null : storage.total(),
storage == null ? null : storage.used(),
clock.instant()));
status = proxmoxClient.nodeStatus(node.getApiHost(), node.getName());
} catch (ProxmoxApiException | IllegalStateException e) {
log.warn("Node {} live probe failed: {}", node.getName(), e.getMessage());
live.add(NodeLiveResponse.unreachable(node.getId(), node.getName()));
continue;
}
// The client hands back whatever the PVE envelope held, so a 200
// with no data is possible and is not a reachable answer.
if (status == null) {
log.warn("Node {} live probe returned an empty status payload", node.getName());
live.add(NodeLiveResponse.unreachable(node.getId(), node.getName()));
continue;
}
NodeStorageStatus storage = null;
try {
storage = guestStorage(node);
} catch (ProxmoxApiException | IllegalStateException e) {
log.warn("Node {} guest-storage read failed, tile keeps the status half: {}",
node.getName(), e.getMessage());
}
live.add(new NodeLiveResponse(node.getId(), node.getName(), true,
status.memory() == null ? null : status.memory().total(),
status.memory() == null ? null : status.memory().used(),
status.cpu(),
storage == null ? null : storage.total(),
storage == null ? null : storage.used(),
clock.instant()));
}
return live;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,12 @@ public HeadroomResult headroom(Long orgId) {
* with the node, so one unmeasured ACTIVE node makes the sum a number
* smaller than the truth — which would read as less headroom than there is.
* The whole figure goes null in that case instead.
*
* <p>Which is what an added node looks like: a new node arrives with no
* measurement, so the disk capacity of the whole platform reads null until
* that node is measured, and every surface hanging off it (the dashboard
* disk bar, the capacity trend's disk line) empties for as long. That is
* this rule working, not a regression introduced by adding the node.
*/
public PlatformCapacity capacity() {
List<Node> nodes = nodeRepository.findByStatusOrderByIdAsc(NodeStatus.ACTIVE);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,15 @@
* right now, beside the allocation ratios the database already knows. Every
* measurement is nullable because the answer to "the host is not answering" is
* this row with {@code reachable} false — not a missing row and not an error.
*
* <p>{@code reachable} false also covers a node the platform chose not to ask:
* an OFFLINE node is skipped rather than waited on, and the answer is the same
* one either way — no live measurement is known. Why it is unknown is on the
* panel already, in the node's own status beside these numbers.
*
* <p>The measurements come from two independent hypervisor calls, so a
* reachable node can still carry null storage: the storage half needs a right
* the status half does not.
*/
public record NodeLiveResponse(
long nodeId,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,18 @@

import io.swagger.v3.oas.annotations.media.Schema;
import java.time.Instant;
import kr.ac.pusan.pickle.proxmox.RrdValues;
import kr.ac.pusan.pickle.proxmox.dto.NodeRrdSample;
import org.jspecify.annotations.Nullable;

/**
* Contract schema {@code NodeMetricPoint}. Nullable throughout for the same
* reason as the VM series: an RRD row omits the keys it has no data for, and a
* gap must stay a gap.
*
* <p>{@code time} is the exception for the same reason it is on the VM series:
* it is required by the contract and a point that cannot be placed on the axis
* cannot be charted, so a timeless row is dropped where the series is mapped.
*/
public record NodeMetricPointResponse(
Instant time,
Expand All @@ -27,23 +32,20 @@ public record NodeMetricPointResponse(
@Nullable Double netinBps,
@Nullable Double netoutBps) {

/** Callers drop timeless rows first (see the class note); everything else may be null. */
public static NodeMetricPointResponse from(NodeRrdSample sample) {
return new NodeMetricPointResponse(
sample.time() == null ? null : Instant.ofEpochSecond(sample.time()),
Instant.ofEpochSecond(sample.time()),
sample.cpu(),
sample.iowait(),
sample.loadavg(),
bytes(sample.memtotal()),
bytes(sample.memused()),
bytes(sample.swaptotal()),
bytes(sample.swapused()),
bytes(sample.roottotal()),
bytes(sample.rootused()),
RrdValues.bytes(sample.memtotal()),
RrdValues.bytes(sample.memused()),
RrdValues.bytes(sample.swaptotal()),
RrdValues.bytes(sample.swapused()),
RrdValues.bytes(sample.roottotal()),
RrdValues.bytes(sample.rootused()),
sample.netin(),
sample.netout());
}

private static Long bytes(Double value) {
return value == null ? null : Math.round(value);
}
}
5 changes: 4 additions & 1 deletion src/main/java/kr/ac/pusan/pickle/inventory/Node.java
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,10 @@ public class Node {
@Column(name = "ip_pool_id")
private Long ipPoolId;

/** Thin-pool size in GB, measured by the infra inventory script; null until measured (V76). */
/**
* Thin-pool size in GB, measured on the host by an operations tool; null
* until measured (V76).
*/
@Column(name = "disk_capacity_gb")
private Long diskCapacityGb;

Expand Down
18 changes: 18 additions & 0 deletions src/main/java/kr/ac/pusan/pickle/proxmox/RrdValues.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package kr.ac.pusan.pickle.proxmox;

/** How RRD encodes the values both metric series read, in one place. */
public final class RrdValues {

private RrdValues() {
}

/**
* RRD carries byte counters as doubles (consolidated averages), so every
* byte field of a metric point is rounded back to a whole count here. A
* gap stays a gap: an absent counter maps to null and never to zero, which
* a chart would draw as a measured floor.
*/
public static Long bytes(Double value) {
return value == null ? null : Math.round(value);
}
}
15 changes: 12 additions & 3 deletions src/main/java/kr/ac/pusan/pickle/vm/VmMetricsService.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

/**
* Usage time series of one VM (contract op {@code getVmMetrics}), read live
Expand All @@ -31,6 +30,13 @@
* The one case that is genuinely not an error is a VM with no guest behind it
* yet (or a deleted one): there is nothing to ask about, so it answers 200 with
* {@code available: false} and never touches Proxmox.
*
* <p>Deliberately not {@code @Transactional}: the hypervisor read can sit for
* the whole client read timeout, and holding a pooled connection across it lets
* a stalled PVE plus a polling chart drain the pool out from under unrelated
* endpoints. Both database reads below carry their own transaction (the access
* lookup and the repository call), and only basic columns of the entities they
* return are touched afterwards, so nothing lazy is left outside one.
*/
@Service
public class VmMetricsService {
Expand All @@ -53,7 +59,6 @@ public VmMetricsService(VmAccessService vmAccessService, NodeRepository nodeRepo
this.clock = clock;
}

@Transactional(readOnly = true)
public VmMetricsResponse metrics(AuthenticatedUser actor, long vmId, RrdTimeframe timeframe) {
Vm vm = vmAccessService.of(actor, vmId).requireVisible();
if (vm.getProxmoxVmid() == null || vm.getStatus() == VmStatus.DELETED) {
Expand All @@ -67,10 +72,14 @@ public VmMetricsResponse metrics(AuthenticatedUser actor, long vmId, RrdTimefram
points = proxmoxClient
.vmRrdData(node.getApiHost(), node.getName(), vm.getProxmoxVmid(), timeframe)
.stream()
// A row with no timestamp has no place on the axis.
.filter(sample -> sample.time() != null)
.map(VmMetricPointResponse::from)
.toList();
} catch (ProxmoxApiException e) {
} catch (ProxmoxApiException | IllegalStateException e) {
// The message carries the PVE reason and never the token (client layer).
// An unconfigured token refuses the same way from the caller's side:
// PVE cannot be asked, so the answer is the outage, not a 500.
log.warn("VM {} usage read failed on node {}: {}", vmId, node.getName(),
e.getMessage());
throw metricsUnavailable();
Expand Down
20 changes: 11 additions & 9 deletions src/main/java/kr/ac/pusan/pickle/vm/dto/VmMetricPointResponse.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,19 @@

import io.swagger.v3.oas.annotations.media.Schema;
import java.time.Instant;
import kr.ac.pusan.pickle.proxmox.RrdValues;
import kr.ac.pusan.pickle.proxmox.dto.VmRrdSample;
import org.jspecify.annotations.Nullable;

/**
* Contract schema {@code VmMetricPoint}. Every metric is nullable because an
* RRD row omits the keys for intervals the VM was not running in, and that
* absence is the honest gap a chart must draw rather than a zero.
*
* <p>{@code time} is the exception: it is the point's position on the axis and
* the contract's only required field here, so a row that arrives without one
* is dropped where the series is mapped rather than shipped as a null the
* console would have to guess about.
*/
public record VmMetricPointResponse(
Instant time,
Expand All @@ -25,21 +31,17 @@ public record VmMetricPointResponse(
@Nullable Double diskReadBps,
@Nullable Double diskWriteBps) {

/** Callers drop timeless rows first (see the class note); everything else may be null. */
public static VmMetricPointResponse from(VmRrdSample sample) {
return new VmMetricPointResponse(
sample.time() == null ? null : Instant.ofEpochSecond(sample.time()),
Instant.ofEpochSecond(sample.time()),
sample.cpu(),
bytes(sample.mem()),
bytes(sample.memhost()),
bytes(sample.maxmem()),
RrdValues.bytes(sample.mem()),
RrdValues.bytes(sample.memhost()),
RrdValues.bytes(sample.maxmem()),
sample.netin(),
sample.netout(),
sample.diskread(),
sample.diskwrite());
}

/** RRD carries byte counters as doubles (consolidated averages); a gap stays a gap. */
private static Long bytes(Double value) {
return value == null ? null : Math.round(value);
}
}
Loading