Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions src/api/status.rs
Original file line number Diff line number Diff line change
Expand Up @@ -214,14 +214,12 @@ pub async fn get_status() -> Json<Vec<ServiceStatus>> {
} else {
"stopped".to_string()
}
} else if container.ports.is_empty() || container.ip_address.is_empty() {
"stopped".to_string()
} else {
if container.ports.is_empty() || container.ip_address.is_empty()
{
"stopped".to_string()
} else {
"running".to_string()
}
"running".to_string()
}

},
cpu_percentage: container_stats.as_ref().map(|s| s.cpu_percentage),
cpu_percentage_relative: container_stats
Expand Down
118 changes: 115 additions & 3 deletions src/config/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ use std::sync::Arc;
use std::{
collections::HashMap,
path::PathBuf,
path::Path,
sync::OnceLock,
time::{Duration, SystemTime},
};
Expand Down Expand Up @@ -289,7 +290,7 @@ pub async fn watch_directory(config_dir: PathBuf) -> notify::Result<()> {
Ok(())
}

async fn process_event(event: DebouncedEvent, config_dir: &PathBuf) {
async fn process_event(event: DebouncedEvent, config_dir: &Path) {
let config_store = CONFIG_STORE.get().unwrap();
let scaling_tasks = SCALING_TASKS.get().unwrap();

Expand Down Expand Up @@ -646,7 +647,7 @@ pub async fn handle_orphans(config: &ServiceConfig) -> Result<()> {
}

if let Err(e) = runtime
.remove_pod_network(&network_name, &service_name)
.remove_pod_network(&network_name, service_name)
.await
{
slog::error!(log, "Failed to remove network";
Expand Down Expand Up @@ -799,7 +800,7 @@ pub async fn handle_orphans(config: &ServiceConfig) -> Result<()> {

// Then try to remove the network
if let Err(e) = runtime
.remove_pod_network(&network_name, &service_name)
.remove_pod_network(&network_name, service_name)
.await
{
slog::error!(slog_scope::logger(), "Failed to remove orphaned network";
Expand Down Expand Up @@ -1061,3 +1062,114 @@ pub async fn handle_config_update(service_name: &str, config: ServiceConfig) ->

Ok(())
}

#[cfg(test)]
mod tests {
use super::*;
use crate::container::scaling::manager::UnifiedScalingManager;
use crate::container::scaling::manager::ScalingDecision;
use std::collections::HashMap;
use std::time::{Duration};
use uuid::Uuid;
use serde_json::Value;

fn mock_service_config() -> ServiceConfig {
ServiceConfig {
name: "test_service".to_string(),
network: Some("test_network".to_string()),
spec: ServiceSpec { containers: vec![] },
memory_limit: Some(Value::Number(1000.into())),
pull_policy: None,
cpu_limit: Some(Value::Number(2.into())),
resource_thresholds: Some(ResourceThresholds {
cpu_percentage: Some(70),
cpu_percentage_relative: Some(80),
memory_percentage: Some(75),
metrics_strategy: PodMetricsStrategy::Maximum,
}),
instance_count: InstanceCount { min: 1, max: 10 },
adopt_orphans: false,
interval_seconds: Some(30),
image_check_interval: Some(Duration::from_secs(300)),
rolling_update_config: None,
volumes: None,
codel: None,
scaling_policy: Some(ScalingPolicy {
cooldown_duration: Some(Duration::from_secs(60)),
scale_down_threshold_percentage: Some(50.0),
}),
}
}

#[test]
fn test_scaling_policy_defaults() {
let policy = ScalingPolicy::default();
assert_eq!(policy.get_cooldown_duration(), Duration::from_secs(60));
assert_eq!(policy.get_scale_down_threshold(), 50.0);
}

#[tokio::test]
async fn test_unified_scaling_manager_no_change_on_cooldown() {
let config = mock_service_config();
let mut manager = UnifiedScalingManager::new(
"test_service".to_string(),
config,
None,
None,
);

let result = manager.evaluate(3, &HashMap::new()).await;
assert!(matches!(result, ScalingDecision::NoChange));
}

#[tokio::test]
async fn test_unified_scaling_manager_scale_up() {
let config = mock_service_config();
let mut manager = UnifiedScalingManager::new(
"test_service".to_string(),
config,
None,
None,
);

let mut pod_stats = HashMap::new();
pod_stats.insert(Uuid::new_v4(), PodStats {
cpu_percentage: 85.0,
cpu_percentage_relative: 90.0,
memory_usage: 900,
memory_limit: 1000,
});

let result = manager.evaluate(3, &pod_stats).await;
assert!(matches!(result, ScalingDecision::NoChange));
}

#[tokio::test]
async fn test_unified_scaling_manager_scale_down() {
let config = mock_service_config();
let mut manager = UnifiedScalingManager::new(
"test_service".to_string(),
config,
None,
None,
);

let mut pod_stats = HashMap::new();
pod_stats.insert(Uuid::new_v4(), PodStats {
cpu_percentage: 10.0,
cpu_percentage_relative: 15.0,
memory_usage: 200,
memory_limit: 1000,
});

let result = manager.evaluate(3, &pod_stats).await;
assert!(matches!(result, ScalingDecision::NoChange));
}

#[test]
fn test_service_config_instance_count() {
let config = mock_service_config();
assert_eq!(config.instance_count.min, 1);
assert_eq!(config.instance_count.max, 10);
}
}
4 changes: 2 additions & 2 deletions src/config/utils.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// src/config/utils.rs
use std::path::PathBuf;
use std::path::Path;

use anyhow::{anyhow, Result};
use uuid::Uuid;
Expand Down Expand Up @@ -65,7 +65,7 @@ pub async fn get_config_by_service(service_name: &str) -> Option<ServiceConfig>
}
}

pub fn get_relative_config_path(full_path: &PathBuf, config_dir: &PathBuf) -> Option<String> {
pub fn get_relative_config_path(full_path: &Path, config_dir: &Path) -> Option<String> {
let config_dir_str = config_dir.to_str()?;
let full_path_str = full_path.to_str()?;

Expand Down
2 changes: 1 addition & 1 deletion src/container/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -412,7 +412,7 @@ fn calculate_cpu_percentages(
// Since absolute_cpu is across all cores, we need to compare with allocated_cpu * 100
let relative = (absolute_cpu / online_cpus) / allocated_cpu;
// Convert to percentage and clamp between 0-100
(relative * 100.0).max(0.0).min(100.0)
(relative * 100.0).clamp(0.0,100.0)
} else {
0.0 // Avoid division by zero
}
Expand Down
9 changes: 4 additions & 5 deletions src/container/runtimes/docker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,7 @@ impl ContainerRuntime for DockerRuntime {

// Setup volume mounts first and keep temp_dir alive
let (temp_dir, mounts) = self
.setup_volume_mounts(container, &container_name, &service_config)
.setup_volume_mounts(container, &container_name, service_config)
.await?;
if let Some(dir) = temp_dir {
temp_dirs.push(dir);
Expand Down Expand Up @@ -624,7 +624,7 @@ impl ContainerRuntime for DockerRuntime {
match service_config.pull_policy {
Some(PullPolicyValue::Always) => {
match self
.pull_image(service_name, containers, &service_config)
.pull_image(service_name, containers, service_config)
.await
{
Ok(_) => {}
Expand Down Expand Up @@ -763,10 +763,9 @@ impl ContainerRuntime for DockerRuntime {
}

let service_name = name
.splitn(2, "__")
.split("__")
.next()
.expect("Split always returns at least one element");

let service_cfg = get_config_by_service(service_name).await.unwrap();

let nano_cpus = service_cfg
Expand Down Expand Up @@ -820,7 +819,7 @@ impl ContainerRuntime for DockerRuntime {
port: c
.ports
.unwrap_or_default()
.get(0)
.first()
.and_then(|p| p.public_port)
.unwrap_or(0),
})
Expand Down
17 changes: 8 additions & 9 deletions src/container/scaling/codel.rs
Original file line number Diff line number Diff line change
Expand Up @@ -224,16 +224,15 @@ impl CoDelMetrics {
status_code: None,
});
}
} else {
if self.first_above_time.is_some() {
slog::info!(slog_scope::logger(), "Latency back below target";
"service" => &self.service_name,
"min_sojourn_ms" => min_sojourn.as_millis(),
"avg_sojourn_ms" => avg_sojourn.as_millis()
);
self.first_above_time = None;
}
} else if self.first_above_time.is_some() {
slog::info!(slog_scope::logger(), "Latency back below target";
"service" => &self.service_name,
"min_sojourn_ms" => min_sojourn.as_millis(),
"avg_sojourn_ms" => avg_sojourn.as_millis()
);
self.first_above_time = None;
}


None
}
Expand Down
29 changes: 15 additions & 14 deletions src/container/scaling/manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use uuid::Uuid;
use crate::config::{PodStats, ResourceThresholds, ServiceConfig};
use crate::container::scaling::codel::CoDelMetrics;

#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct ScalingPolicy {
/// Duration to wait between scaling actions
#[serde(
Expand Down Expand Up @@ -44,15 +44,6 @@ impl ScalingPolicy {
}
}

impl Default for ScalingPolicy {
fn default() -> Self {
Self {
cooldown_duration: None,
scale_down_threshold_percentage: None,
}
}
}

#[derive(Debug, Clone)]
pub enum ScalingDecision {
ScaleUp(u32),
Expand Down Expand Up @@ -195,7 +186,7 @@ impl UnifiedScalingManager {

async fn evaluate_resources(
&self,
current_instances: usize,
_current_instances: usize,
pod_stats: &HashMap<Uuid, PodStats>,
) -> Option<ScalingDecision> {
let thresholds = self.resource_thresholds.as_ref()?;
Expand All @@ -205,7 +196,7 @@ impl UnifiedScalingManager {

for stats in pod_stats.values() {
let memory_percentage = if stats.memory_limit > 0 {
(stats.memory_usage as f64 / stats.memory_limit as f64 * 100.0)
stats.memory_usage as f64 / stats.memory_limit as f64 * 100.0
} else {
0.0
};
Expand Down Expand Up @@ -258,8 +249,18 @@ impl UnifiedScalingManager {
pub fn get_state(&self) -> String {
match &self.state {
ScalingState::Normal => "normal".to_string(),
ScalingState::CoDelScalingUp { .. } => "codel_scaling_up".to_string(),
ScalingState::ResourceScalingDown { .. } => "resource_scaling_down".to_string(),
ScalingState::CoDelScalingUp { since, last_scale } => {
format!(
"codel_scaling_up_{}",
last_scale.duration_since(*since).as_secs()
)
}
ScalingState::ResourceScalingDown { since } => {
format!(
"resource_scaling_down_{}",
since.duration_since(Instant::now()).as_secs()
)
}
ScalingState::Cooldown { until } => {
format!(
"cooldown_{}",
Expand Down
4 changes: 2 additions & 2 deletions src/proxy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ pub async fn run_proxy_for_service(service_name: String, config: ServiceConfig)
// Get read lock to access instance data
let store = instance_store.read().await;
if let Some(instances) = store.get(&service_name) {
for (_, metadata) in instances {
for metadata in instances.values() {
for container in &metadata.containers {
for port_info in &container.ports {
if let Some(container_node_port) = port_info.node_port {
Expand Down Expand Up @@ -248,7 +248,7 @@ pub async fn run_proxy_for_service(service_name: String, config: ServiceConfig)
{
let store = instance_store.read().await;
if let Some(instances) = store.get(&service_name) {
for (_, metadata) in instances {
for metadata in instances.values() {
for container in &metadata.containers {
for port_info in &container.ports {
if let Some(container_node_port) = port_info.node_port {
Expand Down