diff --git a/src/core/cm/launcher/balancer.cpp b/src/core/cm/launcher/balancer.cpp index d0ac1db96..cb141cacc 100644 --- a/src/core/cm/launcher/balancer.cpp +++ b/src/core/cm/launcher/balancer.cpp @@ -27,6 +27,17 @@ void Balancer::Init(InstanceManager& instanceManager, imagemanager::ItemInfoProv mNetworkManager = &networkManager; } +bool Balancer::OverrideEnvVars(const OverrideEnvVarsRequest& envVars) +{ + if (mEnvVarsOverrides == envVars) { + return false; + } + + mEnvVarsOverrides = envVars; + + return true; +} + RetWithError Balancer::SetSubjects(const Array>& subjects) { if (auto err = mSubjects.Assign(subjects); !err.IsNone()) { @@ -90,7 +101,7 @@ Error Balancer::SetupInstanceInfo(const oci::ServiceConfig& servConf, const Node const RunInstanceRequest& request, const oci::IndexContentDescriptor& imageDescriptor, const String& runtimeID, const Instance& instance, aos::InstanceInfo& info) { - // create instance info, InstanceNetworkParameters are added after network updates + // Create instance info, InstanceNetworkParameters are added after network updates static_cast(info) = instance.GetInfo().mInstanceIdent; info.mManifestDigest = imageDescriptor.mDigest; info.mRuntimeID = runtimeID; @@ -104,6 +115,35 @@ Error Balancer::SetupInstanceInfo(const oci::ServiceConfig& servConf, const Node return AOS_ERROR_WRAP(err); } + if (auto err = ApplyEnvVarOverrides(info); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; +} + +Error Balancer::ApplyEnvVarOverrides(aos::InstanceInfo& info) +{ + for (const auto& item : mEnvVarsOverrides.mItems) { + if (!item.Match(info)) { + continue; + } + + for (const auto& envVarInfo : item.mVariables) { + auto found = info.mEnvVars.FindIf( + [&envVarInfo](const EnvVar& existing) { return existing.mName == envVarInfo.mName; }); + + if (found != info.mEnvVars.end()) { + found->mValue = envVarInfo.mValue; + } else { + auto envVar = EnvVar {envVarInfo.mName, envVarInfo.mValue}; + if (auto err = info.mEnvVars.PushBack(envVar); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + } + } + } + return ErrorEnum::eNone; } @@ -214,6 +254,8 @@ Error Balancer::ScheduleInstance( return err; } + instance.SetEnvVars(imageConfig->mConfig.mEnv); + // Schedule instance. auto reqCPU = GetRequestedCPU(instance, *node, *serviceConfig); auto reqRAM = GetRequestedRAM(instance, *node, *serviceConfig); @@ -705,6 +747,8 @@ Error Balancer::PerformPolicyBalancing(const Array& requests auto reqRAM = GetRequestedRAM(**instance, *node, *serviceConfig); auto reqResources = serviceConfig->mResources; + (*instance)->SetEnvVars(imageConfig->mConfig.mEnv); + if (auto err = node->ScheduleInstance( *instanceInfo, request.mSubjectInfo.mSubjectID, *networkServiceData, reqCPU, reqRAM, reqResources); !err.IsNone()) { diff --git a/src/core/cm/launcher/balancer.hpp b/src/core/cm/launcher/balancer.hpp index b6a8aeaf2..1d86a303c 100644 --- a/src/core/cm/launcher/balancer.hpp +++ b/src/core/cm/launcher/balancer.hpp @@ -60,6 +60,14 @@ class Balancer { */ RetWithError SetSubjects(const Array>& subjects); + /** + * Overrides environment variables. + * + * @param envVars environment variables. + * @return bool true if env vars changed, false otherwise. + */ + bool OverrideEnvVars(const OverrideEnvVarsRequest& envVars); + private: static constexpr auto cAllocatorSize = sizeof(StaticArray) + sizeof(StaticArray) + sizeof(oci::ServiceConfig) + sizeof(oci::ImageConfig) @@ -100,10 +108,9 @@ class Balancer { Error RemoveNetworkForDeletedInstances(); Error SetNetworkParams(bool onlyWithExposedPorts); Error SetupNetworkForNewInstances(); - + Error ApplyEnvVarOverrides(aos::InstanceInfo& info); Error PerformPolicyBalancing(const Array& instances); - - bool IsSubjectEnabled(const Instance& instance); + bool IsSubjectEnabled(const Instance& instance); ImageInfoProvider mImageInfoProvider; InstanceManager* mInstanceManager {}; @@ -112,6 +119,7 @@ class Balancer { networkmanager::NetworkManagerItf* mNetworkManager {}; InstanceRunnerItf* mRunner {}; SubjectArray mSubjects; + OverrideEnvVarsRequest mEnvVarsOverrides; StaticAllocator mAllocator; }; diff --git a/src/core/cm/launcher/instance.cpp b/src/core/cm/launcher/instance.cpp index 4c4d1a630..d4279f4d8 100644 --- a/src/core/cm/launcher/instance.cpp +++ b/src/core/cm/launcher/instance.cpp @@ -75,8 +75,9 @@ Error Instance::Schedule(const aos::InstanceInfo& info, const String& nodeID) mInfo.mNodeID = nodeID; mInfo.mUID = info.mUID; mInfo.mGID = info.mGID; - mInfo.mTimestamp = Time::Now(); - mInfo.mState = InstanceStateEnum::eActive; + // Keep original EnvVars, not overridden. + mInfo.mTimestamp = Time::Now(); + mInfo.mState = InstanceStateEnum::eActive; static_cast(mStatus) = static_cast(info); mStatus.mNodeID = nodeID; diff --git a/src/core/cm/launcher/instance.hpp b/src/core/cm/launcher/instance.hpp index 0e89903b1..cb0a43b23 100644 --- a/src/core/cm/launcher/instance.hpp +++ b/src/core/cm/launcher/instance.hpp @@ -145,6 +145,13 @@ class Instance { */ void UpdateMonitoringData(const MonitoringData& monitoringData); + /** + * Sets instance environment variables. + * + * @param envVars environment variables. + */ + void SetEnvVars(const EnvVarArray& envVars) { mInfo.mEnvVars = envVars; } + /** * Removes instance. * diff --git a/src/core/cm/launcher/itf/storage.hpp b/src/core/cm/launcher/itf/storage.hpp index 8f17261d6..54ea61f03 100644 --- a/src/core/cm/launcher/itf/storage.hpp +++ b/src/core/cm/launcher/itf/storage.hpp @@ -9,6 +9,7 @@ #include #include +#include namespace aos::cm::launcher { @@ -91,6 +92,11 @@ struct InstanceInfo { */ bool mIsUnitSubject {}; + /** + * Environment variables assigned to the instance. + */ + EnvVarArray mEnvVars; + /** * Compares instance info. * @@ -101,7 +107,8 @@ struct InstanceInfo { { return mInstanceIdent == rhs.mInstanceIdent && mManifestDigest == rhs.mManifestDigest && mNodeID == rhs.mNodeID && mPrevNodeID == rhs.mPrevNodeID && mRuntimeID == rhs.mRuntimeID && mUID == rhs.mUID && mGID == rhs.mGID - && mTimestamp == rhs.mTimestamp && mState == rhs.mState && mIsUnitSubject == rhs.mIsUnitSubject; + && mTimestamp == rhs.mTimestamp && mState == rhs.mState && mIsUnitSubject == rhs.mIsUnitSubject + && mEnvVars == rhs.mEnvVars; } /** @@ -163,6 +170,21 @@ class StorageItf { * @return Error. */ virtual Error GetActiveInstances(Array& instances) const = 0; + + /** + * Saves override environment variables request. + * + * @param envVars override environment variables request. + * @return Error. + */ + virtual Error SaveOverrideEnvVars(const OverrideEnvVarsRequest& envVars) = 0; + + /** + * Gets override environment variables request. + * + * @return override environment variables request. + */ + virtual Error GetOverrideEnvVars(OverrideEnvVarsRequest& envVars) = 0; }; /** @}*/ diff --git a/src/core/cm/launcher/launcher.cpp b/src/core/cm/launcher/launcher.cpp index 275a3a592..8cdd4e150 100644 --- a/src/core/cm/launcher/launcher.cpp +++ b/src/core/cm/launcher/launcher.cpp @@ -116,6 +116,19 @@ Error Launcher::Start() // Notify status listeners. NotifyInstanceStatusListeners(mInstanceStatuses); + // Load override environment variables. + if (auto err = mStorage->GetOverrideEnvVars(mOverrideEnvVars); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + mBalancer.OverrideEnvVars(mOverrideEnvVars); + + // Start timer to periodically check override environment variables TTLs. + auto onEnvVarsTTLTimerTick = [this](void*) { OverrideEnvVars(mOverrideEnvVars); }; + if (auto err = mEnvVarsTTLTimer.Start(Time::cMinutes, onEnvVarsTTLTimerTick, false); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + // Start monitoring thread. mDisableProcessUpdates = false; mUpdatedNodes.Clear(); @@ -147,6 +160,11 @@ Error Launcher::Stop() return AOS_ERROR_WRAP(err); } + // Stop TTL timer. + if (auto err = mEnvVarsTTLTimer.Stop(); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + // Stop managers. if (auto err = mInstanceManager.Stop(); !err.IsNone()) { return err; @@ -248,11 +266,35 @@ Error Launcher::UnsubscribeListener(instancestatusprovider::ListenerItf& listene Error Launcher::OverrideEnvVars(const OverrideEnvVarsRequest& envVars) { - (void)envVars; - LOG_DBG() << "Override env vars"; - return ErrorEnum::eNone; + UniqueLock lock {mMutex}; + + mOverrideEnvVars = envVars; + + // Remove variables with expired TTLs. + auto now = Time::Now(); + + for (auto& item : mOverrideEnvVars.mItems) { + item.mVariables.RemoveIf([&now](const EnvVarInfo& envVarInfo) { + return envVarInfo.mTTL.HasValue() && envVarInfo.mTTL.GetValue() < now; + }); + } + + mOverrideEnvVars.mItems.RemoveIf([](const EnvVarsInstanceInfo& item) { return item.mVariables.IsEmpty(); }); + + // Override environment variables. + if (!mBalancer.OverrideEnvVars(mOverrideEnvVars)) { + return ErrorEnum::eNone; + } + + // Save override environment variables. + if (auto err = mStorage->SaveOverrideEnvVars(mOverrideEnvVars); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + // Rebalance instances. + return Rebalance(lock); } /*********************************************************************************************************************** diff --git a/src/core/cm/launcher/launcher.hpp b/src/core/cm/launcher/launcher.hpp index 71384d333..42982c02e 100644 --- a/src/core/cm/launcher/launcher.hpp +++ b/src/core/cm/launcher/launcher.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include "itf/envvarhandler.hpp" #include "itf/instancestatusreceiver.hpp" @@ -200,6 +201,9 @@ class Launcher : public LauncherItf, Optional mNewSubjects; Mutex mNewSubjectsMutex; + OverrideEnvVarsRequest mOverrideEnvVars; + Timer mEnvVarsTTLTimer; + Mutex mMutex; StaticAllocator mAllocator; }; diff --git a/src/core/cm/launcher/tests/launcher.cpp b/src/core/cm/launcher/tests/launcher.cpp index bf295c182..4fd824632 100644 --- a/src/core/cm/launcher/tests/launcher.cpp +++ b/src/core/cm/launcher/tests/launcher.cpp @@ -275,7 +275,8 @@ StaticString BuildManifestDigest(const std::string& itemID, con } aos::InstanceInfo CreateAosInstanceInfo(const InstanceIdent& id, const std::string& imageID, - const std::string& runtimeID, uint32_t uid, gid_t gid, const String& ip, uint64_t priority) + const std::string& runtimeID, uint32_t uid, gid_t gid, const String& ip, uint64_t priority, + const EnvVarArray& envVars = {}, SubjectTypeEnum subjectType = SubjectTypeEnum::eGroup) { aos::InstanceInfo result; @@ -288,6 +289,7 @@ aos::InstanceInfo CreateAosInstanceInfo(const InstanceIdent& id, const std::stri result.mManifestDigest = imagemanager::ImageStoreStub::BuildManifestDigest(itemID, image); result.mRuntimeID = runtimeID.c_str(); + result.mSubjectType = subjectType; result.mUID = uid; result.mGID = gid; result.mPriority = priority; @@ -296,6 +298,7 @@ aos::InstanceInfo CreateAosInstanceInfo(const InstanceIdent& id, const std::stri result.mNetworkParameters.EmplaceValue(); result.mNetworkParameters->mIP = (std::string("172.17.0.") + ip.CStr()).c_str(); result.mNetworkParameters->mSubnet = "172.17.0.0/16"; + result.mEnvVars = envVars; return result; } @@ -519,6 +522,22 @@ UnitNodeInfo CreateNodeInfo(const std::string& nodeID, size_t maxDMIPS, size_t t return nodeInfo; } +EnvVar CreateEnvVar(const std::string& name, const std::string& value) +{ + EnvVar var; + + var.mName = name.c_str(); + var.mValue = value.c_str(); + + return var; +} + +template +Array ConvertToArray(const std::initializer_list& list) +{ + return Array(list.begin(), list.size()); +} + /*********************************************************************************************************************** * Tests **********************************************************************************************************************/ @@ -1761,4 +1780,108 @@ TEST_F(CMLauncherTest, SubjectChanged) ASSERT_TRUE(mLauncher.Stop().IsNone()); } +TEST_F(CMLauncherTest, OverrideEnvVars) +{ + using namespace std::chrono_literals; + + Config cfg; + cfg.mNodesConnectionTimeout = 1 * Time::cMinutes; + + // Initialize stubs + mStorageState.Init(); + mStorageState.SetTotalStateSize(1024); + mStorageState.SetTotalStorageSize(1024); + + mNodeInfoProvider.Init(); + mImageStore.Init(); + mNetworkManager.Init(); + mInstanceStatusProvider.Init(); + mMonitoringProvider.Init(); + mResourceManager.Init(); + mStorage.Init(); + + auto nodeInfoLocalSM = CreateNodeInfo(cNodeIDLocalSM, 1000, 1024, {CreateRuntime(cRunnerRunc)}, {}); + mNodeInfoProvider.AddNodeInfo(cNodeIDLocalSM, nodeInfoLocalSM); + + auto nodeConfig = std::make_unique(); + CreateNodeConfig(*nodeConfig, cNodeIDLocalSM); + mResourceManager.SetNodeConfig(cNodeIDLocalSM, cNodeTypeVM, *nodeConfig); + + // Service config + auto serviceConfig = std::make_unique(); + CreateServiceConfig(*serviceConfig, {cRunnerRunc}); + AddService(cService1, cImageID1, *serviceConfig, CreateImageConfig()); + + mInstanceRunner.Init(mLauncher); + + // Init launcher + ASSERT_TRUE(mLauncher + .Init(cfg, mStorage, mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mImageStore, + mResourceManager, mStorageState, mNetworkManager, mMonitoringProvider, mAlertsProvider, + mIdentProvider, ValidateGID, ValidateUID) + .IsNone()); + + InstanceStatusListenerStub instanceStatusListener; + mLauncher.SubscribeListener(instanceStatusListener); + + ASSERT_TRUE(mLauncher.Start().IsNone()); + + // 1) Run a single instance + auto runRequest = std::make_unique>(); + runRequest->PushBack(CreateRunRequest(cService1, cSubject1, 50, 1)); + + auto runStatuses = std::make_unique>(); + ASSERT_TRUE(mLauncher.RunInstances(*runRequest, *runStatuses).IsNone()); + + // Wait until we have at least some statuses recorded + ASSERT_TRUE(instanceStatusListener.WaitForNotifyCount(3, 2000ms)); + + // 2) Change override env vars with different TTLs + EnvVarsInstanceInfo envVarsInfo; + EnvVarInfo envVar1, envVar2, envVar3; + + envVarsInfo.mItemID.EmplaceValue(cService1); + + // Env var 1: with expired TTL + static_cast(envVar1) = CreateEnvVar("OVERRIDE_VAR1", "override_value1"); + envVar1.mTTL.SetValue(Time::Now().Add(-1 * Time::cHours)); + envVarsInfo.mVariables.PushBack(envVar1); + + // Env var 2: with non-expired TTL + static_cast(envVar2) = CreateEnvVar("OVERRIDE_VAR2", "override_value2"); + envVar2.mTTL.SetValue(Time::Now().Add(1 * Time::cHours)); + envVarsInfo.mVariables.PushBack(envVar2); + + // Env var 3: without TTL + static_cast(envVar3) = CreateEnvVar("OVERRIDE_VAR3", "override_value3"); + envVarsInfo.mVariables.PushBack(envVar3); + + OverrideEnvVarsRequest overrideRequest; + + overrideRequest.mItems.PushBack(envVarsInfo); + + ASSERT_TRUE(mLauncher.OverrideEnvVars(overrideRequest).IsNone()); + + // 3) Wait for rebalancing finished + // Expect one more notification caused by rebalance after override env vars update + ASSERT_TRUE(instanceStatusListener.WaitForNotifyCount(4, 2000ms)); + + // 4) Check result - verify only non-expired and no-TTL env vars are sent to instance runner + // (envVar1 with expired TTL should be filtered out) + auto expectedEnvVarsList + = {CreateEnvVar("OVERRIDE_VAR2", "override_value2"), CreateEnvVar("OVERRIDE_VAR3", "override_value3")}; + + InstanceRunnerStub::NodeRunRequest localSMRequest = {{}, + {CreateAosInstanceInfo(CreateInstanceIdent(cService1, cSubject1, 0), cImageID1, cRunnerRunc, 5000, 5000, "3", + 50, ConvertToArray(expectedEnvVarsList), SubjectTypeEnum::eGroup)}}; + + std::map expectedRunRequests; + expectedRunRequests[cNodeIDLocalSM] = localSMRequest; + + // Compare actual with expected + EXPECT_EQ(mInstanceRunner.GetNodeInstances(), expectedRunRequests); + + ASSERT_TRUE(mLauncher.Stop().IsNone()); +} + } // namespace aos::cm::launcher diff --git a/src/core/cm/launcher/tests/stubs/storagestub.hpp b/src/core/cm/launcher/tests/stubs/storagestub.hpp index a7115e517..dc4786a8d 100644 --- a/src/core/cm/launcher/tests/stubs/storagestub.hpp +++ b/src/core/cm/launcher/tests/stubs/storagestub.hpp @@ -7,6 +7,7 @@ #define AOS_CM_LAUNCHER_STUBS_STORAGESTUB_HPP_ #include +#include #include @@ -71,6 +72,20 @@ class StorageStub : public StorageItf { return Error(); } + Error SaveOverrideEnvVars(const OverrideEnvVarsRequest& envVars) override + { + *mOverrideEnvVarsRequest = envVars; + + return Error(); + } + + Error GetOverrideEnvVars(OverrideEnvVarsRequest& envVars) override + { + envVars = *mOverrideEnvVarsRequest; + + return Error(); + } + Error GetActiveInstances(Array& instances) const override { instances.Clear(); @@ -92,7 +107,8 @@ class StorageStub : public StorageItf { void ClearInstances() { mInstanceInfo.clear(); } private: - std::map mInstanceInfo; + std::map mInstanceInfo; + std::unique_ptr mOverrideEnvVarsRequest = std::make_unique(); }; } // namespace aos::cm::launcher diff --git a/src/core/common/ocispec/itf/imagespec.hpp b/src/core/common/ocispec/itf/imagespec.hpp index 289ef053c..05c719291 100644 --- a/src/core/common/ocispec/itf/imagespec.hpp +++ b/src/core/common/ocispec/itf/imagespec.hpp @@ -254,7 +254,7 @@ struct Rootfs { */ struct Config { StaticArray, cMaxNumExposedPorts> mExposedPorts; - StaticArray, cMaxNumEnvVariables> mEnv; + EnvVarArray mEnv; StaticArray, cMaxParamCount> mEntryPoint; StaticArray, cMaxParamCount> mCmd; StaticString mWorkingDir; diff --git a/src/core/common/tools/algorithm.hpp b/src/core/common/tools/algorithm.hpp index 970b83eb9..100693b43 100644 --- a/src/core/common/tools/algorithm.hpp +++ b/src/core/common/tools/algorithm.hpp @@ -29,6 +29,11 @@ struct LessThanComparator { template class AlgorithmItf { public: + /** + * Destructor. + */ + virtual ~AlgorithmItf() = default; + /** * Returns current container size. * diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index 6b33d0085..324e741b4 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -982,14 +982,14 @@ Error NetworkManager::RemoveNetworks(const Array& networ err = errClearNetwork; } - it = mNetworkProviders.Erase(it); - LOG_DBG() << "Remove network from storage: networkID=" << it->mFirst; if (auto errRemoveStorage = mStorage->RemoveNetworkInfo(it->mFirst); !errRemoveStorage.IsNone() && err.IsNone()) { err = errRemoveStorage; } + + it = mNetworkProviders.Erase(it); } else { ++it; }