Skip to content
This repository was archived by the owner on Jul 8, 2025. It is now read-only.

Commit 2ea7e9b

Browse files
slissonslisson
committed
feat: prepare draft branch before starting a workspace instance
1 parent be14420 commit 2ea7e9b

9 files changed

Lines changed: 258 additions & 39 deletions

File tree

‎gradle/libs.versions.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ zt-zip = { group = "org.zeroturnaround", name = "zt-zip", version = "1.17" }
5656
modelix-syncPlugin3 = { group = "org.modelix.mps", name = "mps-sync-plugin3", version.ref = "modelix-core" }
5757
modelix-mpsPlugins-generator = { group = "org.modelix.mps", name = "generator-execution-plugin", version.ref = "modelix-mps-plugins" }
5858
modelix-mpsPlugins-diff = { group = "org.modelix.mps", name = "diff-plugin", version.ref = "modelix-mps-plugins" }
59-
modelix-api-server-stubs = { group = "org.modelix", name = "api-server-stubs-ktor", version = "1.1.0" }
59+
modelix-api-server-stubs = { group = "org.modelix", name = "api-server-stubs-ktor", version = "1.2.0" }
6060

6161
[bundles]
6262
ktor-client = [

‎workspace-client-plugin/src/main/kotlin/org/modelix/workspace/client/WorkspaceClientStartupActivity.kt‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,9 @@ package org.modelix.workspace.client
33
import com.intellij.openapi.project.DumbService
44
import com.intellij.openapi.project.Project
55
import com.intellij.openapi.startup.StartupActivity
6-
import io.github.oshai.kotlinlogging.KotlinLogging
76
import org.modelix.model.lazy.RepositoryId
87
import org.modelix.mps.sync3.IModelSyncService
98

10-
private val LOG = KotlinLogging.logger { }
11-
129
class WorkspaceClientStartupActivity : StartupActivity {
1310
override fun runActivity(project: Project) {
1411
println("### Workspace client loaded for project: ${project.name}")
@@ -18,16 +15,21 @@ class WorkspaceClientStartupActivity : StartupActivity {
1815

1916
val syncEnabled: Boolean = getEnvOrLog("WORKSPACE_MODEL_SYNC_ENABLED") == "true"
2017
if (syncEnabled) {
18+
println("model sync is enabled")
2119
val modelUri: String? = getEnvOrLog("MODEL_URI")
22-
val repoId: String? = getEnvOrLog("REPOSITORY_ID")
20+
val repoId: RepositoryId? = getEnvOrLog("REPOSITORY_ID")?.let { RepositoryId(it) }
2321
val branchName: String? = getEnvOrLog("REPOSITORY_BRANCH")
2422
val jwt: String? = getEnvOrLog("INITIAL_JWT_TOKEN")
23+
println("model server: $modelUri")
24+
println("repository: $repoId")
25+
println("branch: $branchName")
26+
println("JWT: $jwt")
2527
if (modelUri != null && repoId != null) {
26-
val connection = IModelSyncService.getInstance(project).addServer(modelUri)
28+
val connection = IModelSyncService.getInstance(project).addServer(modelUri, repositoryId = repoId)
2729
if (jwt != null) {
2830
connection.setTokenProvider { jwt }
2931
}
30-
connection.bind(RepositoryId(repoId).getBranchReference(branchName))
32+
connection.bind(repoId.getBranchReference(branchName))
3133
}
3234
}
3335
}
@@ -36,7 +38,7 @@ class WorkspaceClientStartupActivity : StartupActivity {
3638
private fun getEnvOrLog(name: String): String? {
3739
val value = System.getenv(name)
3840
if (value == null) {
39-
LOG.warn { "Environment variable $name is not set." }
41+
println("Environment variable $name is not set.")
4042
}
4143
return value
4244
}

‎workspace-manager/src/main/kotlin/org/modelix/services/gitconnector/DraftPreparationTask.kt‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package org.modelix.services.gitconnector
2+
3+
import kotlinx.coroutines.CoroutineScope
4+
import org.modelix.model.client2.IModelClientV2
5+
import org.modelix.model.lazy.BranchReference
6+
import org.modelix.workspace.manager.TaskInstance
7+
8+
class DraftPreparationTask(
9+
scope: CoroutineScope,
10+
val key: Key,
11+
val gitManager: GitConnectorManager,
12+
val modelClient: IModelClientV2,
13+
) : TaskInstance<BranchReference>(scope) {
14+
override suspend fun process(): BranchReference {
15+
val draft = requireNotNull(gitManager.getDraft(key.draftId)) { "Draft not found: ${key.draftId}" }
16+
val gitRepoConfig = requireNotNull(gitManager.getRepository(draft.gitRepositoryId)) {
17+
"Git repository config not found: ${draft.gitRepositoryId}"
18+
}
19+
val modelixBranch = gitRepoConfig.getModelixRepositoryId().getBranchReference(draft.modelixBranchName)
20+
if (modelClient.listBranches(modelixBranch.repositoryId).contains(modelixBranch)) {
21+
return modelixBranch
22+
}
23+
24+
val importTask = gitManager.getOrCreateImportTask(draft.gitRepositoryId, draft.gitBranchName)
25+
val importedVersion = importTask.waitForOutput()
26+
modelClient.push(modelixBranch, importedVersion, importedVersion)
27+
return modelixBranch
28+
}
29+
30+
data class Key(
31+
val draftId: String,
32+
)
33+
}

‎workspace-manager/src/main/kotlin/org/modelix/services/gitconnector/GitConnectorManager.kt‎

Lines changed: 56 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
11
package org.modelix.services.gitconnector
22

33
import kotlinx.coroutines.CoroutineScope
4+
import org.modelix.model.client2.IModelClientV2
5+
import org.modelix.model.lazy.RepositoryId
46
import org.modelix.services.gitconnector.stubs.models.GitBranchStatusData
57
import org.modelix.services.gitconnector.stubs.models.GitRepositoryConfig
68
import org.modelix.services.workspaces.FileSystemPersistence
79
import org.modelix.services.workspaces.PersistedState
10+
import org.modelix.workspace.manager.KestraClient
11+
import org.modelix.workspace.manager.ReusableTasks
812
import org.modelix.workspace.manager.SharedMutableState
913
import java.io.File
1014

@@ -17,16 +21,62 @@ class GitConnectorManager(
1721
),
1822
defaultState = { GitConnectorData() },
1923
).state,
24+
val modelClient: IModelClientV2,
25+
val kestraClient: KestraClient,
2026
) {
27+
28+
private val importTasks = ReusableTasks<GitImportTask.Key, GitImportTask>()
29+
private val draftTasks = ReusableTasks<DraftPreparationTask.Key, DraftPreparationTask>()
30+
31+
fun getOrCreateImportTask(gitRepoId: String, gitBranchName: String): GitImportTask {
32+
val data = connectorData.getValue()
33+
val repo = requireNotNull(data.repositories[gitRepoId]) { "Repository not found: $gitRepoId" }
34+
val branch = requireNotNull(repo.status?.branches?.find { it.name == gitBranchName }) {
35+
"Branch not found: $gitBranchName"
36+
}
37+
val gitRevision = requireNotNull(branch.gitCommitHash) {
38+
"Git commit hash for branch unknown: $gitBranchName"
39+
}
40+
val key = GitImportTask.Key(
41+
repo = repo.copy(status = null),
42+
gitBranchName = gitBranchName,
43+
gitRevision = gitRevision,
44+
modelixBranchName = "git-import/$gitBranchName",
45+
)
46+
return importTasks.getOrCreateTask(key) {
47+
GitImportTask(
48+
key = key,
49+
scope = scope,
50+
kestraClient = kestraClient,
51+
modelClient = modelClient,
52+
)
53+
}
54+
}
55+
56+
fun getOrCreateDraftPreparationTask(draftId: String): DraftPreparationTask {
57+
val key = DraftPreparationTask.Key(
58+
draftId = draftId,
59+
)
60+
61+
return draftTasks.getOrCreateTask(key) {
62+
DraftPreparationTask(
63+
scope = scope,
64+
key = key,
65+
gitManager = this,
66+
modelClient = modelClient,
67+
)
68+
}
69+
}
70+
2171
suspend fun updateRemoteBranches(repository: GitRepositoryConfig): List<GitBranchStatusData> {
22-
val repositoryId = repository.id
72+
val gitRepositoryId = repository.id
2373
val fetchTask = GitFetchTask(scope, repository)
2474
val resultResult = fetchTask.waitForOutput()
2575
return connectorData.update {
26-
val oldRepositoryData = it.repositories[repositoryId] ?: return@update it
76+
val oldRepositoryData = it.repositories[gitRepositoryId] ?: return@update it
2777
val newRepositoryData = oldRepositoryData.merge(resultResult.remoteRefs)
28-
it.copy(repositories = it.repositories + (repositoryId to newRepositoryData))
29-
}.repositories[repositoryId]?.status?.branches ?: emptyList()
78+
it.copy(repositories = it.repositories + (gitRepositoryId to newRepositoryData))
79+
}.repositories[gitRepositoryId]?.status?.branches.orEmpty()
3080
}
3181

3282
fun getRepository(id: String): GitRepositoryConfig? {
@@ -35,3 +85,5 @@ class GitConnectorManager(
3585

3686
fun getDraft(id: String) = connectorData.getValue().drafts[id]
3787
}
88+
89+
fun GitRepositoryConfig.getModelixRepositoryId() = RepositoryId((modelixRepository ?: id))

‎workspace-manager/src/main/kotlin/org/modelix/services/gitconnector/GitImportTask.kt‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
package org.modelix.services.gitconnector
2+
3+
import kotlinx.coroutines.CoroutineScope
4+
import kotlinx.coroutines.delay
5+
import org.modelix.model.IVersion
6+
import org.modelix.model.client2.IModelClientV2
7+
import org.modelix.model.lazy.RepositoryId
8+
import org.modelix.services.gitconnector.stubs.models.GitRepositoryConfig
9+
import org.modelix.workspace.manager.KestraClient
10+
import org.modelix.workspace.manager.TaskInstance
11+
import kotlin.time.Duration.Companion.seconds
12+
13+
class GitImportTask(
14+
val key: Key,
15+
scope: CoroutineScope,
16+
val kestraClient: KestraClient,
17+
val modelClient: IModelClientV2,
18+
) : TaskInstance<IVersion>(scope) {
19+
20+
private val repoId = requireNotNull(key.repo.modelixRepository?.let { RepositoryId(it) }) { "Repository ID missing" }
21+
private val branchRef = repoId.getBranchReference(key.modelixBranchName)
22+
private val jobLabels = mapOf("taskId" to id.toString())
23+
private suspend fun modelixBranchExists() = modelClient.listBranches(repoId).contains(branchRef)
24+
private suspend fun jobIsRunning() = kestraClient.getRunningImportJobIds(jobLabels).isNotEmpty()
25+
26+
override suspend fun process(): IVersion {
27+
if (modelixBranchExists()) {
28+
return modelClient.lazyLoadVersion(branchRef)
29+
}
30+
31+
val remote = requireNotNull(key.repo.remotes?.firstOrNull()) { "No remotes specified" }
32+
val modelixBranch =
33+
RepositoryId((key.repo.modelixRepository ?: key.repo.id)).getBranchReference("git-import/${key.gitBranchName}")
34+
kestraClient.enqueueGitImport(
35+
gitRepoUrl = remote.url,
36+
gitUser = remote.credentials?.username,
37+
gitPassword = remote.credentials?.password,
38+
gitRevision = key.gitRevision,
39+
modelixBranch = modelixBranch,
40+
labels = jobLabels,
41+
)
42+
43+
while (true) {
44+
if (modelixBranchExists()) {
45+
val version = modelClient.lazyLoadVersion(modelixBranch)
46+
if (version.gitCommit == key.gitRevision) {
47+
return version
48+
}
49+
}
50+
check(jobIsRunning()) { "Import failed" }
51+
delay(3.seconds)
52+
}
53+
}
54+
55+
data class Key(
56+
val repo: GitRepositoryConfig,
57+
val gitBranchName: String,
58+
val gitRevision: String,
59+
val modelixBranchName: String,
60+
)
61+
}
62+
63+
val IVersion.gitCommit: String? get() = getAttributes()["git-commit"]

workspace-manager/src/main/kotlin/org/modelix/workspace/manager/WorkspacesController.kt renamed to workspace-manager/src/main/kotlin/org/modelix/services/workspaces/WorkspacesController.kt

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package org.modelix.workspace.manager
1+
package org.modelix.services.workspaces
22

33
import io.ktor.http.HttpStatusCode
44
import io.ktor.server.application.ApplicationCall
@@ -30,7 +30,13 @@ import org.modelix.services.workspaces.stubs.models.WorkspaceInstanceList
3030
import org.modelix.services.workspaces.stubs.models.WorkspaceInstanceState
3131
import org.modelix.services.workspaces.stubs.models.WorkspaceInstanceStateObject
3232
import org.modelix.services.workspaces.stubs.models.WorkspaceList
33-
import org.modelix.workspace.manager.WorkspaceJobQueue.Companion.HELM_PREFIX
33+
import org.modelix.workspace.manager.WorkspaceBuildManager
34+
import org.modelix.workspace.manager.WorkspaceInstancesManager
35+
import org.modelix.workspace.manager.WorkspaceJobQueue
36+
import org.modelix.workspace.manager.WorkspaceManager
37+
import org.modelix.workspace.manager.heapSizeFromContainerLimit
38+
import org.modelix.workspace.manager.putFile
39+
import org.modelix.workspace.manager.respondTarGz
3440
import org.modelix.workspaces.DEFAULT_MPS_VERSION
3541
import org.modelix.workspaces.WorkspaceConfigForBuild
3642
import org.modelix.workspaces.WorkspaceProgressItems
@@ -92,7 +98,13 @@ class WorkspacesController(
9298
val newWorkspace = workspaceConfig.copy(
9399
id = UUID.randomUUID().toString(),
94100
mpsVersion = workspaceConfig.mpsVersion.takeIf { it.isNotEmpty() } ?: DEFAULT_MPS_VERSION,
95-
memoryLimit = workspaceConfig.memoryLimit?.takeIf { it.isNotEmpty() }?.let { runCatching { Quantity(it).toSuffixedString() }.getOrNull() } ?: "2Gi",
101+
memoryLimit = workspaceConfig.memoryLimit?.takeIf { it.isNotEmpty() }?.let {
102+
runCatching {
103+
Quantity(
104+
it,
105+
).toSuffixedString()
106+
}.getOrNull()
107+
} ?: "2Gi",
96108
)
97109
manager.putWorkspace(newWorkspace)
98110
call.getUserName()?.let { manager.assignOwner(newWorkspace.id, it) }
@@ -258,11 +270,11 @@ class WorkspacesController(
258270
call.respondTarGz { tar ->
259271
@Suppress("ktlint")
260272
tar.putFile("Dockerfile", """
261-
FROM ${HELM_PREFIX}docker-registry:5000/modelix/workspace-client-baseimage:${System.getenv("MPS_BASEIMAGE_VERSION")}-mps$mpsVersion
273+
FROM ${WorkspaceJobQueue.HELM_PREFIX}docker-registry:5000/modelix/workspace-client-baseimage:${System.getenv("MPS_BASEIMAGE_VERSION")}-mps$mpsVersion
262274
263275
ENV modelix_workspace_id=${workspace.id}
264276
ENV modelix_workspace_task_id=${taskId}
265-
ENV modelix_workspace_server=http://${HELM_PREFIX}workspace-manager:28104/
277+
ENV modelix_workspace_server=http://${WorkspaceJobQueue.HELM_PREFIX}workspace-manager:28104/
266278
ENV INITIAL_JWT_TOKEN=$jwtToken
267279
268280
RUN /etc/cont-init.d/10-init-users.sh && /etc/cont-init.d/99-set-user-home.sh
@@ -284,7 +296,7 @@ class WorkspacesController(
284296
285297
RUN mkdir /config/home/job \
286298
&& cd /config/home/job \
287-
&& wget -q "http://${HELM_PREFIX}workspace-manager:28104/static/workspace-job.tar" \
299+
&& wget -q "http://${WorkspaceJobQueue.HELM_PREFIX}workspace-manager:28104/static/workspace-job.tar" \
288300
&& tar -xf workspace-job.tar \
289301
&& cd /mps-projects/workspace-${workspace.id} \
290302
&& /config/home/job/workspace-job/bin/workspace-job \

‎workspace-manager/src/main/kotlin/org/modelix/workspace/manager/KestraClient.kt‎

Lines changed: 35 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import kotlinx.serialization.json.jsonArray
2424
import kotlinx.serialization.json.jsonObject
2525
import kotlinx.serialization.json.jsonPrimitive
2626
import org.modelix.authorization.ModelixJWTUtil
27+
import org.modelix.model.lazy.BranchReference
2728
import org.modelix.model.lazy.RepositoryId
2829
import org.modelix.model.server.ModelServerPermissionSchema
2930
import org.modelix.workspaces.InternalWorkspaceConfig
@@ -42,13 +43,17 @@ class KestraClient(val jwtUtil: ModelixJWTUtil) {
4243
}
4344

4445
suspend fun getRunningImportJobIds(workspaceId: String): List<String> {
46+
return getRunningImportJobIds(mapOf("workspace" to workspaceId))
47+
}
48+
49+
suspend fun getRunningImportJobIds(labels: Map<String, String>): List<String> {
4550
val responseObject: JsonObject = httpClient.get {
4651
url {
4752
takeFrom(kestraApiEndpoint)
4853
appendPathSegments("executions", "search")
4954
parameters.append("namespace", "modelix")
5055
parameters.append("flowId", "git_import")
51-
parameters.append("labels", "workspace:$workspaceId")
56+
parameters.appendAll("labels", labels.map { "${it.key}:${it.value}" })
5257
parameters.append("state", "CREATED")
5358
parameters.append("state", "QUEUED")
5459
parameters.append("state", "RUNNING")
@@ -64,36 +69,53 @@ class KestraClient(val jwtUtil: ModelixJWTUtil) {
6469

6570
suspend fun enqueueGitImport(workspace: InternalWorkspaceConfig): JsonObject {
6671
val gitRepo = workspace.gitRepositories.first()
72+
return enqueueGitImport(
73+
gitRepoUrl = gitRepo.url,
74+
gitUser = gitRepo.credentials?.user,
75+
gitPassword = gitRepo.credentials?.password,
76+
gitRevision = "origin/${gitRepo.branch}",
77+
modelixBranch = RepositoryId("workspace_${workspace.id}").getBranchReference("git-import"),
78+
labels = mapOf("workspace" to workspace.id),
79+
)
80+
}
6781

82+
suspend fun enqueueGitImport(
83+
gitRepoUrl: String,
84+
gitUser: String?,
85+
gitPassword: String?,
86+
gitRevision: String,
87+
modelixBranch: BranchReference,
88+
labels: Map<String, String>,
89+
): JsonObject {
6890
updateGitImportFlow()
6991

70-
val targetBranch = RepositoryId("workspace_${workspace.id}").getBranchReference("git-import")
7192
val token = jwtUtil.createAccessToken(
7293
"git-import@modelix.org",
7394
listOf(
74-
ModelServerPermissionSchema.repository(targetBranch.repositoryId).create.fullId,
75-
ModelServerPermissionSchema.branch(targetBranch).rewrite.fullId,
95+
ModelServerPermissionSchema.repository(modelixBranch.repositoryId).create.fullId,
96+
ModelServerPermissionSchema.branch(modelixBranch).rewrite.fullId,
7697
),
7798
)
7899

79100
val response = httpClient.post {
80101
url {
81102
takeFrom(kestraApiEndpoint)
82103
appendPathSegments("executions", "modelix", "git_import")
83-
parameters["labels"] = "workspace:${workspace.id}"
104+
if (labels.isNotEmpty()) {
105+
parameters.appendAll("labels", labels.map { "${it.key}:${it.value}" })
106+
}
84107
}
85108
setBody(
86109
MultiPartFormDataContent(
87110
formData {
88-
append("git_url", gitRepo.url)
89-
append("git_revision", "origin/${gitRepo.branch}")
90-
append("modelix_repo_name", "workspace_${workspace.id}")
91-
append("modelix_target_branch", "git-import")
111+
append("git_url", gitRepoUrl)
112+
append("git_revision", gitRevision)
113+
append("git_limit", "50")
114+
append("modelix_repo_name", modelixBranch.repositoryId.id)
115+
append("modelix_target_branch", modelixBranch.branchName)
92116
append("token", token)
93-
gitRepo.credentials?.also { credentials ->
94-
append("git_user", credentials.user)
95-
append("git_pw", credentials.password)
96-
}
117+
gitUser?.let { append("git_user", it) }
118+
gitPassword?.let { append("git_pw", it) }
97119
},
98120
),
99121
)

0 commit comments

Comments
 (0)